--- a/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,23 +22,21 @@
# ignore the pygments UserWarnings
import warnings
-import cPickle
import zlib
warnings.filterwarnings('ignore', category=UserWarning,
message='.*was already imported',
module='.*pygments')
-import __builtin__
-# '_' is available in builtins to mark internationalized string but should
-# not be used to do the actual translation
-if not hasattr(__builtin__, '_'):
- __builtin__._ = unicode
+from six import PY2, binary_type, text_type
+from six.moves import builtins
CW_SOFTWARE_ROOT = __path__[0]
import sys, os, logging
-from StringIO import StringIO
+from io import BytesIO
+
+from six.moves import cPickle as pickle
from logilab.common.deprecation import deprecated
from logilab.common.logging_ext import set_log_methods
@@ -56,6 +54,14 @@
from cubicweb._exceptions import *
from logilab.common.registry import ObjectNotFound, NoSelectableObject, RegistryNotFound
+
+# '_' is available to mark internationalized string but should not be used to
+# do the actual translation
+_ = text_type
+if not hasattr(builtins, '_'):
+ builtins._ = deprecated("[3.22] Use 'from cubicweb import _'")(_)
+
+
# convert eid to the right type, raise ValueError if it's not a valid eid
@deprecated('[3.17] typed_eid() was removed. replace it with int() when needed.')
def typed_eid(eid):
@@ -66,17 +72,19 @@
#import threading
#threading.settrace(log_thread)
-class Binary(StringIO):
- """customize StringIO to make sure we don't use unicode"""
- def __init__(self, buf=''):
- assert isinstance(buf, (str, buffer, bytearray)), \
- "Binary objects must use raw strings, not %s" % buf.__class__
- StringIO.__init__(self, buf)
+class Binary(BytesIO):
+ """class to hold binary data. Use BytesIO to prevent use of unicode data"""
+ _allowed_types = (binary_type, bytearray, buffer if PY2 else memoryview)
+
+ def __init__(self, buf=b''):
+ assert isinstance(buf, self._allowed_types), \
+ "Binary objects must use bytes/buffer objects, not %s" % buf.__class__
+ super(Binary, self).__init__(buf)
def write(self, data):
- assert isinstance(data, (str, buffer, bytearray)), \
- "Binary objects must use raw strings, not %s" % data.__class__
- StringIO.write(self, data)
+ assert isinstance(data, self._allowed_types), \
+ "Binary objects must use bytes/buffer objects, not %s" % data.__class__
+ super(Binary, self).write(data)
def to_file(self, fobj):
"""write a binary to disk
@@ -132,22 +140,22 @@
def zpickle(cls, obj):
""" return a Binary containing a gzipped pickle of obj """
retval = cls()
- retval.write(zlib.compress(cPickle.dumps(obj, protocol=2)))
+ retval.write(zlib.compress(pickle.dumps(obj, protocol=2)))
return retval
def unzpickle(self):
""" decompress and loads the stream before returning it """
- return cPickle.loads(zlib.decompress(self.getvalue()))
+ return pickle.loads(zlib.decompress(self.getvalue()))
def check_password(eschema, value):
- return isinstance(value, (str, Binary))
+ return isinstance(value, (binary_type, Binary))
BASE_CHECKERS['Password'] = check_password
def str_or_binary(value):
if isinstance(value, Binary):
return value
- return str(value)
+ return binary_type(value)
BASE_CONVERTERS['Password'] = str_or_binary
@@ -255,4 +263,3 @@
not be processed, a memory allocation error occurred during processing,
etc.
"""
-
--- a/__pkginfo__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/__pkginfo__.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,8 +22,8 @@
modname = distname = "cubicweb"
-numversion = (3, 21, 5)
-version = '.'.join(str(num) for num in numversion)
+numversion = (3, 21, 99)
+version = '.'.join(str(num) for num in numversion) + '.dev0'
description = "a repository of entities / relations for knowledge management"
author = "Logilab"
@@ -39,10 +39,11 @@
]
__depends__ = {
+ 'six': '>= 1.4.0',
'logilab-common': '>= 0.63.1',
'logilab-mtconverter': '>= 0.8.0',
'rql': '>= 0.31.2',
- 'yams': '>= 0.40.0',
+ 'yams': '>= 0.41.1',
#gettext # for xgettext, msgcat, etc...
# web dependencies
'lxml': '',
--- a/_exceptions.py Tue Dec 15 19:05:41 2015 +0100
+++ b/_exceptions.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,6 +21,8 @@
from warnings import warn
+from six import PY3, text_type
+
from logilab.common.decorators import cachedproperty
from yams import ValidationError
@@ -30,23 +32,24 @@
class CubicWebException(Exception):
"""base class for cubicweb server exception"""
msg = ""
- def __str__(self):
+ def __unicode__(self):
if self.msg:
if self.args:
return self.msg % tuple(self.args)
else:
return self.msg
else:
- return u' '.join(unicode(arg) for arg in self.args)
+ return u' '.join(text_type(arg) for arg in self.args)
+ __str__ = __unicode__ if PY3 else lambda self: self.__unicode__().encode('utf-8')
class ConfigurationError(CubicWebException):
"""a misconfiguration error"""
class InternalError(CubicWebException):
- """base class for exceptions which should not occurs"""
+ """base class for exceptions which should not occur"""
class SecurityError(CubicWebException):
- """base class for cubicweb server security exception"""
+ """base class for cubicweb server security exceptions"""
class RepositoryError(CubicWebException):
"""base class for repository exceptions"""
@@ -185,7 +188,7 @@
commit time.
:type txuuix: int
- :param txuuid: Unique identifier of the partialy undone transaction
+ :param txuuid: Unique identifier of the partially undone transaction
:type errors: list
:param errors: List of errors occurred during undoing
@@ -204,4 +207,3 @@
# pylint: disable=W0611
from logilab.common.clcommands import BadCommandUsage
-
--- a/_gcdebug.py Tue Dec 15 19:05:41 2015 +0100
+++ b/_gcdebug.py Wed Dec 16 11:23:48 2015 +0100
@@ -15,6 +15,7 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+from __future__ import print_function
import gc, types, weakref
@@ -68,7 +69,7 @@
except KeyError:
ocounters[key] = 1
if isinstance(obj, viewreferrersclasses):
- print ' ', obj, referrers(obj, showobjs, maxlevel)
+ print(' ', obj, referrers(obj, showobjs, maxlevel))
garbage = [repr(obj) for obj in gc.garbage]
return counters, ocounters, garbage
--- a/crypto.py Tue Dec 15 19:05:41 2015 +0100
+++ b/crypto.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,9 +18,10 @@
"""Simple cryptographic routines, based on python-crypto."""
__docformat__ = "restructuredtext en"
-from pickle import dumps, loads
from base64 import b64encode, b64decode
+from six.moves import cPickle as pickle
+
from Crypto.Cipher import Blowfish
@@ -34,7 +35,7 @@
def encrypt(data, seed):
- string = dumps(data)
+ string = pickle.dumps(data)
string = string + '*' * (8 - len(string) % 8)
string = b64encode(_cypherer(seed).encrypt(string))
return unicode(string)
@@ -43,4 +44,4 @@
def decrypt(string, seed):
# pickle ignores trailing characters so we do not need to strip them off
string = _cypherer(seed).decrypt(b64decode(string))
- return loads(string)
+ return pickle.loads(string)
--- a/cubicweb.spec Tue Dec 15 19:05:41 2015 +0100
+++ b/cubicweb.spec Wed Dec 16 11:23:48 2015 +0100
@@ -20,10 +20,11 @@
BuildArch: noarch
Requires: %{python}
+Requires: %{python}-six >= 1.4.0
Requires: %{python}-logilab-common >= 0.63.1
Requires: %{python}-logilab-mtconverter >= 0.8.0
Requires: %{python}-rql >= 0.31.2
-Requires: %{python}-yams >= 0.40.0
+Requires: %{python}-yams >= 0.41.1
Requires: %{python}-logilab-database >= 1.13.0
Requires: %{python}-passlib
Requires: %{python}-lxml
--- a/cwconfig.py Tue Dec 15 19:05:41 2015 +0100
+++ b/cwconfig.py Wed Dec 16 11:23:48 2015 +0100
@@ -164,9 +164,9 @@
Directory where pid files will be written
"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
-_ = unicode
import sys
import os
@@ -179,6 +179,8 @@
basename, isdir, dirname, splitext)
from warnings import warn, filterwarnings
+from six import text_type
+
from logilab.common.decorators import cached, classproperty
from logilab.common.deprecation import deprecated
from logilab.common.logging_ext import set_log_methods, init_log
@@ -186,7 +188,7 @@
ConfigurationMixIn, merge_options)
from cubicweb import (CW_SOFTWARE_ROOT, CW_MIGRATION_MAP,
- ConfigurationError, Binary)
+ ConfigurationError, Binary, _)
from cubicweb.toolsutils import create_dir
CONFIGURATIONS = []
@@ -350,7 +352,7 @@
}),
('umask',
{'type' : 'int',
- 'default': 077,
+ 'default': 0o077,
'help': 'permission umask for files created by the server',
'group': 'main', 'level': 2,
}),
@@ -503,7 +505,7 @@
deps = {}
else:
deps = dict( (x[len('cubicweb-'):], v)
- for x, v in gendeps.iteritems()
+ for x, v in gendeps.items()
if x.startswith('cubicweb-'))
for depcube in deps:
try:
@@ -650,7 +652,7 @@
self.adjust_sys_path()
self.load_defaults()
# will be properly initialized later by _gettext_init
- self.translations = {'en': (unicode, lambda ctx, msgid: unicode(msgid) )}
+ self.translations = {'en': (text_type, lambda ctx, msgid: text_type(msgid) )}
self._site_loaded = set()
# don't register ReStructured Text directives by simple import, avoid pb
# with eg sphinx.
@@ -960,7 +962,7 @@
i = 1
while exists(path) and i < 100: # arbitrary limit to avoid infinite loop
try:
- file(path, 'a')
+ open(path, 'a')
break
except IOError:
path = '%s-%s.log' % (basepath, i)
@@ -994,6 +996,13 @@
rtdir = abspath(os.environ.get('CW_RUNTIME_DIR', default))
return join(rtdir, '%s-%s.pid' % (self.appid, self.name))
+ # config -> repository
+
+ def repository(self, vreg=None):
+ from cubicweb.server.repository import Repository
+ from cubicweb.server.utils import TasksManager
+ return Repository(self, TasksManager(), vreg=vreg)
+
# instance methods used to get instance specific resources #############
def __init__(self, appid, debugmode=False, creating=False):
@@ -1001,7 +1010,7 @@
# set to true while creating an instance
self.creating = creating
super(CubicWebConfiguration, self).__init__(debugmode)
- fake_gettext = (unicode, lambda ctx, msgid: unicode(msgid))
+ fake_gettext = (text_type, lambda ctx, msgid: text_type(msgid))
for lang in self.available_languages():
self.translations[lang] = fake_gettext
self._cubes = None
@@ -1050,7 +1059,8 @@
def save(self):
"""write down current configuration"""
- self.generate_config(open(self.main_config_file(), 'w'))
+ with open(self.main_config_file(), 'w') as fobj:
+ self.generate_config(fobj)
def check_writeable_uid_directory(self, path):
"""check given directory path exists, belongs to the user running the
@@ -1101,7 +1111,7 @@
version = self.cube_version(pkg)
infos.append('%s-%s' % (pkg, version))
infos.append('cubicweb-%s' % str(self.cubicweb_version()))
- return md5(';'.join(infos)).hexdigest()
+ return md5((';'.join(infos)).encode('ascii')).hexdigest()
def load_configuration(self, **kw):
"""load instance's configuration files"""
@@ -1156,7 +1166,7 @@
def _gettext_init(self):
"""set language for gettext"""
- from cubicweb.gettext import translation
+ from cubicweb.cwgettext import translation
path = join(self.apphome, 'i18n')
for language in self.available_languages():
self.info("loading language %s", language)
@@ -1181,13 +1191,8 @@
def set_sources_mode(self, sources):
if not 'all' in sources:
- print 'warning: ignoring specified sources, requires a repository '\
- 'configuration'
-
- def migration_handler(self):
- """return a migration handler instance"""
- from cubicweb.migration import MigrationHelper
- return MigrationHelper(self, verbosity=self.verbosity)
+ print('warning: ignoring specified sources, requires a repository '
+ 'configuration')
def i18ncompile(self, langs=None):
from cubicweb import i18n
--- a/cwctl.py Tue Dec 15 19:05:41 2015 +0100
+++ b/cwctl.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,6 +18,7 @@
"""the cubicweb-ctl tool, based on logilab.common.clcommands to
provide a pluggable commands system.
"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -28,7 +29,6 @@
from warnings import warn, filterwarnings
from os import remove, listdir, system, pathsep
from os.path import exists, join, isfile, isdir, dirname, abspath
-from urlparse import urlparse
try:
from os import kill, getpgid
@@ -38,6 +38,8 @@
def getpgid():
"""win32 getpgid implementation"""
+from six.moves.urllib.parse import urlparse
+
from logilab.common.clcommands import CommandLine
from logilab.common.shellutils import ASK
from logilab.common.configuration import merge_options
@@ -113,15 +115,15 @@
_allinstances = list_instances(regdir)
if isfile(join(regdir, 'startorder')):
allinstances = []
- for line in file(join(regdir, 'startorder')):
+ for line in open(join(regdir, 'startorder')):
line = line.strip()
if line and not line.startswith('#'):
try:
_allinstances.remove(line)
allinstances.append(line)
except ValueError:
- print ('ERROR: startorder file contains unexistant '
- 'instance %s' % line)
+ print('ERROR: startorder file contains unexistant '
+ 'instance %s' % line)
allinstances += _allinstances
else:
allinstances = _allinstances
@@ -146,7 +148,7 @@
status = 0
for appid in args:
if askconfirm:
- print '*'*72
+ print('*'*72)
if not ASK.confirm('%s instance %r ?' % (self.name, appid)):
continue
try:
@@ -184,13 +186,13 @@
forkcmd = None
for appid in args:
if askconfirm:
- print '*'*72
+ print('*'*72)
if not ASK.confirm('%s instance %r ?' % (self.name, appid)):
continue
if forkcmd:
status = system('%s %s' % (forkcmd, appid))
if status:
- print '%s exited with status %s' % (forkcmd, status)
+ print('%s exited with status %s' % (forkcmd, status))
else:
self.run_arg(appid)
@@ -224,19 +226,19 @@
from cubicweb.migration import ConfigurationProblem
if mode == 'all':
- print 'CubicWeb %s (%s mode)' % (cwcfg.cubicweb_version(), cwcfg.mode)
- print
+ print('CubicWeb %s (%s mode)' % (cwcfg.cubicweb_version(), cwcfg.mode))
+ print()
if mode in ('all', 'config', 'configurations'):
- print 'Available configurations:'
+ print('Available configurations:')
for config in CONFIGURATIONS:
- print '*', config.name
+ print('*', config.name)
for line in config.__doc__.splitlines():
line = line.strip()
if not line:
continue
- print ' ', line
- print
+ print(' ', line)
+ print()
if mode in ('all', 'cubes'):
cfgpb = ConfigurationProblem(cwcfg)
@@ -244,11 +246,11 @@
cubesdir = pathsep.join(cwcfg.cubes_search_path())
namesize = max(len(x) for x in cwcfg.available_cubes())
except ConfigurationError as ex:
- print 'No cubes available:', ex
+ print('No cubes available:', ex)
except ValueError:
- print 'No cubes available in %s' % cubesdir
+ print('No cubes available in %s' % cubesdir)
else:
- print 'Available cubes (%s):' % cubesdir
+ print('Available cubes (%s):' % cubesdir)
for cube in cwcfg.available_cubes():
try:
tinfo = cwcfg.cube_pkginfo(cube)
@@ -257,59 +259,59 @@
except (ConfigurationError, AttributeError) as ex:
tinfo = None
tversion = '[missing cube information: %s]' % ex
- print '* %s %s' % (cube.ljust(namesize), tversion)
+ print('* %s %s' % (cube.ljust(namesize), tversion))
if self.config.verbose:
if tinfo:
descr = getattr(tinfo, 'description', '')
if not descr:
descr = tinfo.__doc__
if descr:
- print ' '+ ' \n'.join(descr.splitlines())
+ print(' '+ ' \n'.join(descr.splitlines()))
modes = detect_available_modes(cwcfg.cube_dir(cube))
- print ' available modes: %s' % ', '.join(modes)
- print
+ print(' available modes: %s' % ', '.join(modes))
+ print()
if mode in ('all', 'instances'):
try:
regdir = cwcfg.instances_dir()
except ConfigurationError as ex:
- print 'No instance available:', ex
- print
+ print('No instance available:', ex)
+ print()
return
instances = list_instances(regdir)
if instances:
- print 'Available instances (%s):' % regdir
+ print('Available instances (%s):' % regdir)
for appid in instances:
modes = cwcfg.possible_configurations(appid)
if not modes:
- print '* %s (BROKEN instance, no configuration found)' % appid
+ print('* %s (BROKEN instance, no configuration found)' % appid)
continue
- print '* %s (%s)' % (appid, ', '.join(modes))
+ print('* %s (%s)' % (appid, ', '.join(modes)))
try:
config = cwcfg.config_for(appid, modes[0])
except Exception as exc:
- print ' (BROKEN instance, %s)' % exc
+ print(' (BROKEN instance, %s)' % exc)
continue
else:
- print 'No instance available in %s' % regdir
- print
+ print('No instance available in %s' % regdir)
+ print()
if mode == 'all':
# configuration management problem solving
cfgpb.solve()
if cfgpb.warnings:
- print 'Warnings:\n', '\n'.join('* '+txt for txt in cfgpb.warnings)
+ print('Warnings:\n', '\n'.join('* '+txt for txt in cfgpb.warnings))
if cfgpb.errors:
- print 'Errors:'
+ print('Errors:')
for op, cube, version, src in cfgpb.errors:
if op == 'add':
- print '* cube', cube,
+ print('* cube', cube, end=' ')
if version:
- print ' version', version,
- print 'is not installed, but required by %s' % src
+ print(' version', version, end=' ')
+ print('is not installed, but required by %s' % src)
else:
- print '* cube %s version %s is installed, but version %s is required by %s' % (
- cube, cfgpb.cubes[cube], version, src)
+ print('* cube %s version %s is installed, but version %s is required by %s' % (
+ cube, cfgpb.cubes[cube], version, src))
def check_options_consistency(config):
if config.automatic and config.config_level > 0:
@@ -380,20 +382,20 @@
templdirs = [cwcfg.cube_dir(cube)
for cube in cubes]
except ConfigurationError as ex:
- print ex
- print '\navailable cubes:',
- print ', '.join(cwcfg.available_cubes())
+ print(ex)
+ print('\navailable cubes:', end=' ')
+ print(', '.join(cwcfg.available_cubes()))
return
# create the registry directory for this instance
- print '\n'+underline_title('Creating the instance %s' % appid)
+ print('\n'+underline_title('Creating the instance %s' % appid))
create_dir(config.apphome)
# cubicweb-ctl configuration
if not self.config.automatic:
- print '\n'+underline_title('Configuring the instance (%s.conf)'
- % configname)
+ print('\n'+underline_title('Configuring the instance (%s.conf)'
+ % configname))
config.input_config('main', self.config.config_level)
# configuration'specific stuff
- print
+ print()
helper.bootstrap(cubes, self.config.automatic, self.config.config_level)
# input for cubes specific options
if not self.config.automatic:
@@ -402,23 +404,23 @@
and odict.get('level') <= self.config.config_level)
for section in sections:
if section not in ('main', 'email', 'web'):
- print '\n' + underline_title('%s options' % section)
+ print('\n' + underline_title('%s options' % section))
config.input_config(section, self.config.config_level)
# write down configuration
config.save()
self._handle_win32(config, appid)
- print '-> generated config %s' % config.main_config_file()
+ print('-> generated config %s' % config.main_config_file())
# handle i18n files structure
# in the first cube given
from cubicweb import i18n
langs = [lang for lang, _ in i18n.available_catalogs(join(templdirs[0], 'i18n'))]
errors = config.i18ncompile(langs)
if errors:
- print '\n'.join(errors)
+ print('\n'.join(errors))
if self.config.automatic \
or not ASK.confirm('error while compiling message catalogs, '
'continue anyway ?'):
- print 'creation not completed'
+ print('creation not completed')
return
# create the additional data directory for this instance
if config.appdatahome != config.apphome: # true in dev mode
@@ -427,9 +429,9 @@
if config['uid']:
from logilab.common.shellutils import chown
# this directory should be owned by the uid of the server process
- print 'set %s as owner of the data directory' % config['uid']
+ print('set %s as owner of the data directory' % config['uid'])
chown(config.appdatahome, config['uid'])
- print '\n-> creation done for %s\n' % repr(config.apphome)[1:-1]
+ print('\n-> creation done for %s\n' % repr(config.apphome)[1:-1])
if not self.config.no_db_create:
helper.postcreate(self.config.automatic, self.config.config_level)
@@ -487,7 +489,7 @@
if ex.errno != errno.ENOENT:
raise
confignames = ', '.join([config.name for config in configs])
- print '-> instance %s (%s) deleted.' % (appid, confignames)
+ print('-> instance %s (%s) deleted.' % (appid, confignames))
# instance commands ########################################################
@@ -551,7 +553,7 @@
the --force option."
raise ExecutionError(msg % (appid, pidf))
if helper.start_server(config) == 1:
- print 'instance %s started' % appid
+ print('instance %s started' % appid)
def init_cmdline_log_threshold(config, loglevel):
@@ -606,7 +608,7 @@
except OSError:
# already removed by twistd
pass
- print 'instance %s stopped' % appid
+ print('instance %s stopped' % appid)
class RestartInstanceCommand(StartInstanceCommand):
@@ -630,7 +632,7 @@
# get instances in startorder
for appid in args:
if askconfirm:
- print '*'*72
+ print('*'*72)
if not ASK.confirm('%s instance %r ?' % (self.name, appid)):
continue
StopInstanceCommand(self.logger).stop_instance(appid)
@@ -677,14 +679,14 @@
status = 0
for mode in cwcfg.possible_configurations(appid):
config = cwcfg.config_for(appid, mode)
- print '[%s-%s]' % (appid, mode),
+ print('[%s-%s]' % (appid, mode), end=' ')
try:
pidf = config['pid-file']
except KeyError:
- print 'buggy instance, pid file not specified'
+ print('buggy instance, pid file not specified')
continue
if not exists(pidf):
- print "doesn't seem to be running"
+ print("doesn't seem to be running")
status = 1
continue
pid = int(open(pidf).read().strip())
@@ -692,10 +694,10 @@
try:
getpgid(pid)
except OSError:
- print "should be running with pid %s but the process can not be found" % pid
+ print("should be running with pid %s but the process can not be found" % pid)
status = 1
continue
- print "running with pid %s" % (pid)
+ print("running with pid %s" % (pid))
return status
class UpgradeInstanceCommand(InstanceCommandFork):
@@ -756,7 +758,7 @@
)
def upgrade_instance(self, appid):
- print '\n' + underline_title('Upgrading the instance %s' % appid)
+ print('\n' + underline_title('Upgrading the instance %s' % appid))
from logilab.common.changelog import Version
config = cwcfg.config_for(appid)
instance_running = exists(config['pid-file'])
@@ -767,11 +769,11 @@
set_sources_mode(self.config.ext_sources or ('migration',))
# get instance and installed versions for the server and the componants
mih = config.migration_handler()
- repo = mih.repo_connect()
+ repo = mih.repo
vcconf = repo.get_versions()
helper = self.config_helper(config, required=False)
if self.config.force_cube_version:
- for cube, version in self.config.force_cube_version.iteritems():
+ for cube, version in self.config.force_cube_version.items():
vcconf[cube] = Version(version)
toupgrade = []
for cube in config.cubes():
@@ -797,30 +799,30 @@
# run cubicweb/componants migration scripts
if self.config.fs_only or toupgrade:
for cube, fromversion, toversion in toupgrade:
- print '-> migration needed from %s to %s for %s' % (fromversion, toversion, cube)
+ print('-> migration needed from %s to %s for %s' % (fromversion, toversion, cube))
with mih.cnx:
with mih.cnx.security_enabled(False, False):
mih.migrate(vcconf, reversed(toupgrade), self.config)
else:
- print '-> no data migration needed for instance %s.' % appid
+ print('-> no data migration needed for instance %s.' % appid)
# rewrite main configuration file
mih.rewrite_configuration()
mih.shutdown()
# handle i18n upgrade
if not self.i18nupgrade(config):
return
- print
+ print()
if helper:
helper.postupgrade(repo)
- print '-> instance migrated.'
+ print('-> instance migrated.')
if instance_running and not (CWDEV or self.config.nostartstop):
# restart instance through fork to get a proper environment, avoid
# uicfg pb (and probably gettext catalogs, to check...)
forkcmd = '%s start %s' % (sys.argv[0], appid)
status = system(forkcmd)
if status:
- print '%s exited with status %s' % (forkcmd, status)
- print
+ print('%s exited with status %s' % (forkcmd, status))
+ print()
def i18nupgrade(self, config):
# handle i18n upgrade:
@@ -832,10 +834,10 @@
langs = [lang for lang, _ in i18n.available_catalogs(join(templdir, 'i18n'))]
errors = config.i18ncompile(langs)
if errors:
- print '\n'.join(errors)
+ print('\n'.join(errors))
if not ASK.confirm('Error while compiling message catalogs, '
'continue anyway?'):
- print '-> migration not completed.'
+ print('-> migration not completed.')
return False
return True
@@ -856,10 +858,9 @@
config.quick_start = True
if hasattr(config, 'set_sources_mode'):
config.set_sources_mode(('migration',))
- repo = config.migration_handler().repo_connect()
- vcconf = repo.get_versions()
+ vcconf = config.repository().get_versions()
for key in sorted(vcconf):
- print key+': %s.%s.%s' % vcconf[key]
+ print(key+': %s.%s.%s' % vcconf[key])
class ShellCommand(Command):
"""Run an interactive migration shell on an instance. This is a python shell
@@ -940,9 +941,9 @@
repo = get_repository(appuri)
cnx = connect(repo, login=login, password=pwd, mulcnx=False)
except AuthenticationError as ex:
- print ex
+ print(ex)
except (KeyboardInterrupt, EOFError):
- print
+ print()
sys.exit(0)
else:
break
@@ -1003,7 +1004,7 @@
config.init_cubes(repo.get_cubes())
errors = config.i18ncompile()
if errors:
- print '\n'.join(errors)
+ print('\n'.join(errors))
class ListInstancesCommand(Command):
@@ -1015,7 +1016,7 @@
"""run the command with its specific arguments"""
regdir = cwcfg.instances_dir()
for appid in sorted(listdir(regdir)):
- print appid
+ print(appid)
class ListCubesCommand(Command):
@@ -1026,7 +1027,7 @@
def run(self, args):
"""run the command with its specific arguments"""
for cube in cwcfg.available_cubes():
- print cube
+ print(cube)
class ConfigureInstanceCommand(InstanceCommand):
"""Configure instance.
@@ -1048,7 +1049,7 @@
def configure_instance(self, appid):
if self.config.param is not None:
appcfg = cwcfg.config_for(appid)
- for key, value in self.config.param.iteritems():
+ for key, value in self.config.param.items():
try:
appcfg.global_set_option(key, value)
except KeyError:
@@ -1138,17 +1139,15 @@
def run(args):
"""command line tool"""
import os
- sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
- sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0)
filterwarnings('default', category=DeprecationWarning)
cwcfg.load_cwctl_plugins()
try:
CWCTL.run(args)
except ConfigurationError as err:
- print 'ERROR: ', err
+ print('ERROR: ', err)
sys.exit(1)
except ExecutionError as err:
- print err
+ print(err)
sys.exit(2)
if __name__ == '__main__':
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/cwgettext.py Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,118 @@
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+
+import gettext
+
+
+class cwGNUTranslations(gettext.GNUTranslations):
+ # The encoding of a msgctxt and a msgid in a .mo file is
+ # msgctxt + "\x04" + msgid (gettext version >= 0.15)
+ CONTEXT_ENCODING = "%s\x04%s"
+
+ def pgettext(self, context, message):
+ ctxt_msg_id = self.CONTEXT_ENCODING % (context, message)
+ missing = object()
+ tmsg = self._catalog.get(ctxt_msg_id, missing)
+ if tmsg is missing:
+ if self._fallback:
+ return self._fallback.pgettext(context, message)
+ return message
+ # Encode the Unicode tmsg back to an 8-bit string, if possible
+ if self._output_charset:
+ return tmsg.encode(self._output_charset)
+ elif self._charset:
+ return tmsg.encode(self._charset)
+ return tmsg
+
+ def lpgettext(self, context, message):
+ ctxt_msg_id = self.CONTEXT_ENCODING % (context, message)
+ missing = object()
+ tmsg = self._catalog.get(ctxt_msg_id, missing)
+ if tmsg is missing:
+ if self._fallback:
+ return self._fallback.lpgettext(context, message)
+ return message
+ if self._output_charset:
+ return tmsg.encode(self._output_charset)
+ return tmsg.encode(locale.getpreferredencoding())
+
+ def npgettext(self, context, msgid1, msgid2, n):
+ ctxt_msg_id = self.CONTEXT_ENCODING % (context, msgid1)
+ try:
+ tmsg = self._catalog[(ctxt_msg_id, self.plural(n))]
+ if self._output_charset:
+ return tmsg.encode(self._output_charset)
+ elif self._charset:
+ return tmsg.encode(self._charset)
+ return tmsg
+ except KeyError:
+ if self._fallback:
+ return self._fallback.npgettext(context, msgid1, msgid2, n)
+ if n == 1:
+ return msgid1
+ else:
+ return msgid2
+
+ def lnpgettext(self, context, msgid1, msgid2, n):
+ ctxt_msg_id = self.CONTEXT_ENCODING % (context, msgid1)
+ try:
+ tmsg = self._catalog[(ctxt_msg_id, self.plural(n))]
+ if self._output_charset:
+ return tmsg.encode(self._output_charset)
+ return tmsg.encode(locale.getpreferredencoding())
+ except KeyError:
+ if self._fallback:
+ return self._fallback.lnpgettext(context, msgid1, msgid2, n)
+ if n == 1:
+ return msgid1
+ else:
+ return msgid2
+
+ def upgettext(self, context, message):
+ ctxt_message_id = self.CONTEXT_ENCODING % (context, message)
+ missing = object()
+ tmsg = self._catalog.get(ctxt_message_id, missing)
+ if tmsg is missing:
+ # XXX logilab patch for compat w/ catalog generated by cw < 3.5
+ return self.ugettext(message)
+ if self._fallback:
+ return self._fallback.upgettext(context, message)
+ return unicode(message)
+ return tmsg
+
+ def unpgettext(self, context, msgid1, msgid2, n):
+ ctxt_message_id = self.CONTEXT_ENCODING % (context, msgid1)
+ try:
+ tmsg = self._catalog[(ctxt_message_id, self.plural(n))]
+ except KeyError:
+ if self._fallback:
+ return self._fallback.unpgettext(context, msgid1, msgid2, n)
+ if n == 1:
+ tmsg = unicode(msgid1)
+ else:
+ tmsg = unicode(msgid2)
+ return tmsg
+
+
+def translation(domain, localedir=None, languages=None,
+ class_=None, fallback=False, codeset=None):
+ if class_ is None:
+ class_ = cwGNUTranslations
+ return gettext.translation(domain, localedir=localedir,
+ languages=languages, class_=class_,
+ fallback=fallback, codeset=codeset)
--- a/cwvreg.py Tue Dec 15 19:05:41 2015 +0100
+++ b/cwvreg.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,7 +20,7 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
import sys
from os.path import join, dirname, realpath
@@ -28,6 +28,8 @@
from datetime import datetime, date, time, timedelta
from functools import reduce
+from six import text_type, binary_type
+
from logilab.common.decorators import cached, clear_cache
from logilab.common.deprecation import deprecated, class_deprecated
from logilab.common.modutils import cleanup_sys_modules
@@ -221,9 +223,9 @@
"""
obj = self.select(oid, req, rset=rset, **kwargs)
res = obj.render(**kwargs)
- if isinstance(res, unicode):
+ if isinstance(res, text_type):
return res.encode(req.encoding)
- assert isinstance(res, str)
+ assert isinstance(res, binary_type)
return res
def possible_views(self, req, rset=None, **kwargs):
@@ -382,7 +384,7 @@
return [item for item in super(CWRegistryStore, self).items()
if not item[0] in ('propertydefs', 'propertyvalues')]
def iteritems(self):
- return (item for item in super(CWRegistryStore, self).iteritems()
+ return (item for item in super(CWRegistryStore, self).items()
if not item[0] in ('propertydefs', 'propertyvalues'))
def values(self):
@@ -492,7 +494,7 @@
"""
self.schema = schema
for registry, regcontent in self.items():
- for objects in regcontent.itervalues():
+ for objects in regcontent.values():
for obj in objects:
obj.schema = schema
@@ -543,7 +545,7 @@
self.unregister(obj)
super(CWRegistryStore, self).initialization_completed()
if 'uicfg' in self: # 'uicfg' is not loaded in a pure repository mode
- for rtags in self['uicfg'].itervalues():
+ for rtags in self['uicfg'].values():
for rtag in rtags:
# don't check rtags if we don't want to cleanup_unused_appobjects
rtag.init(self.schema, check=self.config.cleanup_unused_appobjects)
@@ -576,7 +578,7 @@
if withsitewide:
return sorted(k for k in self['propertydefs']
if not k.startswith('sources.'))
- return sorted(k for k, kd in self['propertydefs'].iteritems()
+ return sorted(k for k, kd in self['propertydefs'].items()
if not kd['sitewide'] and not k.startswith('sources.'))
def register_property(self, key, type, help, default=None, vocabulary=None,
@@ -653,4 +655,3 @@
'TZTime': time,
'Interval': timedelta,
})
-
--- a/dataimport/csv.py Tue Dec 15 19:05:41 2015 +0100
+++ b/dataimport/csv.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,18 +16,20 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""Functions to help importing CSV data"""
+from __future__ import absolute_import, print_function
-from __future__ import absolute_import
-
+import codecs
import csv as csvmod
import warnings
import os.path as osp
+from six import PY2, PY3, string_types
+
from logilab.common import shellutils
def count_lines(stream_or_filename):
- if isinstance(stream_or_filename, basestring):
+ if isinstance(stream_or_filename, string_types):
f = open(stream_or_filename)
else:
f = stream_or_filename
@@ -48,10 +50,8 @@
if quote is not None:
quotechar = quote
warnings.warn("[3.20] 'quote' kwarg is deprecated, use 'quotechar' instead")
- if isinstance(stream_or_path, basestring):
- if not osp.exists(stream_or_path):
- raise Exception("file doesn't exists: %s" % stream_or_path)
- stream = open(stream_or_path)
+ if isinstance(stream_or_path, string_types):
+ stream = open(stream_or_path, 'rb')
else:
stream = stream_or_path
rowcount = count_lines(stream)
@@ -64,7 +64,7 @@
yield urow
if withpb:
pb.update()
- print ' %s rows imported' % rowcount
+ print(' %s rows imported' % rowcount)
def ucsvreader(stream, encoding='utf-8', delimiter=',', quotechar='"',
@@ -77,6 +77,8 @@
separators) will be skipped. This is useful for Excel exports which may be
full of such lines.
"""
+ if PY3:
+ stream = codecs.getreader(encoding)(stream)
if separator is not None:
delimiter = separator
warnings.warn("[3.20] 'separator' kwarg is deprecated, use 'delimiter' instead")
@@ -86,28 +88,33 @@
it = iter(csvmod.reader(stream, delimiter=delimiter, quotechar=quotechar))
if not ignore_errors:
if skipfirst:
- it.next()
+ next(it)
for row in it:
- decoded = [item.decode(encoding) for item in row]
+ if PY2:
+ decoded = [item.decode(encoding) for item in row]
+ else:
+ decoded = row
if not skip_empty or any(decoded):
yield decoded
else:
if skipfirst:
try:
- row = it.next()
+ row = next(it)
except csvmod.Error:
pass
# Safe version, that can cope with error in CSV file
while True:
try:
- row = it.next()
+ row = next(it)
# End of CSV, break
except StopIteration:
break
# Error in CSV, ignore line and continue
except csvmod.Error:
continue
- decoded = [item.decode(encoding) for item in row]
+ if PY2:
+ decoded = [item.decode(encoding) for item in row]
+ else:
+ decoded = row
if not skip_empty or any(decoded):
yield decoded
-
--- a/dataimport/deprecated.py Tue Dec 15 19:05:41 2015 +0100
+++ b/dataimport/deprecated.py Wed Dec 16 11:23:48 2015 +0100
@@ -58,10 +58,13 @@
.. BUG file with one column are not parsable
.. TODO rollback() invocation is not possible yet
"""
+from __future__ import print_function
import sys
import traceback
-from StringIO import StringIO
+from io import StringIO
+
+from six import add_metaclass
from logilab.common import attrdict, shellutils
from logilab.common.date import strptime
@@ -78,7 +81,7 @@
>>> data = lazytable(ucsvreader(open(filename)))
"""
- header = reader.next()
+ header = next(reader)
for row in reader:
yield dict(zip(header, row))
@@ -103,7 +106,7 @@
@deprecated('[3.21] deprecated')
def tell(msg):
- print msg
+ print(msg)
@deprecated('[3.21] deprecated')
@@ -115,9 +118,9 @@
return answer == 'Y'
+@add_metaclass(class_deprecated)
class catch_error(object):
"""Helper for @contextmanager decorator."""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.21] deprecated'
def __init__(self, ctl, key='unexpected error', msg=None):
@@ -166,7 +169,9 @@
if res[dest] is None:
break
except ValueError as err:
- raise ValueError('error with %r field: %s' % (src, err)), None, sys.exc_info()[-1]
+ exc = ValueError('error with %r field: %s' % (src, err))
+ exc.__traceback__ = sys.exc_info()[-1]
+ raise exc
return res
@@ -254,6 +259,7 @@
if k is not None and len(v) > 1]
+@add_metaclass(class_deprecated)
class ObjectStore(object):
"""Store objects in memory for *faster* validation (development mode)
@@ -264,7 +270,6 @@
>>> group = store.prepare_insert_entity('CWUser', name=u'unknown')
>>> store.prepare_insert_relation(user, 'in_group', group)
"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.21] use the new importer API'
def __init__(self):
@@ -289,7 +294,7 @@
"""Given an entity type and eid, updates the corresponding fake entity with specified
attributes and inlined relations.
"""
- assert eid in self.types[etype], 'Trying to update with wrong type {}'.format(etype)
+ assert eid in self.types[etype], 'Trying to update with wrong type %s' % etype
data = self.eids[eid]
data.update(kwargs)
@@ -335,6 +340,7 @@
self.prepare_insert_relation(eid_from, rtype, eid_to, **kwargs)
+@add_metaclass(class_deprecated)
class CWImportController(object):
"""Controller of the data import process.
@@ -343,7 +349,6 @@
>>> ctl.data = dict_of_data_tables
>>> ctl.run()
"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.21] use the new importer API'
def __init__(self, store, askerror=0, catcherrors=None, tell=tell,
@@ -421,7 +426,7 @@
self.tell(pformat(sorted(error[1])))
def _print_stats(self):
- nberrors = sum(len(err) for err in self.errors.itervalues())
+ nberrors = sum(len(err) for err in self.errors.values())
self.tell('\nImport statistics: %i entities, %i types, %i relations and %i errors'
% (self.store.nb_inserted_entities,
self.store.nb_inserted_types,
@@ -456,5 +461,3 @@
return callfunc_every(self.store.commit,
self.commitevery,
self.get_data(datakey))
-
-
--- a/dataimport/importer.py Tue Dec 15 19:05:41 2015 +0100
+++ b/dataimport/importer.py Wed Dec 16 11:23:48 2015 +0100
@@ -69,7 +69,7 @@
def use_extid_as_cwuri_filter(extentities):
for extentity in extentities:
if extentity.extid not in extid2eid:
- extentity.values.setdefault('cwuri', set([unicode(extentity.extid)]))
+ extentity.values.setdefault('cwuri', set([extentity.extid.decode('utf-8')]))
yield extentity
return use_extid_as_cwuri_filter
@@ -83,15 +83,15 @@
def __init__(self, cnx, source=None):
self.cnx = cnx
- self._rql_template = 'Any S,O WHERE S {} O'
+ self._rql_template = 'Any S,O WHERE S %s O'
self._kwargs = {}
if source is not None:
- self._rql_template += ', S cw_source SO, O cw_source SO, SO eid %(s)s'
+ self._rql_template += ', S cw_source SO, O cw_source SO, SO eid %%(s)s'
self._kwargs['s'] = source.eid
def __getitem__(self, rtype):
"""Return a set of (subject, object) eids already related by `rtype`"""
- rql = self._rql_template.format(rtype)
+ rql = self._rql_template % rtype
return set(tuple(x) for x in self.cnx.execute(rql, self._kwargs))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dataimport/massive_store.py Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,742 @@
+# coding: utf-8
+# copyright 2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+from datetime import datetime
+from collections import defaultdict
+from io import StringIO
+
+from six.moves import range
+
+from yams.constraints import SizeConstraint
+
+from psycopg2 import ProgrammingError
+
+from cubicweb.dataimport import stores, pgstore
+from cubicweb.utils import make_uid
+from cubicweb.server.sqlutils import SQL_PREFIX
+
+
+class MassiveObjectStore(stores.RQLObjectStore):
+ """
+ Store for massive import of data, with delayed insertion of meta data.
+
+ WARNINGS:
+ - This store may be only used with PostgreSQL for now, as it relies
+ on the COPY FROM method, and on specific PostgreSQL tables to get all
+ the indexes.
+ - This store can only insert relations that are not inlined (i.e.,
+ which do *not* have inlined=True in their definition in the schema).
+
+ It should be used as follows:
+
+ store = MassiveObjectStore(cnx)
+ store.init_rtype_table('Person', 'lives_in', 'Location')
+ ...
+
+ store.prepare_insert_entity('Person', subj_iid_attribute=person_iid, ...)
+ store.prepare_insert_entity('Location', obj_iid_attribute=location_iid, ...)
+ ...
+
+ # subj_iid_attribute and obj_iid_attribute are argument names
+ # chosen by the user (e.g. "cwuri"). These names can be identical.
+ # person_iid and location_iid are unique IDs and depend on the data
+ # (e.g URI).
+ store.flush()
+ store.relate_by_iid(person_iid, 'lives_in', location_iid)
+ # For example:
+ store.prepare_insert_entity('Person',
+ cwuri='http://dbpedia.org/toto',
+ name='Toto')
+ store.prepare_insert_entity('Location',
+ uri='http://geonames.org/11111',
+ name='Somewhere')
+ store.flush()
+ store.relate_by_iid('http://dbpedia.org/toto',
+ 'lives_in',
+ 'http://geonames.org/11111')
+ # Finally
+ store.convert_relations('Person', 'lives_in', 'Location',
+ 'subj_iid_attribute', 'obj_iid_attribute')
+ # For the previous example:
+ store.convert_relations('Person', 'lives_in', 'Location', 'cwuri', 'uri')
+ ...
+ store.commit()
+ store.finish()
+ """
+ # size of eid range reserved by the store for each batch
+ eids_seq_range = 10000
+ # initial eid (None means use the value in the db)
+ eids_seq_start = None
+ # max size of the iid, used to create the iid_eid conversion table
+ iid_maxsize = 1024
+
+ def __init__(self, cnx,
+ on_commit_callback=None, on_rollback_callback=None,
+ slave_mode=False,
+ source=None):
+ """ Create a MassiveObject store, with the following attributes:
+
+ - cnx: CubicWeb cnx
+ """
+ super(MassiveObjectStore, self).__init__(cnx)
+ self.logger = logging.getLogger('dataimport.massive_store')
+ self._cnx = cnx
+ self.sql = cnx.system_sql
+ self._data_uri_relations = defaultdict(list)
+
+ # etypes for which we have a uri_eid_%(etype)s table
+ self._init_uri_eid = set()
+ # etypes for which we have a uri_eid_%(e)s_idx index
+ self._uri_eid_inserted = set()
+ # set of rtypes for which we have a %(rtype)s_relation_iid_tmp table
+ self._uri_rtypes = set()
+ # set of etypes whose tables are created
+ self._entities = set()
+ # set of rtypes for which we have a %(rtype)s_relation_tmp table
+ self._rtypes = set()
+
+ self.slave_mode = slave_mode
+ self.default_values = get_default_values(cnx.vreg.schema)
+ pg_schema = cnx.repo.config.system_source_config.get('db-namespace') or 'public'
+ self._dbh = PGHelper(self._cnx, pg_schema)
+ self._data_entities = defaultdict(list)
+ self._data_relations = defaultdict(list)
+ self._now = datetime.now()
+ self._default_cwuri = make_uid('_auto_generated')
+ self._count_cwuri = 0
+ self.on_commit_callback = on_commit_callback
+ self.on_rollback_callback = on_rollback_callback
+ # Do our meta tables already exist?
+ self._init_massive_metatables()
+ # Internal markers of initialization
+ if self.eids_seq_start is not None and not self.slave_mode:
+ self._cnx.system_sql(self._cnx.repo.system_source.dbhelper.sql_restart_numrange(
+ 'entities_id_seq', initial_value=self.eids_seq_start + 1))
+ cnx.commit()
+ self.get_next_eid = lambda g=self._get_eid_gen(): next(g)
+ # recreate then when self.finish() is called
+ if not self.slave_mode:
+ self._drop_metatables_constraints()
+ if source is None:
+ source = cnx.repo.system_source
+ self.source = source
+ self._etype_eid_idx = dict(cnx.execute('Any XN,X WHERE X is CWEType, X name XN'))
+ cnx.read_security = False
+ cnx.write_security = False
+
+ ### INIT FUNCTIONS ########################################################
+
+ def init_rtype_table(self, etype_from, rtype, etype_to):
+ """ Build temporary table for standard rtype """
+ # Create an uri_eid table for each etype for a better
+ # control of which etype is concerned by a particular
+ # possibly multivalued relation.
+ for etype in (etype_from, etype_to):
+ if etype and etype not in self._init_uri_eid:
+ self._init_uri_eid_table(etype)
+ if rtype not in self._uri_rtypes:
+ # Create the temporary table
+ if not self._cnx.repo.schema.rschema(rtype).inlined:
+ try:
+ sql = 'CREATE TABLE %(r)s_relation_iid_tmp (uri_from character ' \
+ 'varying(%(s)s), uri_to character varying(%(s)s))'
+ self.sql(sql % {'r': rtype, 's': self.iid_maxsize})
+ except ProgrammingError:
+ # XXX Already exist (probably due to multiple import)
+ pass
+ else:
+ self.logger.warning("inlined relation %s: cannot insert it", rtype)
+ # Add it to the initialized set
+ self._uri_rtypes.add(rtype)
+
+ def _init_uri_eid_table(self, etype):
+ """ Build a temporary table for id/eid convertion
+ """
+ try:
+ sql = "CREATE TABLE uri_eid_%(e)s (uri character varying(%(size)s), eid integer)"
+ self.sql(sql % {'e': etype.lower(), 'size': self.iid_maxsize,})
+ except ProgrammingError:
+ # XXX Already exist (probably due to multiple import)
+ pass
+ # Add it to the initialized set
+ self._init_uri_eid.add(etype)
+
+ def _init_massive_metatables(self):
+ # Check if our tables are not already created (i.e. a restart)
+ self._initialized_table_created = self._dbh.table_exists('cwmassive_initialized')
+ self._constraint_table_created = self._dbh.table_exists('cwmassive_constraints')
+ self._metadata_table_created = self._dbh.table_exists('cwmassive_metadata')
+
+ ### RELATE FUNCTION #######################################################
+
+ def relate_by_iid(self, iid_from, rtype, iid_to):
+ """Add new relation based on the internal id (iid)
+ of the entities (not the eid)"""
+ # Push data
+ if isinstance(iid_from, unicode):
+ iid_from = iid_from.encode('utf-8')
+ if isinstance(iid_to, unicode):
+ iid_to = iid_to.encode('utf-8')
+ self._data_uri_relations[rtype].append({'uri_from': iid_from, 'uri_to': iid_to})
+
+ ### FLUSH FUNCTIONS #######################################################
+
+ def flush_relations(self):
+ """ Flush the relations data
+ """
+ for rtype, data in self._data_uri_relations.items():
+ if not data:
+ self.logger.info('No data for rtype %s', rtype)
+ buf = StringIO('\n'.join(['%(uri_from)s\t%(uri_to)s' % d for d in data]))
+ if not buf:
+ self.logger.info('Empty Buffer for rtype %s', rtype)
+ continue
+ cursor = self._cnx.cnxset.cu
+ if not self._cnx.repo.schema.rschema(rtype).inlined:
+ cursor.copy_from(buf, '%s_relation_iid_tmp' % rtype.lower(),
+ null='NULL', columns=('uri_from', 'uri_to'))
+ else:
+ self.logger.warning("inlined relation %s: cannot insert it", rtype)
+ buf.close()
+ # Clear data cache
+ self._data_uri_relations[rtype] = []
+
+ def fill_uri_eid_table(self, etype, uri_label):
+ """ Fill the uri_eid table
+ """
+ self.logger.info('Fill uri_eid for etype %s', etype)
+ sql = 'INSERT INTO uri_eid_%(e)s SELECT cw_%(l)s, cw_eid FROM cw_%(e)s'
+ self.sql(sql % {'l': uri_label, 'e': etype.lower()})
+ # Add indexes
+ self.sql('CREATE INDEX uri_eid_%(e)s_idx ON uri_eid_%(e)s' '(uri)' % {'e': etype.lower()})
+ # Set the etype as converted
+ self._uri_eid_inserted.add(etype)
+
+ def convert_relations(self, etype_from, rtype, etype_to,
+ uri_label_from='cwuri', uri_label_to='cwuri'):
+ """ Flush the converted relations
+ """
+ # Always flush relations to be sure
+ self.logger.info('Convert relations %s %s %s', etype_from, rtype, etype_to)
+ self.flush_relations()
+ if uri_label_from and etype_from not in self._uri_eid_inserted:
+ self.fill_uri_eid_table(etype_from, uri_label_from)
+ if uri_label_to and etype_to not in self._uri_eid_inserted:
+ self.fill_uri_eid_table(etype_to, uri_label_to)
+ if self._cnx.repo.schema.rschema(rtype).inlined:
+ self.logger.warning("Can't insert inlined relation %s", rtype)
+ return
+ if uri_label_from and uri_label_to:
+ sql = '''INSERT INTO %(r)s_relation (eid_from, eid_to) SELECT DISTINCT O1.eid, O2.eid
+ FROM %(r)s_relation_iid_tmp AS T, uri_eid_%(ef)s as O1, uri_eid_%(et)s as O2
+ WHERE O1.uri=T.uri_from AND O2.uri=T.uri_to AND NOT EXISTS (
+ SELECT 1 FROM %(r)s_relation AS TT WHERE TT.eid_from=O1.eid AND TT.eid_to=O2.eid);
+ '''
+ elif uri_label_to:
+ sql = '''INSERT INTO %(r)s_relation (eid_from, eid_to) SELECT DISTINCT
+ CAST(T.uri_from AS INTEGER), O1.eid
+ FROM %(r)s_relation_iid_tmp AS T, uri_eid_%(et)s as O1
+ WHERE O1.uri=T.uri_to AND NOT EXISTS (
+ SELECT 1 FROM %(r)s_relation AS TT WHERE
+ TT.eid_from=CAST(T.uri_from AS INTEGER) AND TT.eid_to=O1.eid);
+ '''
+ elif uri_label_from:
+ sql = '''INSERT INTO %(r)s_relation (eid_from, eid_to) SELECT DISTINCT O1.eid, T.uri_to
+ O1.eid, CAST(T.uri_to AS INTEGER)
+ FROM %(r)s_relation_iid_tmp AS T, uri_eid_%(ef)s as O1
+ WHERE O1.uri=T.uri_from AND NOT EXISTS (
+ SELECT 1 FROM %(r)s_relation AS TT WHERE
+ TT.eid_from=O1.eid AND TT.eid_to=CAST(T.uri_to AS INTEGER));
+ '''
+ try:
+ self.sql(sql % {'r': rtype.lower(),
+ 'et': etype_to.lower() if etype_to else u'',
+ 'ef': etype_from.lower() if etype_from else u''})
+ except Exception as ex:
+ self.logger.error("Can't insert relation %s: %s", rtype, ex)
+
+ ### SQL UTILITIES #########################################################
+
+ def drop_and_store_indexes_constraints(self, tablename):
+ # Drop indexes and constraints
+ if not self._constraint_table_created:
+ # Create a table to save the constraints
+ # Allow reload even after crash
+ sql = "CREATE TABLE cwmassive_constraints (origtable text, query text, type varchar(256))"
+ self.sql(sql)
+ self._constraint_table_created = True
+ self._drop_table_constraints_indexes(tablename)
+
+ def _drop_table_constraints_indexes(self, tablename):
+ """ Drop and store table constraints and indexes """
+ indexes, constraints = self._dbh.application_indexes_constraints(tablename)
+ for name, query in constraints.items():
+ sql = 'INSERT INTO cwmassive_constraints VALUES (%(e)s, %(c)s, %(t)s)'
+ self.sql(sql, {'e': tablename, 'c': query, 't': 'constraint'})
+ sql = 'ALTER TABLE %s DROP CONSTRAINT %s CASCADE' % (tablename, name)
+ self.sql(sql)
+ for name, query in indexes.items():
+ sql = 'INSERT INTO cwmassive_constraints VALUES (%(e)s, %(c)s, %(t)s)'
+ self.sql(sql, {'e': tablename, 'c': query, 't': 'index'})
+ sql = 'DROP INDEX %s' % name
+ self.sql(sql)
+
+ def reapply_constraint_index(self, tablename):
+ if not self._dbh.table_exists('cwmassive_constraints'):
+ self.logger.info('The table cwmassive_constraints does not exist')
+ return
+ sql = 'SELECT query FROM cwmassive_constraints WHERE origtable = %(e)s'
+ crs = self.sql(sql, {'e': tablename})
+ for query, in crs.fetchall():
+ self.sql(query)
+ self.sql('DELETE FROM cwmassive_constraints WHERE origtable = %(e)s '
+ 'AND query = %(q)s', {'e': tablename, 'q': query})
+
+ def _drop_metatables_constraints(self):
+ """ Drop all the constraints for the meta data"""
+ for tablename in ('created_by_relation', 'owned_by_relation',
+ 'is_instance_of_relation', 'is_relation',
+ 'entities'):
+ self.drop_and_store_indexes_constraints(tablename)
+
+ def _create_metatables_constraints(self):
+ """ Create all the constraints for the meta data"""
+ for tablename in ('entities',
+ 'created_by_relation', 'owned_by_relation',
+ 'is_instance_of_relation', 'is_relation'):
+ # Indexes and constraints
+ self.reapply_constraint_index(tablename)
+
+ def init_relation_table(self, rtype):
+ """ Get and remove all indexes for performance sake """
+ # Create temporary table
+ if not self.slave_mode and rtype not in self._rtypes:
+ sql = "CREATE TABLE %s_relation_tmp (eid_from integer, eid_to integer)" % rtype.lower()
+ self.sql(sql)
+ # Drop indexes and constraints
+ tablename = '%s_relation' % rtype.lower()
+ self.drop_and_store_indexes_constraints(tablename)
+ # Push the etype in the initialized table for easier restart
+ self.init_create_initialized_table()
+ sql = 'INSERT INTO cwmassive_initialized VALUES (%(e)s, %(t)s)'
+ self.sql(sql, {'e': rtype, 't': 'rtype'})
+ # Mark rtype as "initialized" for faster check
+ self._rtypes.add(rtype)
+
+ def init_create_initialized_table(self):
+ """ Create the cwmassive initialized table
+ """
+ if not self._initialized_table_created:
+ sql = "CREATE TABLE cwmassive_initialized (retype text, type varchar(128))"
+ self.sql(sql)
+ self._initialized_table_created = True
+
+ def init_etype_table(self, etype):
+ """ Add eid sequence to a particular etype table and
+ remove all indexes for performance sake """
+ if etype not in self._entities:
+ # Only for non-initialized etype and not slave mode store
+ if not self.slave_mode:
+ if self.eids_seq_range is None:
+ # Eids are directly set by the entities_id_seq.
+ # We attach this sequence to all the created etypes.
+ sql = ("ALTER TABLE cw_%s ALTER COLUMN cw_eid "
+ "SET DEFAULT nextval('entities_id_seq')" % etype.lower())
+ self.sql(sql)
+ # Drop indexes and constraints
+ tablename = 'cw_%s' % etype.lower()
+ self.drop_and_store_indexes_constraints(tablename)
+ # Push the etype in the initialized table for easier restart
+ self.init_create_initialized_table()
+ sql = 'INSERT INTO cwmassive_initialized VALUES (%(e)s, %(t)s)'
+ self.sql(sql, {'e': etype, 't': 'etype'})
+ # Mark etype as "initialized" for faster check
+ self._entities.add(etype)
+
+
+ ### ENTITIES CREATION #####################################################
+
+ def _get_eid_gen(self):
+ """ Function getting the next eid. This is done by preselecting
+ a given number of eids from the 'entities_id_seq', and then
+ storing them"""
+ while True:
+ last_eid = self._cnx.repo.system_source.create_eid(self._cnx, self.eids_seq_range)
+ for eid in range(last_eid - self.eids_seq_range + 1, last_eid + 1):
+ yield eid
+
+ def _apply_default_values(self, etype, kwargs):
+ """Apply the default values for a given etype, attribute and value."""
+ default_values = self.default_values[etype]
+ missing_keys = set(default_values) - set(kwargs)
+ kwargs.update((key, default_values[key]) for key in missing_keys)
+
+ # store api ################################################################
+
+ def prepare_insert_entity(self, etype, **kwargs):
+ """Given an entity type, attributes and inlined relations, returns the inserted entity's
+ eid.
+ """
+ # Init the table if necessary
+ self.init_etype_table(etype)
+ # Add meta data if not given
+ if 'modification_date' not in kwargs:
+ kwargs['modification_date'] = self._now
+ if 'creation_date' not in kwargs:
+ kwargs['creation_date'] = self._now
+ if 'cwuri' not in kwargs:
+ kwargs['cwuri'] = self._default_cwuri + str(self._count_cwuri)
+ self._count_cwuri += 1
+ if 'eid' not in kwargs and self.eids_seq_range is not None:
+ # If eid is not given and the eids sequence is set,
+ # use the value from the sequence
+ kwargs['eid'] = self.get_next_eid()
+ self._apply_default_values(etype, kwargs)
+ self._data_entities[etype].append(kwargs)
+ return kwargs.get('eid')
+
+ def prepare_insert_relation(self, eid_from, rtype, eid_to, **kwargs):
+ """Insert into the database a relation ``rtype`` between entities with eids ``eid_from``
+ and ``eid_to``.
+ """
+ # Init the table if necessary
+ self.init_relation_table(rtype)
+ self._data_relations[rtype].append({'eid_from': eid_from, 'eid_to': eid_to})
+
+ def flush(self):
+ """Flush the data"""
+ self.flush_entities()
+ self.flush_internal_relations()
+ self.flush_relations()
+
+ def commit(self):
+ """Commit the database transaction."""
+ self.on_commit()
+ super(MassiveObjectStore, self).commit()
+
+ def finish(self):
+ """Remove temporary tables and columns."""
+ self.logger.info("Start cleaning")
+ if self.slave_mode:
+ raise RuntimeError('Store cleanup is not allowed in slave mode')
+ self.logger.info("Start cleaning")
+ # Cleanup relations tables
+ for etype in self._init_uri_eid:
+ self.sql('DROP TABLE uri_eid_%s' % etype.lower())
+ # Remove relations tables
+ for rtype in self._uri_rtypes:
+ if not self._cnx.repo.schema.rschema(rtype).inlined:
+ self.sql('DROP TABLE %(r)s_relation_iid_tmp' % {'r': rtype})
+ else:
+ self.logger.warning("inlined relation %s: no cleanup to be done for it" % rtype)
+ # Get all the initialized etypes/rtypes
+ if self._dbh.table_exists('cwmassive_initialized'):
+ crs = self.sql('SELECT retype, type FROM cwmassive_initialized')
+ for retype, _type in crs.fetchall():
+ self.logger.info('Cleanup for %s' % retype)
+ if _type == 'etype':
+ # Cleanup entities tables - Recreate indexes
+ self._cleanup_entities(retype)
+ elif _type == 'rtype':
+ # Cleanup relations tables
+ self._cleanup_relations(retype)
+ self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s',
+ {'e': retype})
+ # Create meta constraints (entities, is_instance_of, ...)
+ self._create_metatables_constraints()
+ # Delete the meta data table
+ for table_name in ('cwmassive_initialized', 'cwmassive_constraints', 'cwmassive_metadata'):
+ if self._dbh.table_exists(table_name):
+ self.sql('DROP TABLE %s' % table_name)
+ self.commit()
+
+ ### FLUSH #################################################################
+
+ def on_commit(self):
+ if self.on_commit_callback:
+ self.on_commit_callback()
+
+ def on_rollback(self, exc, etype, data):
+ if self.on_rollback_callback:
+ self.on_rollback_callback(exc, etype, data)
+ self._cnx.rollback()
+ else:
+ raise exc
+
+ def flush_internal_relations(self):
+ """ Flush the relations data
+ """
+ for rtype, data in self._data_relations.items():
+ if not data:
+ # There is no data for these etype for this flush round.
+ continue
+ buf = pgstore._create_copyfrom_buffer(data, ('eid_from', 'eid_to'))
+ if not buf:
+ # The buffer is empty. This is probably due to error in _create_copyfrom_buffer
+ raise ValueError
+ cursor = self._cnx.cnxset.cu
+ # Push into the tmp table
+ cursor.copy_from(buf, '%s_relation_tmp' % rtype.lower(),
+ null='NULL', columns=('eid_from', 'eid_to'))
+ # Clear data cache
+ self._data_relations[rtype] = []
+
+ def flush_entities(self):
+ """ Flush the entities data
+ """
+ for etype, data in self._data_entities.items():
+ if not data:
+ # There is no data for these etype for this flush round.
+ continue
+ # XXX It may be interresting to directly infer the columns'
+ # names from the schema instead of using .keys()
+ columns = data[0].keys()
+ # XXX For now, the _create_copyfrom_buffer does a "row[column]"
+ # which can lead to a key error.
+ # Thus we should create dictionary with all the keys.
+ columns = set()
+ for d in data:
+ columns.update(d.keys())
+ _data = []
+ _base_data = dict.fromkeys(columns)
+ for d in data:
+ _d = _base_data.copy()
+ _d.update(d)
+ _data.append(_d)
+ buf = pgstore._create_copyfrom_buffer(_data, columns)
+ if not buf:
+ # The buffer is empty. This is probably due to error in _create_copyfrom_buffer
+ raise ValueError('Error in buffer creation for etype %s' % etype)
+ columns = ['cw_%s' % attr for attr in columns]
+ cursor = self._cnx.cnxset.cu
+ try:
+ cursor.copy_from(buf, 'cw_%s' % etype.lower(), null='NULL', columns=columns)
+ except Exception as exc:
+ self.on_rollback(exc, etype, data)
+ # Clear data cache
+ self._data_entities[etype] = []
+ self.flush_meta_data()
+
+ def flush_meta_data(self):
+ """ Flush the meta data (entities table, is_instance table, ...)
+ """
+ if self.slave_mode:
+ raise RuntimeError('Flushing meta data is not allow in slave mode')
+ if not self._dbh.table_exists('cwmassive_initialized'):
+ self.logger.info('No information available for initialized etypes/rtypes')
+ return
+ if not self._metadata_table_created:
+ # Keep the correctly flush meta data in database
+ sql = "CREATE TABLE cwmassive_metadata (etype text)"
+ self.sql(sql)
+ self._metadata_table_created = True
+ crs = self.sql('SELECT etype FROM cwmassive_metadata')
+ already_flushed = set(e for e, in crs.fetchall())
+ crs = self.sql('SELECT retype FROM cwmassive_initialized WHERE type = %(t)s',
+ {'t': 'etype'})
+ all_etypes = set(e for e, in crs.fetchall())
+ for etype in all_etypes:
+ if etype not in already_flushed:
+ # Deals with meta data
+ self.logger.info('Flushing meta data for %s' % etype)
+ self.insert_massive_meta_data(etype)
+ sql = 'INSERT INTO cwmassive_metadata VALUES (%(e)s)'
+ self.sql(sql, {'e': etype})
+
+ def _cleanup_entities(self, etype):
+ """ Cleanup etype table """
+ if self.eids_seq_range is None:
+ # Remove DEFAULT eids sequence if added
+ sql = 'ALTER TABLE cw_%s ALTER COLUMN cw_eid DROP DEFAULT;' % etype.lower()
+ self.sql(sql)
+ # Create indexes and constraints
+ tablename = SQL_PREFIX + etype.lower()
+ self.reapply_constraint_index(tablename)
+
+ def _cleanup_relations(self, rtype):
+ """ Cleanup rtype table """
+ # Push into relation table while removing duplicate
+ sql = '''INSERT INTO %(r)s_relation (eid_from, eid_to) SELECT DISTINCT
+ T.eid_from, T.eid_to FROM %(r)s_relation_tmp AS T
+ WHERE NOT EXISTS (SELECT 1 FROM %(r)s_relation AS TT WHERE
+ TT.eid_from=T.eid_from AND TT.eid_to=T.eid_to);''' % {'r': rtype}
+ self.sql(sql)
+ # Drop temporary relation table
+ sql = ('DROP TABLE %(r)s_relation_tmp' % {'r': rtype.lower()})
+ self.sql(sql)
+ # Create indexes and constraints
+ tablename = '%s_relation' % rtype.lower()
+ self.reapply_constraint_index(tablename)
+
+ def insert_massive_meta_data(self, etype):
+ """ Massive insertion of meta data for a given etype, based on SQL statements.
+ """
+ # Push data - Use coalesce to avoid NULL (and get 0), if there is no
+ # entities of this type in the entities table.
+ # Meta data relations
+ self.metagen_push_relation(etype, self._cnx.user.eid, 'created_by_relation')
+ self.metagen_push_relation(etype, self._cnx.user.eid, 'owned_by_relation')
+ self.metagen_push_relation(etype, self.source.eid, 'cw_source_relation')
+ self.metagen_push_relation(etype, self._etype_eid_idx[etype], 'is_relation')
+ self.metagen_push_relation(etype, self._etype_eid_idx[etype], 'is_instance_of_relation')
+ sql = ("INSERT INTO entities (eid, type, asource, extid) "
+ "SELECT cw_eid, '%s', 'system', NULL FROM cw_%s "
+ "WHERE NOT EXISTS (SELECT 1 FROM entities WHERE eid=cw_eid)"
+ % (etype, etype.lower()))
+ self.sql(sql)
+
+ def metagen_push_relation(self, etype, eid_to, rtype):
+ sql = ("INSERT INTO %s (eid_from, eid_to) SELECT cw_eid, %s FROM cw_%s "
+ "WHERE NOT EXISTS (SELECT 1 FROM entities WHERE eid=cw_eid)"
+ % (rtype, eid_to, etype.lower()))
+ self.sql(sql)
+
+
+### CONSTRAINTS MANAGEMENT FUNCTIONS ##########################################
+
+def get_size_constraints(schema):
+ """analyzes yams ``schema`` and returns the list of size constraints.
+
+ The returned value is a dictionary mapping entity types to a
+ sub-dictionnaries mapping attribute names -> max size.
+ """
+ size_constraints = {}
+ # iterates on all entity types
+ for eschema in schema.entities():
+ # for each entity type, iterates on attribute definitions
+ size_constraints[eschema.type] = eschema_constraints = {}
+ for rschema, aschema in eschema.attribute_definitions():
+ # for each attribute, if a size constraint is found,
+ # append it to the size constraint list
+ maxsize = None
+ rdef = rschema.rdef(eschema, aschema)
+ for constraint in rdef.constraints:
+ if isinstance(constraint, SizeConstraint):
+ maxsize = constraint.max
+ eschema_constraints[rschema.type] = maxsize
+ return size_constraints
+
+def get_default_values(schema):
+ """analyzes yams ``schema`` and returns the list of default values.
+
+ The returned value is a dictionary mapping entity types to a
+ sub-dictionnaries mapping attribute names -> default values.
+ """
+ default_values = {}
+ # iterates on all entity types
+ for eschema in schema.entities():
+ # for each entity type, iterates on attribute definitions
+ default_values[eschema.type] = eschema_constraints = {}
+ for rschema, _ in eschema.attribute_definitions():
+ # for each attribute, if a size constraint is found,
+ # append it to the size constraint list
+ if eschema.default(rschema.type) is not None:
+ eschema_constraints[rschema.type] = eschema.default(rschema.type)
+ return default_values
+
+
+class PGHelper(object):
+ def __init__(self, cnx, pg_schema='public'):
+ self.cnx = cnx
+ # Deals with pg schema, see #3216686
+ self.pg_schema = pg_schema
+
+ def application_indexes_constraints(self, tablename):
+ """ Get all the indexes/constraints for a given tablename """
+ indexes = self.application_indexes(tablename)
+ constraints = self.application_constraints(tablename)
+ _indexes = {}
+ for name, query in indexes.items():
+ # Remove pkey indexes (automatically created by constraints)
+ # Specific cases of primary key, see #3224079
+ if name not in constraints:
+ _indexes[name] = query
+ return _indexes, constraints
+
+ def table_exists(self, table_name):
+ sql = "SELECT * from information_schema.tables WHERE table_name=%(t)s AND table_schema=%(s)s"
+ crs = self.cnx.system_sql(sql, {'t': table_name, 's': self.pg_schema})
+ res = crs.fetchall()
+ if res:
+ return True
+ return False
+
+ # def check_if_primary_key_exists_for_table(self, table_name):
+ # sql = ("SELECT constraint_name FROM information_schema.table_constraints "
+ # "WHERE constraint_type = 'PRIMARY KEY' AND table_name=%(t)s AND table_schema=%(s)s")
+ # crs = self.cnx.system_sql(sql, {'t': table_name, 's': self.pg_schema})
+ # res = crs.fetchall()
+ # if res:
+ # return True
+ # return False
+
+ def index_query(self, name):
+ """Get the request to be used to recreate the index"""
+ return self.cnx.system_sql("SELECT pg_get_indexdef(c.oid) "
+ "from pg_catalog.pg_class c "
+ "LEFT JOIN pg_catalog.pg_namespace n "
+ "ON n.oid = c.relnamespace "
+ "WHERE c.relname = %(r)s AND n.nspname=%(n)s",
+ {'r': name, 'n': self.pg_schema}).fetchone()[0]
+
+ def constraint_query(self, name):
+ """Get the request to be used to recreate the constraint"""
+ return self.cnx.system_sql("SELECT pg_get_constraintdef(c.oid) "
+ "from pg_catalog.pg_constraint c "
+ "LEFT JOIN pg_catalog.pg_namespace n "
+ "ON n.oid = c.connamespace "
+ "WHERE c.conname = %(r)s AND n.nspname=%(n)s",
+ {'r': name, 'n': self.pg_schema}).fetchone()[0]
+
+ def application_indexes(self, tablename):
+ """ Iterate over all the indexes """
+ # This SQL query (cf http://www.postgresql.org/message-id/432F450F.4080700@squiz.net)
+ # aims at getting all the indexes for each table.
+ sql = '''SELECT c.relname as "Name"
+ FROM pg_catalog.pg_class c
+ JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid
+ JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid
+ LEFT JOIN pg_catalog.pg_user u ON u.usesysid = c.relowner
+ LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+ WHERE c.relkind IN ('i','')
+ AND c2.relname = '%s'
+ AND n.nspname NOT IN ('pg_catalog', 'pg_toast')
+ AND pg_catalog.pg_table_is_visible(c.oid);''' % tablename
+ indexes_list = self.cnx.system_sql(sql).fetchall()
+ indexes = {}
+ for name, in indexes_list:
+ indexes[name] = self.index_query(name)
+ return indexes
+
+ def application_constraints(self, tablename):
+ """ Iterate over all the constraints """
+ sql = '''SELECT i.conname as "Name"
+ FROM pg_catalog.pg_class c JOIN pg_catalog.pg_constraint i
+ ON i.conrelid = c.oid JOIN pg_catalog.pg_class c2 ON i.conrelid=c2.oid
+ LEFT JOIN pg_catalog.pg_user u ON u.usesysid = c.relowner
+ LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+ WHERE c2.relname = '%s' AND n.nspname NOT IN ('pg_catalog', 'pg_toast')
+ AND pg_catalog.pg_table_is_visible(c.oid);''' % tablename
+ indexes_list = self.cnx.system_sql(sql).fetchall()
+ constraints = {}
+ for name, in indexes_list:
+ query = self.constraint_query(name)
+ constraints[name] = 'ALTER TABLE %s ADD CONSTRAINT %s %s' % (tablename, name, query)
+ return constraints
--- a/dataimport/pgstore.py Tue Dec 15 19:05:41 2015 +0100
+++ b/dataimport/pgstore.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,16 +16,19 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""Postgres specific store"""
+from __future__ import print_function
import warnings
-import cPickle
import os.path as osp
-from StringIO import StringIO
+from io import StringIO
from time import asctime
from datetime import date, datetime, time
from collections import defaultdict
from base64 import b64encode
+from six import string_types, integer_types, text_type
+from six.moves import cPickle as pickle, range
+
from cubicweb.utils import make_uid
from cubicweb.server.utils import eschema_eid
from cubicweb.server.sqlutils import SQL_PREFIX
@@ -48,9 +51,9 @@
_execmany_thread_not_copy_from(cu, statement, data)
else:
if columns is None:
- cu.copy_from(buf, table, null='NULL')
+ cu.copy_from(buf, table, null=u'NULL')
else:
- cu.copy_from(buf, table, null='NULL', columns=columns)
+ cu.copy_from(buf, table, null=u'NULL', columns=columns)
def _execmany_thread(sql_connect, statements, dump_output_dir=None,
support_copy_from=True, encoding='utf-8'):
@@ -79,7 +82,7 @@
columns = list(data[0])
execmany_func(cu, statement, data, table, columns, encoding)
except Exception:
- print 'unable to copy data into table %s' % table
+ print('unable to copy data into table %s' % table)
# Error in import statement, save data in dump_output_dir
if dump_output_dir is not None:
pdata = {'data': data, 'statement': statement,
@@ -87,11 +90,10 @@
filename = make_uid()
try:
with open(osp.join(dump_output_dir,
- '%s.pickle' % filename), 'w') as fobj:
- fobj.write(cPickle.dumps(pdata))
+ '%s.pickle' % filename), 'wb') as fobj:
+ pickle.dump(pdata, fobj)
except IOError:
- print 'ERROR while pickling in', dump_output_dir, filename+'.pickle'
- pass
+ print('ERROR while pickling in', dump_output_dir, filename+'.pickle')
cnx.rollback()
raise
finally:
@@ -101,50 +103,44 @@
def _copyfrom_buffer_convert_None(value, **opts):
'''Convert None value to "NULL"'''
- return 'NULL'
+ return u'NULL'
def _copyfrom_buffer_convert_number(value, **opts):
'''Convert a number into its string representation'''
- return str(value)
+ return text_type(value)
def _copyfrom_buffer_convert_string(value, **opts):
'''Convert string value.
-
- Recognized keywords:
- :encoding: resulting string encoding (default: utf-8)
'''
- encoding = opts.get('encoding','utf-8')
- escape_chars = ((u'\\', ur'\\'), (u'\t', u'\\t'), (u'\r', u'\\r'),
+ escape_chars = ((u'\\', u'\\\\'), (u'\t', u'\\t'), (u'\r', u'\\r'),
(u'\n', u'\\n'))
for char, replace in escape_chars:
value = value.replace(char, replace)
- if isinstance(value, unicode):
- value = value.encode(encoding)
return value
def _copyfrom_buffer_convert_date(value, **opts):
'''Convert date into "YYYY-MM-DD"'''
# Do not use strftime, as it yields issue with date < 1900
# (http://bugs.python.org/issue1777412)
- return '%04d-%02d-%02d' % (value.year, value.month, value.day)
+ return u'%04d-%02d-%02d' % (value.year, value.month, value.day)
def _copyfrom_buffer_convert_datetime(value, **opts):
'''Convert date into "YYYY-MM-DD HH:MM:SS.UUUUUU"'''
# Do not use strftime, as it yields issue with date < 1900
# (http://bugs.python.org/issue1777412)
- return '%s %s' % (_copyfrom_buffer_convert_date(value, **opts),
- _copyfrom_buffer_convert_time(value, **opts))
+ return u'%s %s' % (_copyfrom_buffer_convert_date(value, **opts),
+ _copyfrom_buffer_convert_time(value, **opts))
def _copyfrom_buffer_convert_time(value, **opts):
'''Convert time into "HH:MM:SS.UUUUUU"'''
- return '%02d:%02d:%02d.%06d' % (value.hour, value.minute,
- value.second, value.microsecond)
+ return u'%02d:%02d:%02d.%06d' % (value.hour, value.minute,
+ value.second, value.microsecond)
# (types, converter) list.
_COPYFROM_BUFFER_CONVERTERS = [
(type(None), _copyfrom_buffer_convert_None),
- ((long, int, float), _copyfrom_buffer_convert_number),
- (basestring, _copyfrom_buffer_convert_string),
+ (integer_types + (float,), _copyfrom_buffer_convert_number),
+ (string_types, _copyfrom_buffer_convert_string),
(datetime, _copyfrom_buffer_convert_datetime),
(date, _copyfrom_buffer_convert_date),
(time, _copyfrom_buffer_convert_time),
@@ -164,7 +160,7 @@
rows = []
if columns is None:
if isinstance(data[0], (tuple, list)):
- columns = range(len(data[0]))
+ columns = list(range(len(data[0])))
elif isinstance(data[0], dict):
columns = data[0].keys()
else:
@@ -188,6 +184,7 @@
for types, converter in _COPYFROM_BUFFER_CONVERTERS:
if isinstance(value, types):
value = converter(value, **convert_opts)
+ assert isinstance(value, text_type)
break
else:
raise ValueError("Unsupported value type %s" % type(value))
@@ -310,7 +307,7 @@
self._sql_eid_insertdicts = {}
def flush(self):
- print 'starting flush'
+ print('starting flush')
_entities_sql = self._sql_entities
_relations_sql = self._sql_relations
_inlined_relations_sql = self._sql_inlined_relations
@@ -321,7 +318,7 @@
# In that case, simply update the insert dict and remove
# the need to make the
# UPDATE statement
- for statement, datalist in _inlined_relations_sql.iteritems():
+ for statement, datalist in _inlined_relations_sql.items():
new_datalist = []
# for a given inlined relation,
# browse each couple to be inserted
--- a/dataimport/stores.py Tue Dec 15 19:05:41 2015 +0100
+++ b/dataimport/stores.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,7 +21,7 @@
>>> user_eid = store.prepare_insert_entity('CWUser', login=u'johndoe')
>>> group_eid = store.prepare_insert_entity('CWUser', name=u'unknown')
- >>> store.relate(user_eid, 'in_group', group_eid)
+ >>> store.prepare_insert_relation(user_eid, 'in_group', group_eid)
>>> store.flush()
>>> store.commit()
>>> store.finish()
@@ -61,6 +61,8 @@
from datetime import datetime
from copy import copy
+from six import text_type
+
from logilab.common.deprecation import deprecated
from logilab.common.decorators import cached
@@ -101,7 +103,7 @@
and inlined relations.
"""
entity = self._cnx.entity_from_eid(eid)
- assert entity.cw_etype == etype, 'Trying to update with wrong type {}'.format(etype)
+ assert entity.cw_etype == etype, 'Trying to update with wrong type %s' % etype
# XXX some inlined relations may already exists
entity.cw_set(**kwargs)
@@ -120,6 +122,10 @@
"""Commit the database transaction."""
return self._commit()
+ def finish(self):
+ """Nothing to do once import is terminated for this store."""
+ pass
+
@property
def session(self):
warnings.warn('[3.19] deprecated property.', DeprecationWarning, stacklevel=2)
@@ -168,7 +174,7 @@
"""Given an entity type, attributes and inlined relations, returns the inserted entity's
eid.
"""
- for k, v in kwargs.iteritems():
+ for k, v in kwargs.items():
kwargs[k] = getattr(v, 'eid', v)
entity, rels = self.metagen.base_etype_dicts(etype)
# make a copy to keep cached entity pristine
@@ -183,7 +189,7 @@
kwargs = dict()
if inspect.getargspec(self.add_relation).keywords:
kwargs['subjtype'] = entity.cw_etype
- for rtype, targeteids in rels.iteritems():
+ for rtype, targeteids in rels.items():
# targeteids may be a single eid or a list of eids
inlined = self.rschema(rtype).inlined
try:
@@ -298,7 +304,7 @@
genfunc = self.generate(attr)
if genfunc:
entity.cw_edited.edited_attribute(attr, genfunc(entity))
- if isinstance(extid, unicode):
+ if isinstance(extid, text_type):
extid = extid.encode('utf-8')
return self.source, extid
@@ -320,4 +326,3 @@
def gen_owned_by(self, entity):
return self._cnx.user.eid
-
--- a/dataimport/test/test_csv.py Tue Dec 15 19:05:41 2015 +0100
+++ b/dataimport/test/test_csv.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,7 +17,7 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""unittest for cubicweb.dataimport.csv"""
-from StringIO import StringIO
+from io import BytesIO
from logilab.common.testlib import TestCase, unittest_main
@@ -27,7 +27,7 @@
class UcsvreaderTC(TestCase):
def test_empty_lines_skipped(self):
- stream = StringIO('''a,b,c,d,
+ stream = BytesIO(b'''a,b,c,d,
1,2,3,4,
,,,,
,,,,
@@ -45,7 +45,7 @@
list(csv.ucsvreader(stream, skip_empty=False)))
def test_skip_first(self):
- stream = StringIO('a,b,c,d,\n1,2,3,4,\n')
+ stream = BytesIO(b'a,b,c,d,\n1,2,3,4,\n')
reader = csv.ucsvreader(stream, skipfirst=True, ignore_errors=True)
self.assertEqual(list(reader),
[[u'1', u'2', u'3', u'4', u'']])
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/dataimport/test/test_massive_store.py Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,263 @@
+# -*- coding: utf-8 -*-
+# copyright 2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr -- mailto:contact@logilab.fr
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+"""Massive store test case"""
+
+import itertools
+
+from cubicweb.dataimport import ucsvreader
+from cubicweb.devtools import testlib, PostgresApptestConfiguration
+from cubicweb.devtools import startpgcluster, stoppgcluster
+from cubicweb.dataimport.massive_store import MassiveObjectStore
+
+
+def setUpModule():
+ startpgcluster(__file__)
+
+
+def tearDownModule(*args):
+ stoppgcluster(__file__)
+
+
+class MassImportSimpleTC(testlib.CubicWebTC):
+ configcls = PostgresApptestConfiguration
+ appid = 'data-massimport'
+
+ def cast(self, _type, value):
+ try:
+ return _type(value)
+ except ValueError:
+ return None
+
+ def push_geonames_data(self, dumpname, store):
+ # Push timezones
+ cnx = store._cnx
+ for code, gmt, dst, raw_offset in ucsvreader(open(self.datapath('timeZones.txt'), 'rb'),
+ delimiter='\t'):
+ cnx.create_entity('TimeZone', code=code, gmt=float(gmt),
+ dst=float(dst), raw_offset=float(raw_offset))
+ timezone_code = dict(cnx.execute('Any C, X WHERE X is TimeZone, X code C'))
+ # Push data
+ for ind, infos in enumerate(ucsvreader(open(dumpname, 'rb'),
+ delimiter='\t',
+ ignore_errors=True)):
+ latitude = self.cast(float, infos[4])
+ longitude = self.cast(float, infos[5])
+ population = self.cast(int, infos[14])
+ elevation = self.cast(int, infos[15])
+ gtopo = self.cast(int, infos[16])
+ feature_class = infos[6]
+ if len(infos[6]) != 1:
+ feature_class = None
+ entity = {'name': infos[1],
+ 'asciiname': infos[2],
+ 'alternatenames': infos[3],
+ 'latitude': latitude, 'longitude': longitude,
+ 'feature_class': feature_class,
+ 'alternate_country_code':infos[9],
+ 'admin_code_3': infos[12],
+ 'admin_code_4': infos[13],
+ 'population': population, 'elevation': elevation,
+ 'gtopo30': gtopo, 'timezone': timezone_code.get(infos[17]),
+ 'cwuri': u'http://sws.geonames.org/%s/' % int(infos[0]),
+ 'geonameid': int(infos[0]),
+ }
+ store.prepare_insert_entity('Location', **entity)
+
+ def test_autoflush_metadata(self):
+ with self.admin_access.repo_cnx() as cnx:
+ crs = cnx.system_sql('SELECT * FROM entities WHERE type=%(t)s',
+ {'t': 'Location'})
+ self.assertEqual(len(crs.fetchall()), 0)
+ store = MassiveObjectStore(cnx)
+ store.prepare_insert_entity('Location', name=u'toto')
+ store.flush()
+ store.commit()
+ store.finish()
+ cnx.commit()
+ with self.admin_access.repo_cnx() as cnx:
+ crs = cnx.system_sql('SELECT * FROM entities WHERE type=%(t)s',
+ {'t': 'Location'})
+ self.assertEqual(len(crs.fetchall()), 1)
+
+ def test_massimport_etype_metadata(self):
+ with self.admin_access.repo_cnx() as cnx:
+ store = MassiveObjectStore(cnx)
+ timezone_eid = store.prepare_insert_entity('TimeZone')
+ store.prepare_insert_entity('Location', timezone=timezone_eid)
+ store.flush()
+ store.commit()
+ eid, etname = cnx.execute('Any X, TN WHERE X timezone TZ, X is T, '
+ 'T name TN')[0]
+ self.assertEqual(cnx.entity_from_eid(eid).cw_etype, etname)
+
+ def test_drop_index(self):
+ with self.admin_access.repo_cnx() as cnx:
+ store = MassiveObjectStore(cnx)
+ cnx.commit()
+ with self.admin_access.repo_cnx() as cnx:
+ crs = cnx.system_sql('SELECT indexname FROM pg_indexes')
+ indexes = [r[0] for r in crs.fetchall()]
+ self.assertNotIn('entities_pkey', indexes)
+ self.assertNotIn('unique_entities_extid_idx', indexes)
+ self.assertNotIn('owned_by_relation_pkey', indexes)
+ self.assertNotIn('owned_by_relation_to_idx', indexes)
+
+ def test_drop_index_recreation(self):
+ with self.admin_access.repo_cnx() as cnx:
+ store = MassiveObjectStore(cnx)
+ store.finish()
+ cnx.commit()
+ with self.admin_access.repo_cnx() as cnx:
+ crs = cnx.system_sql('SELECT indexname FROM pg_indexes')
+ indexes = [r[0] for r in crs.fetchall()]
+ self.assertIn('entities_pkey', indexes)
+ self.assertIn('unique_entities_extid_idx', indexes)
+ self.assertIn('owned_by_relation_p_key', indexes)
+ self.assertIn('owned_by_relation_to_idx', indexes)
+
+ def test_eids_seq_range(self):
+ class MyMassiveObjectStore(MassiveObjectStore):
+ eids_seq_range = 1000
+ eids_seq_start = 50000
+
+ with self.admin_access.repo_cnx() as cnx:
+ store = MyMassiveObjectStore(cnx)
+ store.prepare_insert_entity('Location', name=u'toto')
+ store.flush()
+ cnx.commit()
+ with self.admin_access.repo_cnx() as cnx:
+ crs = cnx.system_sql("SELECT * FROM entities_id_seq")
+ self.assertGreater(crs.fetchone()[0], 50000)
+
+ def test_eid_entity(self):
+ class MyMassiveObjectStore(MassiveObjectStore):
+ eids_seq_range = 1000
+ eids_seq_start = 50000
+
+ with self.admin_access.repo_cnx() as cnx:
+ store = MyMassiveObjectStore(cnx)
+ eid = store.prepare_insert_entity('Location', name=u'toto')
+ store.flush()
+ self.assertGreater(eid, 50000)
+
+ def test_eid_entity_2(self):
+ class MyMassiveObjectStore(MassiveObjectStore):
+ eids_seq_range = 1000
+ eids_seq_start = 50000
+
+ with self.admin_access.repo_cnx() as cnx:
+ store = MyMassiveObjectStore(cnx)
+ eid = store.prepare_insert_entity('Location', name=u'toto', eid=10000)
+ store.flush()
+ self.assertEqual(eid, 10000)
+
+ def test_on_commit_callback(self):
+ counter = itertools.count()
+ with self.admin_access.repo_cnx() as cnx:
+ store = MassiveObjectStore(cnx, on_commit_callback=lambda:next(counter))
+ store.prepare_insert_entity('Location', name=u'toto')
+ store.flush()
+ store.commit()
+ self.assertGreaterEqual(next(counter), 1)
+
+ def test_on_rollback_callback(self):
+ counter = itertools.count()
+ with self.admin_access.repo_cnx() as cnx:
+ store = MassiveObjectStore(cnx, on_rollback_callback=lambda *_: next(counter))
+ store.prepare_insert_entity('Location', nm='toto')
+ store.flush()
+ store.commit()
+ self.assertGreaterEqual(next(counter), 1)
+
+ def test_slave_mode_indexes(self):
+ with self.admin_access.repo_cnx() as cnx:
+ slave_store = MassiveObjectStore(cnx, slave_mode=True)
+ with self.admin_access.repo_cnx() as cnx:
+ crs = cnx.system_sql('SELECT indexname FROM pg_indexes')
+ indexes = [r[0] for r in crs.fetchall()]
+ self.assertIn('entities_pkey', indexes)
+ self.assertIn('unique_entities_extid_idx', indexes)
+ self.assertIn('owned_by_relation_p_key', indexes)
+ self.assertIn('owned_by_relation_to_idx', indexes)
+
+ def test_slave_mode_exception(self):
+ with self.admin_access.repo_cnx() as cnx:
+ master_store = MassiveObjectStore(cnx, slave_mode=False)
+ slave_store = MassiveObjectStore(cnx, slave_mode=True)
+ self.assertRaises(RuntimeError, slave_store.flush_meta_data)
+ self.assertRaises(RuntimeError, slave_store.finish)
+
+ def test_simple_insert(self):
+ with self.admin_access.repo_cnx() as cnx:
+ store = MassiveObjectStore(cnx)
+ self.push_geonames_data(self.datapath('geonames.csv'), store)
+ store.flush()
+ store.commit()
+ store.finish()
+ with self.admin_access.repo_cnx() as cnx:
+ rset = cnx.execute('Any X WHERE X is Location')
+ self.assertEqual(len(rset), 4000)
+ rset = cnx.execute('Any X WHERE X is Location, X timezone T')
+ self.assertEqual(len(rset), 4000)
+
+ def test_index_building(self):
+ with self.admin_access.repo_cnx() as cnx:
+ store = MassiveObjectStore(cnx)
+ self.push_geonames_data(self.datapath('geonames.csv'), store)
+ store.flush()
+
+ # Check index
+ crs = cnx.system_sql('SELECT indexname FROM pg_indexes')
+ indexes = [r[0] for r in crs.fetchall()]
+ self.assertNotIn('entities_pkey', indexes)
+ self.assertNotIn('unique_entities_extid_idx', indexes)
+ self.assertNotIn('owned_by_relation_p_key', indexes)
+ self.assertNotIn('owned_by_relation_to_idx', indexes)
+
+ # Cleanup -> index
+ store.finish()
+
+ # Check index again
+ crs = cnx.system_sql('SELECT indexname FROM pg_indexes')
+ indexes = [r[0] for r in crs.fetchall()]
+ self.assertIn('entities_pkey', indexes)
+ self.assertIn('unique_entities_extid_idx', indexes)
+ self.assertIn('owned_by_relation_p_key', indexes)
+ self.assertIn('owned_by_relation_to_idx', indexes)
+
+ def test_multiple_insert(self):
+ with self.admin_access.repo_cnx() as cnx:
+ store = MassiveObjectStore(cnx)
+ store.init_etype_table('TestLocation')
+ store.finish()
+ store = MassiveObjectStore(cnx)
+ store.init_etype_table('TestLocation')
+ store.finish()
+
+ def test_multiple_insert_relation(self):
+ with self.admin_access.repo_cnx() as cnx:
+ store = MassiveObjectStore(cnx)
+ store.init_relation_table('used_language')
+ store.finish()
+ store = MassiveObjectStore(cnx)
+ store.init_relation_table('used_language')
+ store.finish()
+
+
+if __name__ == '__main__':
+ from logilab.common.testlib import unittest_main
+ unittest_main()
--- a/dataimport/test/test_pgstore.py Tue Dec 15 19:05:41 2015 +0100
+++ b/dataimport/test/test_pgstore.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,9 +20,11 @@
import datetime as DT
+from six import PY2
from logilab.common.testlib import TestCase, unittest_main
from cubicweb.dataimport import pgstore
+from cubicweb.devtools import testlib
class CreateCopyFromBufferTC(TestCase):
@@ -31,24 +33,24 @@
def test_convert_none(self):
cnvt = pgstore._copyfrom_buffer_convert_None
- self.assertEqual('NULL', cnvt(None))
+ self.assertEqual(u'NULL', cnvt(None))
def test_convert_number(self):
cnvt = pgstore._copyfrom_buffer_convert_number
- self.assertEqual('42', cnvt(42))
- self.assertEqual('42', cnvt(42L))
- self.assertEqual('42.42', cnvt(42.42))
+ self.assertEqual(u'42', cnvt(42))
+ if PY2:
+ self.assertEqual(u'42', cnvt(long(42)))
+ self.assertEqual(u'42.42', cnvt(42.42))
def test_convert_string(self):
cnvt = pgstore._copyfrom_buffer_convert_string
# simple
- self.assertEqual('babar', cnvt('babar'))
+ self.assertEqual(u'babar', cnvt('babar'))
# unicode
- self.assertEqual('\xc3\xa9l\xc3\xa9phant', cnvt(u'éléphant'))
- self.assertEqual('\xe9l\xe9phant', cnvt(u'éléphant', encoding='latin1'))
+ self.assertEqual(u'éléphant', cnvt(u'éléphant'))
# escaping
- self.assertEqual('babar\\tceleste\\n', cnvt('babar\tceleste\n'))
- self.assertEqual(r'C:\\new\tC:\\test', cnvt('C:\\new\tC:\\test'))
+ self.assertEqual(u'babar\\tceleste\\n', cnvt(u'babar\tceleste\n'))
+ self.assertEqual(u'C:\\\\new\\tC:\\\\test', cnvt(u'C:\\new\tC:\\test'))
def test_convert_date(self):
cnvt = pgstore._copyfrom_buffer_convert_date
@@ -64,18 +66,19 @@
# test buffer
def test_create_copyfrom_buffer_tuple(self):
- data = ((42, 42L, 42.42, u'éléphant', DT.date(666, 1, 13), DT.time(6, 6, 6),
+ l = long if PY2 else int
+ data = ((42, l(42), 42.42, u'éléphant', DT.date(666, 1, 13), DT.time(6, 6, 6),
DT.datetime(666, 6, 13, 6, 6, 6)),
- (6, 6L, 6.6, u'babar', DT.date(2014, 1, 14), DT.time(4, 2, 1),
+ (6, l(6), 6.6, u'babar', DT.date(2014, 1, 14), DT.time(4, 2, 1),
DT.datetime(2014, 1, 1, 0, 0, 0)))
results = pgstore._create_copyfrom_buffer(data)
# all columns
- expected = '''42\t42\t42.42\téléphant\t0666-01-13\t06:06:06.000000\t0666-06-13 06:06:06.000000
+ expected = u'''42\t42\t42.42\téléphant\t0666-01-13\t06:06:06.000000\t0666-06-13 06:06:06.000000
6\t6\t6.6\tbabar\t2014-01-14\t04:02:01.000000\t2014-01-01 00:00:00.000000'''
self.assertMultiLineEqual(expected, results.getvalue())
# selected columns
results = pgstore._create_copyfrom_buffer(data, columns=(1, 3, 6))
- expected = '''42\téléphant\t0666-06-13 06:06:06.000000
+ expected = u'''42\téléphant\t0666-06-13 06:06:06.000000
6\tbabar\t2014-01-01 00:00:00.000000'''
self.assertMultiLineEqual(expected, results.getvalue())
@@ -85,8 +88,18 @@
dict(integer=6, double=6.6, text=u'babar',
date=DT.datetime(2014, 1, 1, 0, 0, 0)))
results = pgstore._create_copyfrom_buffer(data, ('integer', 'text'))
- expected = '''42\téléphant\n6\tbabar'''
- self.assertMultiLineEqual(expected, results.getvalue())
+ expected = u'''42\téléphant\n6\tbabar'''
+ self.assertEqual(expected, results.getvalue())
+
+
+class SQLGenObjectStoreTC(testlib.CubicWebTC):
+
+ def test_prepare_insert_entity(self):
+ with self.admin_access.repo_cnx() as cnx:
+ store = pgstore.SQLGenObjectStore(cnx)
+ eid = store.prepare_insert_entity('CWUser', login=u'toto',
+ upassword=u'pwd')
+ self.assertIsNotNone(eid)
if __name__ == '__main__':
--- a/dataimport/test/unittest_importer.py Tue Dec 15 19:05:41 2015 +0100
+++ b/dataimport/test/unittest_importer.py Wed Dec 16 11:23:48 2015 +0100
@@ -152,14 +152,14 @@
class UseExtidAsCwuriTC(TestCase):
def test(self):
- personne = ExtEntity('Personne', 1, {'nom': set([u'de la lune']),
- 'prenom': set([u'Jean'])})
+ personne = ExtEntity('Personne', b'1', {'nom': set([u'de la lune']),
+ 'prenom': set([u'Jean'])})
mapping = {}
set_cwuri = use_extid_as_cwuri(mapping)
list(set_cwuri((personne,)))
self.assertIn('cwuri', personne.values)
- self.assertEqual(personne.values['cwuri'], set(['1']))
- mapping[1] = 'whatever'
+ self.assertEqual(personne.values['cwuri'], set([u'1']))
+ mapping[b'1'] = 'whatever'
personne.values.pop('cwuri')
list(set_cwuri((personne,)))
self.assertNotIn('cwuri', personne.values)
@@ -167,7 +167,7 @@
def extentities_from_csv(fpath):
"""Yield ExtEntity read from `fpath` CSV file."""
- with open(fpath) as f:
+ with open(fpath, 'rb') as f:
for uri, name, knows in ucsvreader(f, skipfirst=True, skip_empty=False):
yield ExtEntity('Personne', uri,
{'nom': set([name]), 'connait': set([knows])})
--- a/debian/control Tue Dec 15 19:05:41 2015 +0100
+++ b/debian/control Wed Dec 16 11:23:48 2015 +0100
@@ -10,13 +10,14 @@
Build-Depends:
debhelper (>= 7),
python (>= 2.6),
+ python-six (>= 1.4.0),
python-sphinx,
python-logilab-common,
python-unittest2 | python (>= 2.7),
python-logilab-mtconverter,
python-markdown,
python-rql,
- python-yams (>= 0.40.0),
+ python-yams (>= 0.41.1),
python-lxml,
Standards-Version: 3.9.1
Homepage: http://www.cubicweb.org
@@ -155,10 +156,11 @@
${python:Depends},
graphviz,
gettext,
+ python-six (>= 1.4.0),
python-logilab-mtconverter (>= 0.8.0),
python-logilab-common (>= 0.63.1),
python-markdown,
- python-yams (>= 0.40.0),
+ python-yams (>= 0.41.1)
python-rql (>= 0.31.2),
python-lxml
Recommends:
--- a/devtools/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""Test tools for cubicweb"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -24,7 +25,6 @@
import errno
import logging
import shutil
-import pickle
import glob
import subprocess
import warnings
@@ -35,6 +35,9 @@
from os.path import (abspath, realpath, join, exists, split, isabs, isdir)
from functools import partial
+from six import text_type
+from six.moves import cPickle as pickle
+
from logilab.common.date import strptime
from logilab.common.decorators import cached, clear_cache
@@ -92,7 +95,7 @@
DEFAULT_PSQL_SOURCES = DEFAULT_SOURCES.copy()
DEFAULT_PSQL_SOURCES['system'] = DEFAULT_SOURCES['system'].copy()
DEFAULT_PSQL_SOURCES['system']['db-driver'] = 'postgres'
-DEFAULT_PSQL_SOURCES['system']['db-user'] = unicode(getpass.getuser())
+DEFAULT_PSQL_SOURCES['system']['db-user'] = text_type(getpass.getuser())
DEFAULT_PSQL_SOURCES['system']['db-password'] = None
def turn_repo_off(repo):
@@ -109,7 +112,7 @@
try:
repo.close(sessionid)
except BadConnectionId: #this is strange ? thread issue ?
- print 'XXX unknown session', sessionid
+ print('XXX unknown session', sessionid)
for cnxset in repo.cnxsets:
cnxset.close(True)
repo.system_source.shutdown()
@@ -162,7 +165,7 @@
return None, None
return self.anonymous_credential
- def set_anonymous_allowed(self, allowed, anonuser='anon'):
+ def set_anonymous_allowed(self, allowed, anonuser=u'anon'):
if allowed:
self.anonymous_credential = (anonuser, anonuser)
else:
@@ -193,7 +196,7 @@
def sources_file(self):
"""define in subclasses self.sourcefile if necessary"""
if self.sourcefile:
- print 'Reading sources from', self.sourcefile
+ print('Reading sources from', self.sourcefile)
sourcefile = self.sourcefile
if not isabs(sourcefile):
sourcefile = join(self.apphome, sourcefile)
@@ -367,7 +370,8 @@
# XXX set a clearer error message ???
backup_coordinates, config_path = self.db_cache[self.db_cache_key(db_id)]
# reload the config used to create the database.
- config = pickle.loads(open(config_path, 'rb').read())
+ with open(config_path, 'rb') as f:
+ config = pickle.load(f)
# shutdown repo before changing database content
if self._repo is not None:
self._repo.turn_repo_off()
@@ -399,9 +403,8 @@
def _new_repo(self, config):
"""Factory method to create a new Repository Instance"""
- from cubicweb.repoapi import _get_inmemory_repo
config._cubes = None
- repo = _get_inmemory_repo(config)
+ repo = config.repository()
config.repository = lambda x=None: repo
# extending Repository class
repo._has_started = False
@@ -415,7 +418,7 @@
from cubicweb.repoapi import connect
repo = self.get_repo()
sources = self.config.read_sources_file()
- login = unicode(sources['admin']['login'])
+ login = text_type(sources['admin']['login'])
password = sources['admin']['password'] or 'xxx'
cnx = connect(repo, login, password=password)
return cnx
@@ -464,7 +467,7 @@
dbname, data = data.split('-', 1)
db_id, filetype = data.split('.', 1)
entries.setdefault((dbname, db_id), {})[filetype] = filepath
- for (dbname, db_id), entry in entries.iteritems():
+ for (dbname, db_id), entry in entries.items():
# apply necessary transformation from the driver
value = self.process_cache_entry(directory, dbname, db_id, entry)
assert 'config' in entry
@@ -494,7 +497,7 @@
if test_db_id is DEFAULT_EMPTY_DB_ID:
self.init_test_database()
else:
- print 'Building %s for database %s' % (test_db_id, self.dbname)
+ print('Building %s for database %s' % (test_db_id, self.dbname))
self.build_db_cache(DEFAULT_EMPTY_DB_ID)
self.restore_database(DEFAULT_EMPTY_DB_ID)
repo = self.get_repo(startup=True)
@@ -543,7 +546,7 @@
try:
subprocess.check_call(['initdb', '-D', datadir, '-E', 'utf-8', '--locale=C'])
- except OSError, err:
+ except OSError as err:
if err.errno == errno.ENOENT:
raise OSError('"initdb" could not be found. '
'You should add the postgresql bin folder to your PATH '
@@ -562,7 +565,11 @@
subprocess.check_call(['pg_ctl', 'start', '-w', '-D', datadir,
'-o', options],
env=env)
- except OSError, err:
+ except OSError as err:
+ try:
+ os.rmdir(sockdir)
+ except OSError:
+ pass
if err.errno == errno.ENOENT:
raise OSError('"pg_ctl" could not be found. '
'You should add the postgresql bin folder to your PATH '
@@ -575,6 +582,10 @@
datadir = join(os.path.dirname(pyfile), 'data',
'pgdb-%s' % os.path.splitext(os.path.basename(pyfile))[0])
subprocess.call(['pg_ctl', 'stop', '-D', datadir, '-m', 'fast'])
+ try:
+ os.rmdir(DEFAULT_PSQL_SOURCES['system']['db-host'])
+ except OSError:
+ pass
class PostgresTestDataBaseHandler(TestDataBaseHandler):
@@ -678,7 +689,7 @@
@property
def _config_id(self):
- return sha1(self.config.apphome).hexdigest()[:10]
+ return sha1(self.config.apphome.encode('utf-8')).hexdigest()[:10]
def _backup_name(self, db_id): # merge me with parent
backup_name = '_'.join(('cache', self._config_id, self.dbname, db_id))
@@ -834,21 +845,21 @@
found_date = False
for row, rowdesc in zip(rset, rset.description):
for cellindex, (value, vtype) in enumerate(zip(row, rowdesc)):
- if vtype in ('Date', 'Datetime') and type(value) is unicode:
+ if vtype in ('Date', 'Datetime') and isinstance(value, text_type):
found_date = True
value = value.rsplit('.', 1)[0]
try:
row[cellindex] = strptime(value, '%Y-%m-%d %H:%M:%S')
except Exception:
row[cellindex] = strptime(value, '%Y-%m-%d')
- if vtype == 'Time' and type(value) is unicode:
+ if vtype == 'Time' and isinstance(value, text_type):
found_date = True
try:
row[cellindex] = strptime(value, '%H:%M:%S')
except Exception:
# DateTime used as Time?
row[cellindex] = strptime(value, '%Y-%m-%d %H:%M:%S')
- if vtype == 'Interval' and type(value) is int:
+ if vtype == 'Interval' and isinstance(value, int):
found_date = True
row[cellindex] = timedelta(0, value, 0) # XXX value is in number of seconds?
if not found_date:
@@ -882,7 +893,7 @@
We only keep one repo in cache to prevent too much objects to stay alive
(database handler holds a reference to a repository). As at the moment a new
handler is created for each TestCase class and all test methods are executed
- sequentialy whithin this class, there should not have more cache miss that
+ sequentially whithin this class, there should not have more cache miss that
if we had a wider cache as once a Handler stop being used it won't be used
again.
"""
@@ -947,5 +958,3 @@
handler = get_test_db_handler(config)
handler.build_db_cache()
return handler.get_repo_and_cnx()
-
-
--- a/devtools/devctl.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/devctl.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,6 +18,7 @@
"""additional cubicweb-ctl commands and command handlers for cubicweb and
cubicweb's cubes development
"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -25,10 +26,12 @@
# possible (for cubicweb-ctl reactivity, necessary for instance for usable bash
# completion). So import locally in command helpers.
import sys
-from datetime import datetime
+from datetime import datetime, date
from os import mkdir, chdir, path as osp
from warnings import warn
+from six.moves import input
+
from logilab.common import STD_BLACKLIST
from cubicweb.__pkginfo__ import version as cubicwebversion
@@ -83,7 +86,7 @@
def cleanup_sys_modules(config):
# cleanup sys.modules, required when we're updating multiple cubes
- for name, mod in sys.modules.items():
+ for name, mod in list(sys.modules.items()):
if mod is None:
# duh ? logilab.common.os for instance
del sys.modules[name]
@@ -127,7 +130,7 @@
from cubicweb.i18n import add_msg
from cubicweb.schema import NO_I18NCONTEXT, CONSTRAINTS
w('# schema pot file, generated on %s\n'
- % datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
+ % datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S'))
w('# \n')
w('# singular and plural forms for each entity type\n')
w('\n')
@@ -250,7 +253,7 @@
# bw compat, necessary until all translation of relation are
# done properly...
add_msg(w, '%s_object' % rtype)
- for rdef in rschema.rdefs.itervalues():
+ for rdef in rschema.rdefs.values():
if not rdef.description or rdef.description in done:
continue
if (librschema is None or
@@ -267,7 +270,7 @@
for reg, objdict in vreg.items():
if reg in ('boxes', 'contentnavigation'):
continue
- for objects in objdict.itervalues():
+ for objects in objdict.values():
for obj in objects:
objid = '%s_%s' % (reg, obj.__regid__)
if objid in done:
@@ -314,21 +317,21 @@
from cubicweb.i18n import extract_from_tal, execute2
tempdir = tempfile.mkdtemp(prefix='cw-')
cwi18ndir = WebConfiguration.i18n_lib_dir()
- print '-> extract messages:',
- print 'schema',
+ print('-> extract messages:', end=' ')
+ print('schema', end=' ')
schemapot = osp.join(tempdir, 'schema.pot')
potfiles = [schemapot]
potfiles.append(schemapot)
# explicit close necessary else the file may not be yet flushed when
# we'll using it below
- schemapotstream = file(schemapot, 'w')
+ schemapotstream = open(schemapot, 'w')
generate_schema_pot(schemapotstream.write, cubedir=None)
schemapotstream.close()
- print 'TAL',
+ print('TAL', end=' ')
tali18nfile = osp.join(tempdir, 'tali18n.py')
extract_from_tal(find(osp.join(BASEDIR, 'web'), ('.py', '.pt')),
tali18nfile)
- print '-> generate .pot files.'
+ print('-> generate .pot files.')
pyfiles = get_module_files(BASEDIR)
pyfiles += globfind(osp.join(BASEDIR, 'misc', 'migration'), '*.py')
schemafiles = globfind(osp.join(BASEDIR, 'schemas'), '*.py')
@@ -349,12 +352,12 @@
if osp.exists(potfile):
potfiles.append(potfile)
else:
- print '-> WARNING: %s file was not generated' % potfile
- print '-> merging %i .pot files' % len(potfiles)
+ print('-> WARNING: %s file was not generated' % potfile)
+ print('-> merging %i .pot files' % len(potfiles))
cubicwebpot = osp.join(tempdir, 'cubicweb.pot')
cmd = ['msgcat', '-o', cubicwebpot] + potfiles
execute2(cmd)
- print '-> merging main pot file with existing translations.'
+ print('-> merging main pot file with existing translations.')
chdir(cwi18ndir)
toedit = []
for lang in CubicWebNoAppConfiguration.cw_languages():
@@ -368,10 +371,10 @@
# cleanup
rm(tempdir)
# instructions pour la suite
- print '-> regenerated CubicWeb\'s .po catalogs.'
- print '\nYou can now edit the following files:'
- print '* ' + '\n* '.join(toedit)
- print 'when you are done, run "cubicweb-ctl i18ncube yourcube".'
+ print('-> regenerated CubicWeb\'s .po catalogs.')
+ print('\nYou can now edit the following files:')
+ print('* ' + '\n* '.join(toedit))
+ print('when you are done, run "cubicweb-ctl i18ncube yourcube".')
class UpdateCubeCatalogCommand(Command):
@@ -398,25 +401,25 @@
from subprocess import CalledProcessError
for cubedir in cubes:
if not osp.isdir(cubedir):
- print '-> ignoring %s that is not a directory.' % cubedir
+ print('-> ignoring %s that is not a directory.' % cubedir)
continue
try:
toedit = update_cube_catalogs(cubedir)
except CalledProcessError as exc:
- print '\n*** error while updating catalogs for cube', cubedir
- print 'cmd:\n%s' % exc.cmd
- print 'stdout:\n%s\nstderr:\n%s' % exc.data
+ print('\n*** error while updating catalogs for cube', cubedir)
+ print('cmd:\n%s' % exc.cmd)
+ print('stdout:\n%s\nstderr:\n%s' % exc.data)
except Exception:
import traceback
traceback.print_exc()
- print '*** error while updating catalogs for cube', cubedir
+ print('*** error while updating catalogs for cube', cubedir)
return False
else:
# instructions pour la suite
if toedit:
- print '-> regenerated .po catalogs for cube %s.' % cubedir
- print '\nYou can now edit the following files:'
- print '* ' + '\n* '.join(toedit)
+ print('-> regenerated .po catalogs for cube %s.' % cubedir)
+ print('\nYou can now edit the following files:')
+ print('* ' + '\n* '.join(toedit))
print ('When you are done, run "cubicweb-ctl i18ninstance '
'<yourinstance>" to see changes in your instances.')
return True
@@ -429,7 +432,7 @@
from cubicweb.i18n import extract_from_tal, execute2
cube = osp.basename(osp.normpath(cubedir))
tempdir = tempfile.mkdtemp()
- print underline_title('Updating i18n catalogs for cube %s' % cube)
+ print(underline_title('Updating i18n catalogs for cube %s' % cube))
chdir(cubedir)
if osp.exists(osp.join('i18n', 'entities.pot')):
warn('entities.pot is deprecated, rename file to static-messages.pot (%s)'
@@ -439,20 +442,20 @@
potfiles = [osp.join('i18n', 'static-messages.pot')]
else:
potfiles = []
- print '-> extracting messages:',
- print 'schema',
+ print('-> extracting messages:', end=' ')
+ print('schema', end=' ')
schemapot = osp.join(tempdir, 'schema.pot')
potfiles.append(schemapot)
# explicit close necessary else the file may not be yet flushed when
# we'll using it below
- schemapotstream = file(schemapot, 'w')
+ schemapotstream = open(schemapot, 'w')
generate_schema_pot(schemapotstream.write, cubedir)
schemapotstream.close()
- print 'TAL',
+ print('TAL', end=' ')
tali18nfile = osp.join(tempdir, 'tali18n.py')
ptfiles = find('.', ('.py', '.pt'), blacklist=STD_BLACKLIST+('test',))
extract_from_tal(ptfiles, tali18nfile)
- print 'Javascript'
+ print('Javascript')
jsfiles = [jsfile for jsfile in find('.', '.js')
if osp.basename(jsfile).startswith('cub')]
if jsfiles:
@@ -463,7 +466,7 @@
# no pot file created if there are no string to translate
if osp.exists(tmppotfile):
potfiles.append(tmppotfile)
- print '-> creating cube-specific catalog'
+ print('-> creating cube-specific catalog')
tmppotfile = osp.join(tempdir, 'generated.pot')
cubefiles = find('.', '.py', blacklist=STD_BLACKLIST+('test',))
cubefiles.append(tali18nfile)
@@ -473,20 +476,20 @@
if osp.exists(tmppotfile): # doesn't exists of no translation string found
potfiles.append(tmppotfile)
potfile = osp.join(tempdir, 'cube.pot')
- print '-> merging %i .pot files' % len(potfiles)
+ print('-> merging %i .pot files' % len(potfiles))
cmd = ['msgcat', '-o', potfile]
cmd.extend(potfiles)
execute2(cmd)
if not osp.exists(potfile):
- print 'no message catalog for cube', cube, 'nothing to translate'
+ print('no message catalog for cube', cube, 'nothing to translate')
# cleanup
rm(tempdir)
return ()
- print '-> merging main pot file with existing translations:',
+ print('-> merging main pot file with existing translations:', end=' ')
chdir('i18n')
toedit = []
for lang in CubicWebNoAppConfiguration.cw_languages():
- print lang,
+ print(lang, end=' ')
cubepo = '%s.po' % lang
if not osp.exists(cubepo):
shutil.copy(potfile, cubepo)
@@ -496,7 +499,7 @@
ensure_fs_mode(cubepo)
shutil.move('%snew' % cubepo, cubepo)
toedit.append(osp.abspath(cubepo))
- print
+ print()
# cleanup
rm(tempdir)
return toedit
@@ -620,7 +623,7 @@
" Please specify it using the --directory option")
cubesdir = cubespath[0]
if not osp.isdir(cubesdir):
- print "-> creating cubes directory", cubesdir
+ print("-> creating cubes directory", cubesdir)
try:
mkdir(cubesdir)
except OSError as err:
@@ -632,7 +635,7 @@
skeldir = osp.join(BASEDIR, 'skeleton')
default_name = 'cubicweb-%s' % cubename.lower().replace('_', '-')
if verbose:
- distname = raw_input('Debian name for your cube ? [%s]): '
+ distname = input('Debian name for your cube ? [%s]): '
% default_name).strip()
if not distname:
distname = default_name
@@ -644,12 +647,13 @@
if not re.match('[a-z][-a-z0-9]*$', distname):
raise BadCommandUsage(
'cube distname should be a valid debian package name')
- longdesc = shortdesc = raw_input(
+ longdesc = shortdesc = input(
'Enter a short description for your cube: ')
if verbose:
- longdesc = raw_input(
+ longdesc = input(
'Enter a long description (leave empty to reuse the short one): ')
- dependencies = {'cubicweb': '>= %s' % cubicwebversion}
+ dependencies = {'cubicweb': '>= %s' % cubicwebversion,
+ 'six': '>= 1.4.0',}
if verbose:
dependencies.update(self._ask_for_dependencies())
context = {'cubename' : cubename,
@@ -658,7 +662,7 @@
'longdesc' : longdesc or shortdesc,
'dependencies' : dependencies,
'version' : cubicwebversion,
- 'year' : str(datetime.now().year),
+ 'year' : str(date.today().year),
'author': self['author'],
'author-email': self['author-email'],
'author-web-site': self['author-web-site'],
@@ -681,7 +685,7 @@
if answer == 'y':
depcubes.append(cube)
if answer == 'type':
- depcubes = splitstrip(raw_input('type dependencies: '))
+ depcubes = splitstrip(input('type dependencies: '))
break
elif answer == 'skip':
break
@@ -710,7 +714,7 @@
requests = {}
for filepath in args:
try:
- stream = file(filepath)
+ stream = open(filepath)
except OSError as ex:
raise BadCommandUsage("can't open rql log file %s: %s"
% (filepath, ex))
@@ -731,17 +735,17 @@
except Exception as exc:
sys.stderr.write('Line %s: %s (%s)\n' % (lineno, exc, line))
stat = []
- for rql, times in requests.iteritems():
+ for rql, times in requests.items():
stat.append( (sum(time[0] for time in times),
sum(time[1] for time in times),
len(times), rql) )
stat.sort()
stat.reverse()
total_time = sum(clocktime for clocktime, cputime, occ, rql in stat) * 0.01
- print 'Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query'
+ print('Percentage;Cumulative Time (clock);Cumulative Time (CPU);Occurences;Query')
for clocktime, cputime, occ, rql in stat:
- print '%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
- cputime, occ, rql)
+ print('%.2f;%.2f;%.2f;%s;%s' % (clocktime/total_time, clocktime,
+ cputime, occ, rql))
class GenerateSchema(Command):
--- a/devtools/fake.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/fake.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,6 +22,8 @@
from contextlib import contextmanager
+from six import string_types
+
from logilab.database import get_db_helper
from cubicweb.req import RequestSessionBase
@@ -91,7 +93,7 @@
def set_request_header(self, header, value, raw=False):
"""set an incoming HTTP header (for test purpose only)"""
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
value = [value]
if raw:
# adding encoded header is important, else page content
@@ -110,7 +112,7 @@
def build_url_params(self, **kwargs):
# overriden to get predictable resultts
args = []
- for param, values in sorted(kwargs.iteritems()):
+ for param, values in sorted(kwargs.items()):
if not isinstance(values, (list, tuple)):
values = (values,)
for value in values:
--- a/devtools/fill.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/fill.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,6 +17,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""This modules defines func / methods for creating test repositories"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -25,6 +26,10 @@
from copy import deepcopy
from datetime import datetime, date, time, timedelta
from decimal import Decimal
+import inspect
+
+from six import text_type, add_metaclass
+from six.moves import range
from logilab.common import attrdict
from logilab.mtconverter import xml_escape
@@ -173,7 +178,7 @@
generate_tztime = generate_time # XXX implementation should add a timezone
def generate_bytes(self, entity, attrname, index, format=None):
- fakefile = Binary("%s%s" % (attrname, index))
+ fakefile = Binary(("%s%s" % (attrname, index)).encode('ascii'))
fakefile.filename = u"file_%s" % attrname
return fakefile
@@ -224,7 +229,7 @@
"""
for cst in self.eschema.rdef(attrname).constraints:
if isinstance(cst, StaticVocabularyConstraint):
- return unicode(choice(cst.vocabulary()))
+ return text_type(choice(cst.vocabulary()))
return None
# XXX nothing to do here
@@ -254,13 +259,15 @@
for attrname, attrvalue in classdict.items():
if callable(attrvalue):
if attrname.startswith('generate_') and \
- attrvalue.func_code.co_argcount < 2:
+ len(inspect.getargspec(attrvalue).args) < 2:
raise TypeError('generate_xxx must accept at least 1 argument')
setattr(_ValueGenerator, attrname, attrvalue)
return type.__new__(mcs, name, bases, classdict)
+
+@add_metaclass(autoextend)
class ValueGenerator(_ValueGenerator):
- __metaclass__ = autoextend
+ pass
def _default_choice_func(etype, attrname):
@@ -286,7 +293,7 @@
returns acceptable values for this attribute
"""
queries = []
- for index in xrange(entity_num):
+ for index in range(entity_num):
restrictions = []
args = {}
for attrname, value in make_entity(etype, schema, vreg, index, choice_func).items():
@@ -347,7 +354,7 @@
fmt = vreg.property_value('ui.float-format')
value = fmt % value
else:
- value = unicode(value)
+ value = text_type(value)
return entity
@@ -363,7 +370,7 @@
rql += ', %s is %s' % (selectvar, objtype)
rset = cnx.execute(rql)
except Exception:
- print "could restrict eid_list with given constraints (%r)" % constraints
+ print("could restrict eid_list with given constraints (%r)" % constraints)
return []
return set(eid for eid, in rset.rows)
@@ -508,8 +515,8 @@
break
else:
# FIXME: 20 should be read from config
- subjeidsiter = [choice(tuple(subjeids)) for i in xrange(min(len(subjeids), 20))]
- objeidsiter = [choice(tuple(objeids)) for i in xrange(min(len(objeids), 20))]
+ subjeidsiter = [choice(tuple(subjeids)) for i in range(min(len(subjeids), 20))]
+ objeidsiter = [choice(tuple(objeids)) for i in range(min(len(objeids), 20))]
for subjeid, objeid in zip(subjeidsiter, objeidsiter):
if subjeid != objeid and not (subjeid, objeid) in used:
used.add( (subjeid, objeid) )
--- a/devtools/htmlparser.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/htmlparser.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,7 +20,7 @@
import re
import sys
from xml import sax
-from cStringIO import StringIO
+from io import BytesIO
from lxml import etree
@@ -33,7 +33,7 @@
ERR_COUNT = 0
-_REM_SCRIPT_RGX = re.compile(r"<script[^>]*>.*?</script>", re.U|re.M|re.I|re.S)
+_REM_SCRIPT_RGX = re.compile(br"<script[^>]*>.*?</script>", re.M|re.I|re.S)
def _remove_script_tags(data):
"""Remove the script (usually javascript) tags to help the lxml
XMLParser / HTMLParser do their job. Without that, they choke on
@@ -70,7 +70,7 @@
#
# using that, we'll miss most actual validation error we want to
# catch. For now, use dumb regexp
- return _REM_SCRIPT_RGX.sub('', data)
+ return _REM_SCRIPT_RGX.sub(b'', data)
class Validator(object):
@@ -164,10 +164,10 @@
def _parse(self, data):
inpsrc = sax.InputSource()
- inpsrc.setByteStream(StringIO(data))
+ inpsrc.setByteStream(BytesIO(data))
try:
self._parser.parse(inpsrc)
- except sax.SAXParseException, exc:
+ except sax.SAXParseException as exc:
new_exc = AssertionError(u'invalid document: %s' % exc)
new_exc.position = (exc._linenum, exc._colnum)
raise new_exc
@@ -209,7 +209,7 @@
def matching_nodes(self, tag, **attrs):
for elt in self.etree.iterfind(self._iterstr(tag)):
eltattrs = elt.attrib
- for attr, value in attrs.iteritems():
+ for attr, value in attrs.items():
try:
if eltattrs[attr] != value:
break
--- a/devtools/httptest.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/httptest.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,17 +18,18 @@
"""this module contains base classes and utilities for integration with running
http server
"""
+from __future__ import print_function
+
__docformat__ = "restructuredtext en"
import random
import threading
import socket
-import httplib
-from urlparse import urlparse
-from twisted.internet import reactor, error
+from six.moves import range, http_client
+from six.moves.urllib.parse import urlparse
-from cubicweb.etwist.server import run
+
from cubicweb.devtools.testlib import CubicWebTC
from cubicweb.devtools import ApptestConfiguration
@@ -89,6 +90,8 @@
configcls = CubicWebServerConfig
def start_server(self):
+ from twisted.internet import reactor
+ from cubicweb.etwist.server import run
# use a semaphore to avoid starting test while the http server isn't
# fully initilialized
semaphore = threading.Semaphore(0)
@@ -110,12 +113,13 @@
#pre init utils connection
parseurl = urlparse(self.config['base-url'])
assert parseurl.port == self.config['port'], (self.config['base-url'], self.config['port'])
- self._web_test_cnx = httplib.HTTPConnection(parseurl.hostname,
- parseurl.port)
+ self._web_test_cnx = http_client.HTTPConnection(parseurl.hostname,
+ parseurl.port)
self._ident_cookie = None
def stop_server(self, timeout=15):
"""Stop the webserver, waiting for the thread to return"""
+ from twisted.internet import reactor
if self._web_test_cnx is None:
self.web_logout()
self._web_test_cnx.close()
@@ -139,7 +143,7 @@
passwd = user
response = self.web_get("login?__login=%s&__password=%s" %
(user, passwd))
- assert response.status == httplib.SEE_OTHER, response.status
+ assert response.status == http_client.SEE_OTHER, response.status
self._ident_cookie = response.getheader('Set-Cookie')
assert self._ident_cookie
return True
@@ -151,7 +155,7 @@
self._ident_cookie = None
def web_request(self, path='', method='GET', body=None, headers=None):
- """Return an httplib.HTTPResponse object for the specified path
+ """Return an http_client.HTTPResponse object for the specified path
Use available credential if available.
"""
@@ -174,9 +178,10 @@
self.start_server()
def tearDown(self):
+ from twisted.internet import error
try:
self.stop_server()
except error.ReactorNotRunning as err:
# Server could be launched manually
- print err
+ print(err)
super(CubicWebServerTC, self).tearDown()
--- a/devtools/instrument.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/instrument.py Wed Dec 16 11:23:48 2015 +0100
@@ -14,6 +14,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""Instrumentation utilities"""
+from __future__ import print_function
import os
@@ -45,10 +46,10 @@
return _COLORS[key]
def warn(msg, *args):
- print 'WARNING: %s' % (msg % args)
+ print('WARNING: %s' % (msg % args))
def info(msg):
- print 'INFO: ' + msg
+ print('INFO: ' + msg)
class PropagationAnalyzer(object):
@@ -185,7 +186,7 @@
def add_colors_legend(self, graph):
"""Add a legend of used colors to the graph."""
- for package, color in sorted(_COLORS.iteritems()):
+ for package, color in sorted(_COLORS.items()):
graph.add_node(package, color=color, fontcolor=color, shape='record')
--- a/devtools/qunit.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/qunit.py Wed Dec 16 11:23:48 2015 +0100
@@ -15,53 +15,28 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+from __future__ import absolute_import
import os, os.path as osp
-from tempfile import mkdtemp, NamedTemporaryFile, TemporaryFile
-import tempfile
-from Queue import Queue, Empty
-from subprocess import Popen, check_call, CalledProcessError
-from shutil import rmtree, copy as copyfile
-from uuid import uuid4
+import errno
+from tempfile import mkdtemp
+from subprocess import Popen, PIPE, STDOUT
+
+from six.moves.queue import Queue, Empty
# imported by default to simplify further import statements
from logilab.common.testlib import unittest_main, with_tempdir, InnerTest, Tags
-from logilab.common.shellutils import getlogin
+import webtest.http
import cubicweb
from cubicweb.view import View
from cubicweb.web.controller import Controller
from cubicweb.web.views.staticcontrollers import StaticFileController, STATIC_CONTROLLERS
-from cubicweb.devtools.httptest import CubicWebServerTC
-
-
-class VerboseCalledProcessError(CalledProcessError):
-
- def __init__(self, returncode, command, stdout, stderr):
- super(VerboseCalledProcessError, self).__init__(returncode, command)
- self.stdout = stdout
- self.stderr = stderr
-
- def __str__(self):
- str = [ super(VerboseCalledProcessError, self).__str__()]
- if self.stdout.strip():
- str.append('******************')
- str.append('* process stdout *')
- str.append('******************')
- str.append(self.stdout)
- if self.stderr.strip():
- str.append('******************')
- str.append('* process stderr *')
- str.append('******************')
- str.append(self.stderr)
- return '\n'.join(str)
-
+from cubicweb.devtools import webtest as cwwebtest
class FirefoxHelper(object):
- profile_name_mask = 'PYTEST_PROFILE_%(uid)s'
-
def __init__(self, url=None):
self._process = None
self._profile_dir = mkdtemp(prefix='cwtest-ffxprof-')
@@ -70,6 +45,17 @@
self.firefox_cmd = [osp.join(osp.dirname(__file__), 'data', 'xvfb-run.sh'),
'-a', '-s', '-noreset -screen 0 800x600x24'] + self.firefox_cmd
+ def test(self):
+ try:
+ proc = Popen(['firefox', '--help'], stdout=PIPE, stderr=STDOUT)
+ stdout, _ = proc.communicate()
+ return proc.returncode == 0, stdout
+ except OSError as exc:
+ if exc.errno == errno.ENOENT:
+ msg = '[%s] %s' % (errno.errorcode[exc.errno], exc.strerror)
+ return False, msg
+ raise
+
def start(self, url):
self.stop()
cmd = self.firefox_cmd + ['-silent', '--profile', self._profile_dir,
@@ -88,60 +74,46 @@
self.stop()
-class QUnitTestCase(CubicWebServerTC):
+class QUnitTestCase(cwwebtest.CubicWebTestTC):
- tags = CubicWebServerTC.tags | Tags(('qunit',))
+ tags = cwwebtest.CubicWebTestTC.tags | Tags(('qunit',))
# testfile, (dep_a, dep_b)
all_js_tests = ()
def setUp(self):
- self.config.global_set_option('access-control-allow-origin', '*')
super(QUnitTestCase, self).setUp()
self.test_queue = Queue()
class MyQUnitResultController(QUnitResultController):
tc = self
test_queue = self.test_queue
self._qunit_controller = MyQUnitResultController
- self.vreg.register(MyQUnitResultController)
- self.vreg.register(QUnitView)
- self.vreg.register(CWSoftwareRootStaticController)
+ self.webapp.app.appli.vreg.register(MyQUnitResultController)
+ self.webapp.app.appli.vreg.register(QUnitView)
+ self.webapp.app.appli.vreg.register(CWDevtoolsStaticController)
+ self.server = webtest.http.StopableWSGIServer.create(self.webapp.app)
+ self.config.global_set_option('base-url', self.server.application_url)
def tearDown(self):
+ self.server.shutdown()
+ self.webapp.app.appli.vreg.unregister(self._qunit_controller)
+ self.webapp.app.appli.vreg.unregister(QUnitView)
+ self.webapp.app.appli.vreg.unregister(CWDevtoolsStaticController)
super(QUnitTestCase, self).tearDown()
- self.vreg.unregister(self._qunit_controller)
- self.vreg.unregister(QUnitView)
- self.vreg.unregister(CWSoftwareRootStaticController)
-
- def abspath(self, path):
- """use self.__module__ to build absolute path if necessary"""
- if not osp.isabs(path):
- dirname = osp.dirname(__import__(self.__module__).__file__)
- return osp.abspath(osp.join(dirname,path))
- return path
def test_javascripts(self):
for args in self.all_js_tests:
- test_file = self.abspath(args[0])
+ self.assertIn(len(args), (1, 2))
+ test_file = args[0]
if len(args) > 1:
- depends = [self.abspath(dep) for dep in args[1]]
+ depends = args[1]
else:
depends = ()
- if len(args) > 2:
- data = [self.abspath(data) for data in args[2]]
- else:
- data = ()
- for js_test in self._test_qunit(test_file, depends, data):
+ for js_test in self._test_qunit(test_file, depends):
yield js_test
@with_tempdir
- def _test_qunit(self, test_file, depends=(), data_files=(), timeout=10):
- assert osp.exists(test_file), test_file
- for dep in depends:
- assert osp.exists(dep), dep
- for data in data_files:
- assert osp.exists(data), data
-
+ def _test_qunit(self, test_file, depends=(), timeout=10):
QUnitView.test_file = test_file
QUnitView.depends = depends
@@ -149,6 +121,9 @@
self.test_queue.get(False)
browser = FirefoxHelper()
+ isavailable, reason = browser.test()
+ if not isavailable:
+ self.fail('firefox not available or not working properly (%s)' % reason)
browser.start(self.config['base-url'] + "?vid=qunit")
test_count = 0
error = False
@@ -188,6 +163,7 @@
def publish(self, rset=None):
event = self._cw.form['event']
getattr(self, 'handle_%s' % event)()
+ return b''
def handle_module_start(self):
self.__class__._current_module_name = self._cw.form.get('name', '')
@@ -234,20 +210,15 @@
def call(self, **kwargs):
w = self.w
req = self._cw
- data = {
- 'jquery': req.data_url('jquery.js'),
- 'web_test': req.build_url('cwsoftwareroot/devtools/data'),
- }
w(u'''<!DOCTYPE html>
<html>
<head>
<meta http-equiv="content-type" content="application/html; charset=UTF-8"/>
<!-- JS lib used as testing framework -->
- <link rel="stylesheet" type="text/css" media="all" href="%(web_test)s/qunit.css" />
- <script src="%(jquery)s" type="text/javascript"></script>
- <script src="%(web_test)s/cwmock.js" type="text/javascript"></script>
- <script src="%(web_test)s/qunit.js" type="text/javascript"></script>'''
- % data)
+ <link rel="stylesheet" type="text/css" media="all" href="/devtools/qunit.css" />
+ <script src="/data/jquery.js" type="text/javascript"></script>
+ <script src="/devtools/cwmock.js" type="text/javascript"></script>
+ <script src="/devtools/qunit.js" type="text/javascript"></script>''')
w(u'<!-- result report tools -->')
w(u'<script type="text/javascript">')
w(u"var BASE_URL = '%s';" % req.base_url())
@@ -293,14 +264,11 @@
w(u'</script>')
w(u'<!-- Test script dependencies (tested code for example) -->')
- prefix = len(cubicweb.CW_SOFTWARE_ROOT) + 1
for dep in self.depends:
- dep = req.build_url('cwsoftwareroot/') + dep[prefix:]
- w(u' <script src="%s" type="text/javascript"></script>' % dep)
+ w(u' <script src="%s" type="text/javascript"></script>\n' % dep)
w(u' <!-- Test script itself -->')
- test_url = req.build_url('cwsoftwareroot/') + self.test_file[prefix:]
- w(u' <script src="%s" type="text/javascript"></script>' % test_url)
+ w(u' <script src="%s" type="text/javascript"></script>' % self.test_file)
w(u''' </head>
<body>
<div id="qunit-fixture"></div>
@@ -309,16 +277,16 @@
</html>''')
-class CWSoftwareRootStaticController(StaticFileController):
- __regid__ = 'cwsoftwareroot'
+class CWDevtoolsStaticController(StaticFileController):
+ __regid__ = 'devtools'
def publish(self, rset=None):
- staticdir = cubicweb.CW_SOFTWARE_ROOT
+ staticdir = osp.join(osp.dirname(__file__), 'data')
relpath = self.relpath[len(self.__regid__) + 1:]
return self.static_file(osp.join(staticdir, relpath))
-STATIC_CONTROLLERS.append(CWSoftwareRootStaticController)
+STATIC_CONTROLLERS.append(CWDevtoolsStaticController)
if __name__ == '__main__':
--- a/devtools/repotest.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/repotest.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,6 +19,7 @@
This module contains functions to initialize a new repository.
"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -29,10 +30,9 @@
def tuplify(mylist):
return [tuple(item) for item in mylist]
-def snippet_cmp(a, b):
- a = (a[0], [e.expression for e in a[1]])
- b = (b[0], [e.expression for e in b[1]])
- return cmp(a, b)
+def snippet_key(a):
+ # a[0] may be a dict or a key/value tuple
+ return (sorted(dict(a[0]).items()), [e.expression for e in a[1]])
def test_plan(self, rql, expected, kwargs=None):
with self.session.new_cnx() as cnx:
@@ -57,7 +57,7 @@
'expected %s queries, got %s' % (len(equeries), len(queries)))
for i, (rql, sol) in enumerate(queries):
self.assertEqual(rql, equeries[i][0])
- self.assertEqual(sorted(sol), sorted(equeries[i][1]))
+ self.assertEqual(sorted(sorted(x.items()) for x in sol), sorted(sorted(x.items()) for x in equeries[i][1]))
idx = 2
else:
idx = 1
@@ -66,7 +66,7 @@
self.assertEqual(len(step[-1]), len(expected[-1]),
'got %s child steps, expected %s' % (len(step[-1]), len(expected[-1])))
except AssertionError:
- print 'error on step ',
+ print('error on step ', end=' ')
pprint(step[:-1])
raise
children = step[-1]
@@ -115,7 +115,7 @@
schema_eids[x] = x.eid
for x in schema.relations():
schema_eids[x] = x.eid
- for rdef in x.rdefs.itervalues():
+ for rdef in x.rdefs.values():
schema_eids[(rdef.subject, rdef.rtype, rdef.object)] = rdef.eid
return schema_eids
@@ -127,7 +127,7 @@
for x in schema.relations():
x.eid = schema_eids[x]
schema._eid_index[x.eid] = x
- for rdef in x.rdefs.itervalues():
+ for rdef in x.rdefs.values():
rdef.eid = schema_eids[(rdef.subject, rdef.rtype, rdef.object)]
schema._eid_index[rdef.eid] = rdef
@@ -187,7 +187,7 @@
plan = self.qhelper.plan_factory(union, {}, FakeSession(self.repo))
plan.preprocess(union)
for select in union.children:
- select.solutions.sort()
+ select.solutions.sort(key=lambda x: list(x.items()))
#print '********* ppsolutions', solutions
return union
@@ -197,7 +197,7 @@
def setUp(self):
self.o = self.repo.querier
- self.session = self.repo._sessions.values()[0]
+ self.session = next(iter(self.repo._sessions.values()))
self.ueid = self.session.user.eid
assert self.ueid != -1
self.repo._type_source_cache = {} # clear cache
@@ -238,7 +238,7 @@
if simplify:
rqlhelper.simplify(rqlst)
for select in rqlst.children:
- select.solutions.sort()
+ select.solutions.sort(key=lambda x: list(x.items()))
return self.o.plan_factory(rqlst, kwargs, cnx)
def _prepare(self, cnx, rql, kwargs=None):
@@ -286,13 +286,13 @@
if rqlst.TYPE == 'select':
self.repo.vreg.rqlhelper.annotate(rqlst)
for select in rqlst.children:
- select.solutions.sort()
+ select.solutions.sort(key=lambda x: list(x.items()))
else:
- rqlst.solutions.sort()
+ rqlst.solutions.sort(key=lambda x: list(x.items()))
return self.o.plan_factory(rqlst, kwargs, cnx)
-# monkey patch some methods to get predicatable results #######################
+# monkey patch some methods to get predictable results #######################
from cubicweb import rqlrewrite
_orig_iter_relations = rqlrewrite.iter_relations
@@ -300,16 +300,15 @@
_orig_build_variantes = rqlrewrite.RQLRewriter.build_variantes
def _insert_snippets(self, snippets, varexistsmap=None):
- _orig_insert_snippets(self, sorted(snippets, snippet_cmp), varexistsmap)
+ _orig_insert_snippets(self, sorted(snippets, key=snippet_key), varexistsmap)
def _build_variantes(self, newsolutions):
variantes = _orig_build_variantes(self, newsolutions)
sortedvariantes = []
for variante in variantes:
- orderedkeys = sorted((k[1], k[2], v) for k, v in variante.iteritems())
- variante = DumbOrderedDict(sorted(variante.iteritems(),
- lambda a, b: cmp((a[0][1],a[0][2],a[1]),
- (b[0][1],b[0][2],b[1]))))
+ orderedkeys = sorted((k[1], k[2], v) for k, v in variante.items())
+ variante = DumbOrderedDict(sorted(variante.items(),
+ key=lambda a: (a[0][1], a[0][2], a[1])))
sortedvariantes.append( (orderedkeys, variante) )
return [v for ok, v in sorted(sortedvariantes)]
@@ -318,7 +317,7 @@
def _check_permissions(*args, **kwargs):
res, restricted = _orig_check_permissions(*args, **kwargs)
- res = DumbOrderedDict(sorted(res.iteritems(), lambda a, b: cmp(a[1], b[1])))
+ res = DumbOrderedDict(sorted(res.items(), key=lambda x: [y.items() for y in x[1]]))
return res, restricted
def _dummy_check_permissions(self, rqlst):
--- a/devtools/stresstester.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/stresstester.py Wed Dec 16 11:23:48 2015 +0100
@@ -41,6 +41,7 @@
Copyright (c) 2003-2011 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.
http://www.logilab.fr/ -- mailto:contact@logilab.fr
"""
+from __future__ import print_function
import os
import sys
@@ -84,7 +85,7 @@
def usage(status=0):
"""print usage string and exit"""
- print __doc__ % basename(sys.argv[0])
+ print(__doc__ % basename(sys.argv[0]))
sys.exit(status)
@@ -133,7 +134,7 @@
'nb-times=', 'nb-threads=',
'profile', 'report-output=',])
except Exception as ex:
- print ex
+ print(ex)
usage(1)
repeat = 100
threads = 1
@@ -155,7 +156,7 @@
elif opt in ('-P', '--profile'):
prof_file = val
elif opt in ('-o', '--report-output'):
- report_output = file(val, 'w')
+ report_output = open(val, 'w')
if len(args) != 2:
usage(1)
queries = [query for query in lines(args[1]) if not query.startswith('#')]
@@ -166,7 +167,7 @@
from cubicweb.cwconfig import instance_configuration
config = instance_configuration(args[0])
# get local access to the repository
- print "Creating repo", prof_file
+ print("Creating repo", prof_file)
repo = Repository(config, prof_file)
cnxid = repo.connect(user, password=password)
# connection to the CubicWeb repository
--- a/devtools/test/data/js_examples/dep_1.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,1 +0,0 @@
-a = 4;
--- a/devtools/test/data/js_examples/deps_2.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,1 +0,0 @@
-b = a +2;
--- a/devtools/test/data/js_examples/test_simple_failure.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,18 +0,0 @@
-$(document).ready(function() {
-
- QUnit.module("air");
-
- QUnit.test("test 1", function (assert) {
- assert.equal(2, 4);
- });
-
- QUnit.test("test 2", function (assert) {
- assert.equal('', '45');
- assert.equal('1024', '32');
- });
-
- QUnit.module("able");
- QUnit.test("test 3", function (assert) {
- assert.deepEqual(1, 1);
- });
-});
--- a/devtools/test/data/js_examples/test_simple_success.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,17 +0,0 @@
-$(document).ready(function() {
-
- QUnit.module("air");
-
- QUnit.test("test 1", function (assert) {
- assert.equal(2, 2);
- });
-
- QUnit.test("test 2", function (assert) {
- assert.equal('45', '45');
- });
-
- QUnit.module("able");
- QUnit.test("test 3", function (assert) {
- assert.deepEqual(1, 1);
- });
-});
--- a/devtools/test/data/js_examples/test_with_dep.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,9 +0,0 @@
-$(document).ready(function() {
-
- QUnit.module("air");
-
- QUnit.test("test 1", function (assert) {
- assert.equal(a, 4);
- });
-
-});
--- a/devtools/test/data/js_examples/test_with_ordered_deps.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,9 +0,0 @@
-$(document).ready(function() {
-
- QUnit.module("air");
-
- QUnit.test("test 1", function (assert) {
- assert.equal(b, 6);
- });
-
-});
--- a/devtools/test/data/js_examples/utils.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,29 +0,0 @@
-function datetuple(d) {
- return [d.getFullYear(), d.getMonth()+1, d.getDate(),
- d.getHours(), d.getMinutes()];
-}
-
-function pprint(obj) {
- print('{');
- for(k in obj) {
- print(' ' + k + ' = ' + obj[k]);
- }
- print('}');
-}
-
-function arrayrepr(array) {
- return '[' + array.join(', ') + ']';
-}
-
-function assertArrayEquals(array1, array2) {
- if (array1.length != array2.length) {
- throw new crosscheck.AssertionFailure(array1.join(', ') + ' != ' + array2.join(', '));
- }
- for (var i=0; i<array1.length; i++) {
- if (array1[i] != array2[i]) {
-
- throw new crosscheck.AssertionFailure(arrayrepr(array1) + ' and ' + arrayrepr(array2)
- + ' differs at index ' + i);
- }
- }
-}
--- a/devtools/test/data/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/test/data/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -30,4 +30,3 @@
cost = Int()
description = String(maxsize=4096, fulltextindexed=True)
identical_to = SubjectRelation('Bug', symmetric=True)
-
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/devtools/test/data/static/js_examples/dep_1.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,1 @@
+a = 4;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/devtools/test/data/static/js_examples/deps_2.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,1 @@
+b = a +2;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/devtools/test/data/static/js_examples/test_simple_failure.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,18 @@
+$(document).ready(function() {
+
+ QUnit.module("air");
+
+ QUnit.test("test 1", function (assert) {
+ assert.equal(2, 4);
+ });
+
+ QUnit.test("test 2", function (assert) {
+ assert.equal('', '45');
+ assert.equal('1024', '32');
+ });
+
+ QUnit.module("able");
+ QUnit.test("test 3", function (assert) {
+ assert.deepEqual(1, 1);
+ });
+});
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/devtools/test/data/static/js_examples/test_simple_success.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,17 @@
+$(document).ready(function() {
+
+ QUnit.module("air");
+
+ QUnit.test("test 1", function (assert) {
+ assert.equal(2, 2);
+ });
+
+ QUnit.test("test 2", function (assert) {
+ assert.equal('45', '45');
+ });
+
+ QUnit.module("able");
+ QUnit.test("test 3", function (assert) {
+ assert.deepEqual(1, 1);
+ });
+});
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/devtools/test/data/static/js_examples/test_with_dep.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,9 @@
+$(document).ready(function() {
+
+ QUnit.module("air");
+
+ QUnit.test("test 1", function (assert) {
+ assert.equal(a, 4);
+ });
+
+});
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/devtools/test/data/static/js_examples/test_with_ordered_deps.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,9 @@
+$(document).ready(function() {
+
+ QUnit.module("air");
+
+ QUnit.test("test 1", function (assert) {
+ assert.equal(b, 6);
+ });
+
+});
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/devtools/test/data/static/js_examples/utils.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,29 @@
+function datetuple(d) {
+ return [d.getFullYear(), d.getMonth()+1, d.getDate(),
+ d.getHours(), d.getMinutes()];
+}
+
+function pprint(obj) {
+ print('{');
+ for(k in obj) {
+ print(' ' + k + ' = ' + obj[k]);
+ }
+ print('}');
+}
+
+function arrayrepr(array) {
+ return '[' + array.join(', ') + ']';
+}
+
+function assertArrayEquals(array1, array2) {
+ if (array1.length != array2.length) {
+ throw new crosscheck.AssertionFailure(array1.join(', ') + ' != ' + array2.join(', '));
+ }
+ for (var i=0; i<array1.length; i++) {
+ if (array1[i] != array2[i]) {
+
+ throw new crosscheck.AssertionFailure(arrayrepr(array1) + ' and ' + arrayrepr(array2)
+ + ' differs at index ' + i);
+ }
+ }
+}
--- a/devtools/test/unittest_dbfill.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/test/unittest_dbfill.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,6 +21,9 @@
import os.path as osp
import re
import datetime
+import io
+
+from six.moves import range
from logilab.common.testlib import TestCase, unittest_main
@@ -50,7 +53,7 @@
return None
def _available_Person_firstname(self, etype, attrname):
- return [f.strip() for f in file(osp.join(DATADIR, 'firstnames.txt'))]
+ return [f.strip() for f in io.open(osp.join(DATADIR, 'firstnames.txt'), encoding='latin1')]
def setUp(self):
config = ApptestConfiguration('data', apphome=DATADIR)
@@ -86,7 +89,7 @@
# Test for random index
for index in range(5):
cost_value = self.bug_valgen.generate_attribute_value({}, 'cost', index)
- self.assertIn(cost_value, range(index+1))
+ self.assertIn(cost_value, list(range(index+1)))
def test_date(self):
"""test date generation"""
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/devtools/test/unittest_devctl.py Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,47 @@
+# copyright 2003-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+"""unit tests for cubicweb-ctl commands from devtools"""
+
+import os.path as osp
+import sys
+import tempfile
+import shutil
+from subprocess import Popen, PIPE, STDOUT
+from unittest import TestCase
+
+
+class CubicWebCtlTC(TestCase):
+ """test case for devtools commands"""
+
+ def test_newcube(self):
+ cwctl = osp.abspath(osp.join(osp.dirname(__file__), '../../bin/cubicweb-ctl'))
+
+ tmpdir = tempfile.mkdtemp(prefix="temp-cwctl-newcube")
+ try:
+ cmd = [sys.executable, cwctl, 'newcube',
+ '--directory', tmpdir, 'foo']
+ proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
+ stdout, _ = proc.communicate(b'short_desc\n')
+ finally:
+ shutil.rmtree(tmpdir, ignore_errors=True)
+ self.assertEqual(proc.returncode, 0, msg=stdout)
+
+
+if __name__ == '__main__':
+ from unittest import main
+ main()
--- a/devtools/test/unittest_httptest.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/test/unittest_httptest.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,7 +17,7 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""unittest for cubicweb.devtools.httptest module"""
-import httplib
+from six.moves import http_client
from logilab.common.testlib import Tags
from cubicweb.devtools.httptest import CubicWebServerTC
@@ -28,12 +28,12 @@
def test_response(self):
try:
response = self.web_get()
- except httplib.NotConnected as ex:
+ except http_client.NotConnected as ex:
self.fail("Can't connection to test server: %s" % ex)
def test_response_anon(self):
response = self.web_get()
- self.assertEqual(response.status, httplib.OK)
+ self.assertEqual(response.status, http_client.OK)
def test_base_url(self):
if self.config['base-url'] not in self.web_get().read():
@@ -47,20 +47,20 @@
def test_response_denied(self):
response = self.web_get()
- self.assertEqual(response.status, httplib.FORBIDDEN)
+ self.assertEqual(response.status, http_client.FORBIDDEN)
def test_login(self):
response = self.web_get()
- if response.status != httplib.FORBIDDEN:
+ if response.status != http_client.FORBIDDEN:
self.skipTest('Already authenticated, "test_response_denied" must have failed')
# login
self.web_login(self.admlogin, self.admpassword)
response = self.web_get()
- self.assertEqual(response.status, httplib.OK, response.body)
+ self.assertEqual(response.status, http_client.OK, response.body)
# logout
self.web_logout()
response = self.web_get()
- self.assertEqual(response.status, httplib.FORBIDDEN, response.body)
+ self.assertEqual(response.status, http_client.FORBIDDEN, response.body)
--- a/devtools/test/unittest_i18n.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/test/unittest_i18n.py Wed Dec 16 11:23:48 2015 +0100
@@ -32,21 +32,22 @@
"""load a po file and return a set of encountered (msgid, msgctx)"""
msgs = set()
msgid = msgctxt = None
- for line in open(fname):
- if line.strip() in ('', '#'):
- continue
- if line.startswith('msgstr'):
- assert not (msgid, msgctxt) in msgs
- msgs.add( (msgid, msgctxt) )
- msgid = msgctxt = None
- elif line.startswith('msgid'):
- msgid = line.split(' ', 1)[1][1:-1]
- elif line.startswith('msgctx'):
- msgctxt = line.split(' ', 1)[1][1: -1]
- elif msgid is not None:
- msgid += line[1:-1]
- elif msgctxt is not None:
- msgctxt += line[1:-1]
+ with open(fname) as fobj:
+ for line in fobj:
+ if line.strip() in ('', '#'):
+ continue
+ if line.startswith('msgstr'):
+ assert not (msgid, msgctxt) in msgs
+ msgs.add( (msgid, msgctxt) )
+ msgid = msgctxt = None
+ elif line.startswith('msgid'):
+ msgid = line.split(' ', 1)[1][1:-1]
+ elif line.startswith('msgctx'):
+ msgctxt = line.split(' ', 1)[1][1: -1]
+ elif msgid is not None:
+ msgid += line[1:-1]
+ elif msgctxt is not None:
+ msgctxt += line[1:-1]
return msgs
--- a/devtools/test/unittest_qunit.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/test/unittest_qunit.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,15 +1,10 @@
-from logilab.common.testlib import unittest_main
-from cubicweb.devtools.qunit import QUnitTestCase
-
-from os import path as osp
-
-JSTESTDIR = osp.abspath(osp.join(osp.dirname(__file__), 'data', 'js_examples'))
+from cubicweb.devtools import qunit
def js(name):
- return osp.join(JSTESTDIR, name)
+ return '/static/js_examples/' + name
-class QUnitTestCaseTC(QUnitTestCase):
+class QUnitTestCaseTC(qunit.QUnitTestCase):
all_js_tests = (
(js('test_simple_success.js'),),
@@ -28,4 +23,5 @@
if __name__ == '__main__':
- unittest_main()
+ from unittest import main
+ main()
--- a/devtools/test/unittest_testlib.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/test/unittest_testlib.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,9 +17,10 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""unittests for cw.devtools.testlib module"""
-from cStringIO import StringIO
+from io import BytesIO, StringIO
+from unittest import TextTestRunner
-from unittest import TextTestRunner
+from six import PY2
from logilab.common.testlib import TestSuite, TestCase, unittest_main
from logilab.common.registry import yes
@@ -33,7 +34,7 @@
class entity:
cw_etype = 'Entity'
eid = 0
- sio = StringIO('hop\n')
+ sio = BytesIO(b'hop\n')
form = CubicWebTC.fake_form('import',
{'file': ('filename.txt', sio),
'encoding': u'utf-8',
@@ -51,7 +52,7 @@
class WebTestTC(TestCase):
def setUp(self):
- output = StringIO()
+ output = BytesIO() if PY2 else StringIO()
self.runner = TextTestRunner(stream=output)
def test_error_raised(self):
--- a/devtools/test/unittest_webtest.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/test/unittest_webtest.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,4 +1,4 @@
-import httplib
+from six.moves import http_client
from logilab.common.testlib import Tags
from cubicweb.devtools.webtest import CubicWebTestTC
@@ -21,19 +21,19 @@
def test_reponse_denied(self):
res = self.webapp.get('/', expect_errors=True)
- self.assertEqual(httplib.FORBIDDEN, res.status_int)
+ self.assertEqual(http_client.FORBIDDEN, res.status_int)
def test_login(self):
res = self.webapp.get('/', expect_errors=True)
- self.assertEqual(httplib.FORBIDDEN, res.status_int)
+ self.assertEqual(http_client.FORBIDDEN, res.status_int)
self.login(self.admlogin, self.admpassword)
res = self.webapp.get('/')
- self.assertEqual(httplib.OK, res.status_int)
+ self.assertEqual(http_client.OK, res.status_int)
self.logout()
res = self.webapp.get('/', expect_errors=True)
- self.assertEqual(httplib.FORBIDDEN, res.status_int)
+ self.assertEqual(http_client.FORBIDDEN, res.status_int)
if __name__ == '__main__':
--- a/devtools/testlib.py Tue Dec 15 19:05:41 2015 +0100
+++ b/devtools/testlib.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,19 +16,19 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""this module contains base classes and utilities for cubicweb tests"""
-__docformat__ = "restructuredtext en"
+from __future__ import print_function
import sys
import re
-import urlparse
from os.path import dirname, join, abspath
-from urllib import unquote
from math import log
from contextlib import contextmanager
-from warnings import warn
-from types import NoneType
from itertools import chain
+from six import text_type, string_types
+from six.moves import range
+from six.moves.urllib.parse import urlparse, parse_qs, unquote as urlunquote
+
import yams.schema
from logilab.common.testlib import TestCase, InnerTest, Tags
@@ -40,7 +40,7 @@
from logilab.common.shellutils import getlogin
from cubicweb import (ValidationError, NoSelectableObject, AuthenticationError,
- ProgrammingError, BadConnectionId)
+ BadConnectionId)
from cubicweb import cwconfig, devtools, web, server, repoapi
from cubicweb.utils import json
from cubicweb.sobjects import notification
@@ -49,7 +49,7 @@
from cubicweb.server.session import Session
from cubicweb.devtools import SYSTEM_ENTITIES, SYSTEM_RELATIONS, VIEW_VALIDATORS
from cubicweb.devtools import fake, htmlparser, DEFAULT_EMPTY_DB_ID
-from cubicweb.utils import json
+
# low-level utilities ##########################################################
@@ -60,10 +60,11 @@
def do_view(self, arg):
import webbrowser
data = self._getval(arg)
- with file('/tmp/toto.html', 'w') as toto:
+ with open('/tmp/toto.html', 'w') as toto:
toto.write(data)
webbrowser.open('file:///tmp/toto.html')
+
def line_context_filter(line_no, center, before=3, after=None):
"""return true if line are in context
@@ -73,6 +74,7 @@
after = before
return center - before <= line_no <= center + after
+
def unprotected_entities(schema, strict=False):
"""returned a set of each non final entity type, excluding "system" entities
(eg CWGroup, CWUser...)
@@ -83,9 +85,11 @@
protected_entities = yams.schema.BASE_TYPES.union(SYSTEM_ENTITIES)
return set(schema.entities()) - protected_entities
+
class JsonValidator(object):
def parse_string(self, data):
- return json.loads(data)
+ return json.loads(data.decode('ascii'))
+
@contextmanager
def real_error_handling(app):
@@ -109,10 +113,12 @@
# restore
app.error_handler = fake_error_handler
+
# email handling, to test emails sent by an application ########################
MAILBOX = []
+
class Email(object):
"""you'll get instances of Email into MAILBOX during tests that trigger
some notification.
@@ -143,13 +149,17 @@
return '<Email to %s with subject %s>' % (','.join(self.recipients),
self.message.get('Subject'))
+
# the trick to get email into MAILBOX instead of actually sent: monkey patch
# cwconfig.SMTP object
class MockSMTP:
+
def __init__(self, server, port):
pass
+
def close(self):
pass
+
def sendmail(self, fromaddr, recipients, msg):
MAILBOX.append(Email(fromaddr, recipients, msg))
@@ -243,7 +253,6 @@
cnx.commit()
-
# base class for cubicweb tests requiring a full cw environments ###############
class CubicWebTC(TestCase):
@@ -283,7 +292,7 @@
"""provide a new RepoAccess object for a given user
The access is automatically closed at the end of the test."""
- login = unicode(login)
+ login = text_type(login)
access = RepoAccess(self.repo, login, self.requestcls)
self._open_access.add(access)
return access
@@ -293,14 +302,14 @@
try:
self._open_access.pop().close()
except BadConnectionId:
- continue # already closed
+ continue # already closed
@property
def session(self):
"""return admin session"""
return self._admin_session
- #XXX this doesn't need to a be classmethod anymore
+ # XXX this doesn't need to a be classmethod anymore
def _init_repo(self):
"""init the repository and connection to it.
"""
@@ -310,11 +319,10 @@
db_handler.restore_database(self.test_db_id)
self.repo = db_handler.get_repo(startup=True)
# get an admin session (without actual login)
- login = unicode(db_handler.config.default_admin_config['login'])
+ login = text_type(db_handler.config.default_admin_config['login'])
self.admin_access = self.new_access(login)
self._admin_session = self.admin_access._session
-
# config management ########################################################
@classproperty
@@ -323,8 +331,11 @@
Configuration is cached on the test class.
"""
+ if cls is CubicWebTC:
+ # Prevent direct use of CubicWebTC directly to avoid database
+ # caching issues
+ return None
try:
- assert not cls is CubicWebTC, "Don't use CubicWebTC directly to prevent database caching issue"
return cls.__dict__['_config']
except KeyError:
home = abspath(join(dirname(sys.modules[cls.__module__].__file__), cls.appid))
@@ -332,7 +343,7 @@
config.mode = 'test'
return config
- @classmethod # XXX could be turned into a regular method
+ @classmethod # XXX could be turned into a regular method
def init_config(cls, config):
"""configuration initialization hooks.
@@ -345,16 +356,16 @@
been properly bootstrapped.
"""
admincfg = config.default_admin_config
- cls.admlogin = unicode(admincfg['login'])
+ cls.admlogin = text_type(admincfg['login'])
cls.admpassword = admincfg['password']
# uncomment the line below if you want rql queries to be logged
- #config.global_set_option('query-log-file',
- # '/tmp/test_rql_log.' + `os.getpid()`)
+ # config.global_set_option('query-log-file',
+ # '/tmp/test_rql_log.' + `os.getpid()`)
config.global_set_option('log-file', None)
# set default-dest-addrs to a dumb email address to avoid mailbox or
# mail queue pollution
config.global_set_option('default-dest-addrs', ['whatever'])
- send_to = '%s@logilab.fr' % getlogin()
+ send_to = '%s@logilab.fr' % getlogin()
config.global_set_option('sender-addr', send_to)
config.global_set_option('default-dest-addrs', send_to)
config.global_set_option('sender-name', 'cubicweb-test')
@@ -364,15 +375,13 @@
# web resources
try:
config.global_set_option('embed-allowed', re.compile('.*'))
- except Exception: # not in server only configuration
+ except Exception: # not in server only configuration
pass
- config.set_anonymous_allowed(cls.anonymous_allowed)
@property
def vreg(self):
return self.repo.vreg
-
# global resources accessors ###############################################
@property
@@ -404,8 +413,9 @@
self.__class__._repo_init_failed = ex
raise
self.addCleanup(self._close_access)
+ self.config.set_anonymous_allowed(self.anonymous_allowed)
self.setup_database()
- MAILBOX[:] = [] # reset mailbox
+ MAILBOX[:] = [] # reset mailbox
def tearDown(self):
# XXX hack until logilab.common.testlib is fixed
@@ -421,8 +431,10 @@
# monkey patch send mail operation so emails are sent synchronously
_old_mail_postcommit_event = SendMailOp.postcommit_event
SendMailOp.postcommit_event = SendMailOp.sendmails
+
def reverse_SendMailOp_monkey_patch():
SendMailOp.postcommit_event = _old_mail_postcommit_event
+
self.addCleanup(reverse_SendMailOp_monkey_patch)
def setup_database(self):
@@ -446,31 +458,30 @@
else:
return req.user
- @iclassmethod # XXX turn into a class method
+ @iclassmethod # XXX turn into a class method
def create_user(self, req, login=None, groups=('users',), password=None,
email=None, commit=True, **kwargs):
"""create and return a new user entity"""
if password is None:
password = login
if login is not None:
- login = unicode(login)
+ login = text_type(login)
user = req.create_entity('CWUser', login=login,
upassword=password, **kwargs)
req.execute('SET X in_group G WHERE X eid %%(x)s, G name IN(%s)'
% ','.join(repr(str(g)) for g in groups),
{'x': user.eid})
if email is not None:
- req.create_entity('EmailAddress', address=unicode(email),
+ req.create_entity('EmailAddress', address=text_type(email),
reverse_primary_email=user)
user.cw_clear_relation_cache('in_group', 'subject')
if commit:
try:
- req.commit() # req is a session
+ req.commit() # req is a session
except AttributeError:
req.cnx.commit()
return user
-
# other utilities #########################################################
@contextmanager
@@ -518,10 +529,10 @@
similar to `orig_permissions.update(partial_perms)`.
"""
torestore = []
- for erschema, etypeperms in chain(perm_overrides, perm_kwoverrides.iteritems()):
- if isinstance(erschema, basestring):
+ for erschema, etypeperms in chain(perm_overrides, perm_kwoverrides.items()):
+ if isinstance(erschema, string_types):
erschema = self.schema[erschema]
- for action, actionperms in etypeperms.iteritems():
+ for action, actionperms in etypeperms.items():
origperms = erschema.permissions[action]
erschema.set_action_permissions(action, actionperms)
torestore.append([erschema, action, origperms])
@@ -549,7 +560,6 @@
self.assertListEqual(sorted(tr.name for tr in transitions),
sorted(expected))
-
# views and actions registries inspection ##################################
def pviews(self, req, rset):
@@ -585,6 +595,7 @@
@property
def items(self):
return self
+
class fake_box(object):
def action_link(self, action, **kwargs):
return (action.title, action.url())
@@ -611,7 +622,7 @@
try:
view = viewsvreg._select_best(views, req, rset=rset)
if view is None:
- raise NoSelectableObject((req,), {'rset':rset}, views)
+ raise NoSelectableObject((req,), {'rset': rset}, views)
if view.linkable():
yield view
else:
@@ -642,7 +653,6 @@
else:
not_selected(self.vreg, view)
-
# web ui testing utilities #################################################
@property
@@ -650,8 +660,10 @@
def app(self):
"""return a cubicweb publisher"""
publisher = application.CubicWebPublisher(self.repo, self.config)
+
def raise_error_handler(*args, **kwargs):
raise
+
publisher.error_handler = raise_error_handler
return publisher
@@ -703,7 +715,7 @@
for fields that are not tied to the given entity
"""
assert field_dict or entity_field_dicts, \
- 'field_dict and entity_field_dicts arguments must not be both unspecified'
+ 'field_dict and entity_field_dicts arguments must not be both unspecified'
if field_dict is None:
field_dict = {}
form = {'__form_id': formid}
@@ -711,9 +723,11 @@
for field, value in field_dict.items():
fields.append(field)
form[field] = value
+
def _add_entity_field(entity, field, value):
entity_fields.append(field)
form[eid_param(field, entity.eid)] = value
+
for entity, field_dict in entity_field_dicts:
if '__maineid' not in form:
form['__maineid'] = entity.eid
@@ -736,9 +750,9 @@
"""
req = self.request(url=url)
if isinstance(url, unicode):
- url = url.encode(req.encoding) # req.setup_params() expects encoded strings
- querystring = urlparse.urlparse(url)[-2]
- params = urlparse.parse_qs(querystring)
+ url = url.encode(req.encoding) # req.setup_params() expects encoded strings
+ querystring = urlparse(url)[-2]
+ params = parse_qs(querystring)
req.setup_params(params)
return req
@@ -750,9 +764,9 @@
"""
with self.admin_access.web_request(url=url) as req:
if isinstance(url, unicode):
- url = url.encode(req.encoding) # req.setup_params() expects encoded strings
- querystring = urlparse.urlparse(url)[-2]
- params = urlparse.parse_qs(querystring)
+ url = url.encode(req.encoding) # req.setup_params() expects encoded strings
+ querystring = urlparse(url)[-2]
+ params = parse_qs(querystring)
req.setup_params(params)
yield req
@@ -791,9 +805,9 @@
path = location
params = {}
else:
- cleanup = lambda p: (p[0], unquote(p[1]))
+ cleanup = lambda p: (p[0], urlunquote(p[1]))
params = dict(cleanup(p.split('=', 1)) for p in params.split('&') if p)
- if path.startswith(req.base_url()): # may be relative
+ if path.startswith(req.base_url()): # may be relative
path = path[len(req.base_url()):]
return path, params
@@ -812,8 +826,8 @@
"""call the publish method of the application publisher, expecting to
get a Redirect exception
"""
- result = self.app_handle_request(req, path)
- self.assertTrue(300 <= req.status_out <400, req.status_out)
+ self.app_handle_request(req, path)
+ self.assertTrue(300 <= req.status_out < 400, req.status_out)
location = req.get_response_header('location')
return self._parse_location(req, location)
@@ -822,7 +836,6 @@
def expect_redirect_publish(self, *args, **kwargs):
return self.expect_redirect_handle_request(*args, **kwargs)
-
def set_auth_mode(self, authmode, anonuser=None):
self.set_option('auth-mode', authmode)
self.set_option('anonymous-user', anonuser)
@@ -871,8 +884,8 @@
#
# do not set html validators here, we need HTMLValidator for html
# snippets
- #'text/html': DTDValidator,
- #'application/xhtml+xml': DTDValidator,
+ # 'text/html': DTDValidator,
+ # 'application/xhtml+xml': DTDValidator,
'application/xml': htmlparser.XMLValidator,
'text/xml': htmlparser.XMLValidator,
'application/json': JsonValidator,
@@ -884,8 +897,7 @@
}
# maps vid : validator name (override content_type_validators)
vid_validators = dict((vid, htmlparser.VALMAP[valkey])
- for vid, valkey in VIEW_VALIDATORS.iteritems())
-
+ for vid, valkey in VIEW_VALIDATORS.items())
def view(self, vid, rset=None, req=None, template='main-template',
**kwargs):
@@ -907,12 +919,15 @@
view = viewsreg.select(vid, req, rset=rset, **kwargs)
# set explicit test description
if rset is not None:
+ # coerce to "bytes" on py2 because the description will be sent to
+ # sys.stdout/stderr which takes "bytes" on py2 and "unicode" on py3
+ rql = str(rset.printable_rql())
self.set_description("testing vid=%s defined in %s with (%s)" % (
- vid, view.__module__, rset.printable_rql()))
+ vid, view.__module__, rql))
else:
self.set_description("testing vid=%s defined in %s without rset" % (
vid, view.__module__))
- if template is None: # raw view testing, no template
+ if template is None: # raw view testing, no template
viewfunc = view.render
else:
kwargs['view'] = view
@@ -920,7 +935,6 @@
rset=rset, **kwargs)
return self._test_view(viewfunc, view, template, kwargs)
-
def _test_view(self, viewfunc, view, template='main-template', kwargs={}):
"""this method does the actual call to the view
@@ -940,7 +954,9 @@
msg = '[%s in %s] %s' % (klass, view.__regid__, exc)
except Exception:
msg = '[%s in %s] undisplayable exception' % (klass, view.__regid__)
- raise AssertionError, msg, tcbk
+ exc = AssertionError(msg)
+ exc.__traceback__ = tcbk
+ raise exc
return self._check_html(output, view, template)
def get_validator(self, view=None, content_type=None, output=None):
@@ -953,11 +969,11 @@
if content_type is None:
content_type = 'text/html'
if content_type in ('text/html', 'application/xhtml+xml') and output:
- if output.startswith('<!DOCTYPE html>'):
+ if output.startswith(b'<!DOCTYPE html>'):
# only check XML well-formness since HTMLValidator isn't html5
# compatible and won't like various other extensions
default_validator = htmlparser.XMLSyntaxValidator
- elif output.startswith('<?xml'):
+ elif output.startswith(b'<?xml'):
default_validator = htmlparser.DTDValidator
else:
default_validator = htmlparser.HTMLValidator
@@ -973,13 +989,16 @@
def _check_html(self, output, view, template='main-template'):
"""raises an exception if the HTML is invalid"""
output = output.strip()
+ if isinstance(output, text_type):
+ # XXX
+ output = output.encode('utf-8')
validator = self.get_validator(view, output=output)
if validator is None:
- return output # return raw output if no validator is defined
+ return output # return raw output if no validator is defined
if isinstance(validator, htmlparser.DTDValidator):
# XXX remove <canvas> used in progress widget, unknown in html dtd
output = re.sub('<canvas.*?></canvas>', '', output)
- return self.assertWellFormed(validator, output.strip(), context= view.__regid__)
+ return self.assertWellFormed(validator, output.strip(), context=view.__regid__)
def assertWellFormed(self, validator, content, context=None):
try:
@@ -998,7 +1017,7 @@
str_exc = str(exc)
except Exception:
str_exc = 'undisplayable exception'
- msg += str_exc
+ msg += str_exc.encode(sys.getdefaultencoding(), 'replace')
if content is not None:
position = getattr(exc, "position", (0,))[0]
if position:
@@ -1015,7 +1034,9 @@
for idx, line in enumerate(content)
if line_context_filter(idx+1, position))
msg += u'\nfor content:\n%s' % content
- raise AssertionError, msg, tcbk
+ exc = AssertionError(msg)
+ exc.__traceback__ = tcbk
+ raise exc
def assertDocTestFile(self, testfile):
# doctest returns tuple (failure_count, test_count)
@@ -1053,6 +1074,7 @@
# XXX cleanup unprotected_entities & all mess
+
def how_many_dict(schema, cnx, how_many, skip):
"""given a schema, compute how many entities by type we need to be able to
satisfy relations cardinality.
@@ -1081,7 +1103,7 @@
# reverse subj and obj in the above explanation
relmap.setdefault((rschema, obj), []).append(str(subj))
unprotected = unprotected_entities(schema)
- for etype in skip: # XXX (syt) duh? explain or kill
+ for etype in skip: # XXX (syt) duh? explain or kill
unprotected.add(etype)
howmanydict = {}
# step 1, compute a base number of each entity types: number of already
@@ -1096,7 +1118,7 @@
# new num for etype = max(current num, sum(num for possible target etypes))
#
# XXX we should first check there is no cycle then propagate changes
- for (rschema, etype), targets in relmap.iteritems():
+ for (rschema, etype), targets in relmap.items():
relfactor = sum(howmanydict[e] for e in targets)
howmanydict[str(etype)] = max(relfactor, howmanydict[etype])
return howmanydict
@@ -1127,7 +1149,6 @@
def post_populate(self, cnx):
pass
-
@nocoverage
def auto_populate(self, how_many):
"""this method populates the database with `how_many` entities
@@ -1166,8 +1187,8 @@
cnx.execute(rql, args)
except ValidationError as ex:
# failed to satisfy some constraint
- print 'error in automatic db population', ex
- cnx.commit_state = None # reset uncommitable flag
+ print('error in automatic db population', ex)
+ cnx.commit_state = None # reset uncommitable flag
self.post_populate(cnx)
def iter_individual_rsets(self, etypes=None, limit=None):
@@ -1179,7 +1200,7 @@
else:
rql = 'Any X WHERE X is %s' % etype
rset = req.execute(rql)
- for row in xrange(len(rset)):
+ for row in range(len(rset)):
if limit and row > limit:
break
# XXX iirk
@@ -1202,7 +1223,8 @@
# test a mixed query (DISTINCT/GROUP to avoid getting duplicate
# X which make muledit view failing for instance (html validation fails
# because of some duplicate "id" attributes)
- yield req.execute('DISTINCT Any X, MAX(Y) GROUPBY X WHERE X is %s, Y is %s' % (etype1, etype2))
+ yield req.execute('DISTINCT Any X, MAX(Y) GROUPBY X WHERE X is %s, Y is %s' %
+ (etype1, etype2))
# test some application-specific queries if defined
for rql in self.application_rql:
yield req.execute(rql)
@@ -1225,7 +1247,8 @@
# resultset's syntax tree
rset = backup_rset
for action in self.list_actions_for(rset):
- yield InnerTest(self._testname(rset, action.__regid__, 'action'), self._test_action, action)
+ yield InnerTest(self._testname(rset, action.__regid__, 'action'),
+ self._test_action, action)
for box in self.list_boxes_for(rset):
w = [].append
yield InnerTest(self._testname(rset, box.__regid__, 'box'), box.render, w)
@@ -1243,28 +1266,28 @@
tags = AutoPopulateTest.tags | Tags('web', 'generated')
def setUp(self):
- assert not self.__class__ is AutomaticWebTest, 'Please subclass AutomaticWebTest to prevent database caching issue'
+ if self.__class__ is AutomaticWebTest:
+ # Prevent direct use of AutomaticWebTest to avoid database caching
+ # issues.
+ return
super(AutomaticWebTest, self).setUp()
# access to self.app for proper initialization of the authentication
# machinery (else some views may fail)
self.app
- ## one each
def test_one_each_config(self):
self.auto_populate(1)
for rset in self.iter_automatic_rsets(limit=1):
for testargs in self._test_everything_for(rset):
yield testargs
- ## ten each
def test_ten_each_config(self):
self.auto_populate(10)
for rset in self.iter_automatic_rsets(limit=10):
for testargs in self._test_everything_for(rset):
yield testargs
- ## startup views
def test_startup_views(self):
for vid in self.list_startup_views():
with self.admin_access.web_request() as req:
@@ -1284,7 +1307,7 @@
# # XXX broken
# from cubicweb.devtools.apptest import TestEnvironment
# env = testclass._env = TestEnvironment('data', configcls=testclass.configcls)
-# for reg in env.vreg.itervalues():
+# for reg in env.vreg.values():
# reg._selected = {}
# try:
# orig_select_best = reg.__class__.__orig_select_best
@@ -1304,10 +1327,10 @@
# def print_untested_objects(testclass, skipregs=('hooks', 'etypes')):
-# for regname, reg in testclass._env.vreg.iteritems():
+# for regname, reg in testclass._env.vreg.items():
# if regname in skipregs:
# continue
-# for appobjects in reg.itervalues():
+# for appobjects in reg.values():
# for appobject in appobjects:
# if not reg._selected.get(appobject):
# print 'not tested', regname, appobject
--- a/doc/book/devrepo/fti.rst Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/book/devrepo/fti.rst Wed Dec 16 11:23:48 2015 +0100
@@ -94,37 +94,10 @@
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
``db-rebuild-fti`` will call the
-:meth:`~cubicweb.entities.AnyEntity.cw_fti_index_rql_queries` class
+:meth:`~cubicweb.entities.AnyEntity.cw_fti_index_rql_limit` class
method on your entity type.
-.. automethod:: cubicweb.entities.AnyEntity.cw_fti_index_rql_queries
-
-Now, suppose you've got a _huge_ table to index, you probably don't want to
-get all entities at once. So here's a simple customized example that will
-process block of 10000 entities:
-
-.. sourcecode:: python
-
-
- class MyEntityClass(AnyEntity):
- __regid__ = 'MyEntityClass'
-
- @classmethod
- def cw_fti_index_rql_queries(cls, req):
- # get the default RQL method and insert LIMIT / OFFSET instructions
- base_rql = super(SearchIndex, cls).cw_fti_index_rql_queries(req)[0]
- selected, restrictions = base_rql.split(' WHERE ')
- rql_template = '%s ORDERBY X LIMIT %%(limit)s OFFSET %%(offset)s WHERE %s' % (
- selected, restrictions)
- # count how many entities you'll have to index
- count = req.execute('Any COUNT(X) WHERE X is MyEntityClass')[0][0]
- # iterate by blocks of 10000 entities
- chunksize = 10000
- for offset in xrange(0, count, chunksize):
- print 'SENDING', rql_template % {'limit': chunksize, 'offset': offset}
- yield rql_template % {'limit': chunksize, 'offset': offset}
-
-Since you have access to ``req``, you can more or less fetch whatever you want.
+.. automethod:: cubicweb.entities.AnyEntity.cw_fti_index_rql_limit
Customizing :meth:`~cubicweb.entities.adapters.IFTIndexableAdapter.get_words`
--- a/doc/book/devrepo/testing.rst Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/book/devrepo/testing.rst Wed Dec 16 11:23:48 2015 +0100
@@ -324,9 +324,9 @@
def test_blog_rss(self):
with self.admin_access.web_request() as req:
- rset = req.execute('Any B ORDERBY D DESC WHERE B is BlogEntry, '
- 'B created_by U, U login "logilab", B creation_date D')
- self.view('rss', rset, req=req)
+ rset = req.execute('Any B ORDERBY D DESC WHERE B is BlogEntry, '
+ 'B created_by U, U login "logilab", B creation_date D')
+ self.view('rss', rset, req=req)
Testing with other cubes
--- a/doc/book/devweb/views/table.rst Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/book/devweb/views/table.rst Wed Dec 16 11:23:48 2015 +0100
@@ -96,8 +96,8 @@
'resource': MainEntityColRenderer(),
'workpackage': EntityTableColRenderer(
header='Workpackage',
- renderfunc=worpackage_cell,
- sortfunc=worpackage_sortvalue,),
+ renderfunc=workpackage_cell,
+ sortfunc=workpackage_sortvalue,),
'in_state': EntityTableColRenderer(
renderfunc=lambda w,x: w(x.cw_adapt_to('IWorkflowable').printable_state),
sortfunc=lambda x: x.cw_adapt_to('IWorkflowable').printable_state),
--- a/doc/changes/3.21.rst Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/changes/3.21.rst Wed Dec 16 11:23:48 2015 +0100
@@ -1,5 +1,5 @@
-3.21
-====
+3.21 (10 July 2015)
+===================
New features
------------
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/doc/changes/3.22.rst Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,71 @@
+3.22
+====
+
+New features
+------------
+
+* a huge amount of changes were done towards python 3.x support (as yet
+ incomplete). This introduces a new dependency on six, to handle
+ python2/python3 compatibility.
+
+* new cubicweb.dataimport.massive_store module, a postgresql-specific store
+ using the COPY statement to accelerate massive data imports. This
+ functionality was previously part of cubicweb-dataio (there are some API
+ differences with that previous version, however).
+
+* cubes custom sql scripts are executed before creating tables. This allows
+ them to create new types or extensions.
+
+User-visible changes
+--------------------
+
+* the ldapfeed source now depends on the `ldap3` module instead of
+ `python-ldap`.
+
+* replies don't get an ``Expires`` header by default. However when they do,
+ they also get a coherent ``Cache-Control``.
+
+* data files are regenerated at each request, they are no longer cached by
+ ``cubicweb.web.PropertySheet``. Requests for data files missing the instance
+ hash are handled with a redirection instead of a direct reply, to allow
+ correct cache-related reply headers.
+
+API changes
+-----------
+
+* ``config.repository()`` creates a new Repository object each time, instead of
+ returning a cached object
+
+* migration scripts, as well as other scripts executed by ``cubicweb-ctl
+ shell``, are loaded with the print_function flag enabled (for backwards
+ compatibility, if that fails they are re-loaded without that flag)
+
+* the ``cw_fti_index_rql_queries`` method on entity classes is replaced by
+ ``cw_fti_index_rql_limit``, a generator which yields ``ResultSet`` objects
+ containing entities to be indexed. By default, entities are returned 1000 at
+ a time.
+
+* ``IDownloadableAdapter`` API is clarified: ``download_url``,
+ ``download_content_type`` and ``download_file_name`` return unicode objects,
+ ``download_data`` returns bytes.
+
+* the ``Repository.extid2eid()`` entry point for external sources is deprecated.
+ Imports should use one of the stores from the ``cubicweb.dataimport`` package
+ instead.
+
+* the ``cubicweb.repoapi.get_repository()`` function's ``uri`` argument should
+ no longer be used.
+
+* the generic datafeed xml parser is deprecated in favor of the "store" API
+ introduced in cubicweb 3.21.
+
+* the session manager lives in the ``sessions`` registry instead of ``components``.
+
+Deprecated code drops
+---------------------
+
+* the ``cubicweb.server.hooksmanager`` module was removed
+
+* the ``Repository.pinfo()`` method was removed
+
+* the ``cubicweb.utils.SizeConstrainedList`` class was removed
--- a/doc/changes/changelog.rst Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/changes/changelog.rst Wed Dec 16 11:23:48 2015 +0100
@@ -2,6 +2,7 @@
Changelog history
===================
+.. include:: 3.22.rst
.. include:: 3.21.rst
.. include:: 3.20.rst
.. include:: 3.19.rst
--- a/doc/changes/index.rst Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/changes/index.rst Wed Dec 16 11:23:48 2015 +0100
@@ -4,6 +4,7 @@
.. toctree::
:maxdepth: 1
+ 3.22
3.21
3.20
3.19
--- a/doc/dev/features_list.rst Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/dev/features_list.rst Wed Dec 16 11:23:48 2015 +0100
@@ -2,21 +2,21 @@
CubicWeb features
=================
-This page tries to resume features found in the bare cubicweb framework,
+This page summarizes features found in the bare cubicweb framework and indicates
how mature and documented they are.
:code maturity (CM):
- 0: experimental, not ready at all for production, may be killed
- - 1: draft / unsatisfying, api may change in a near future, much probably in long
- term
+ - 1: draft / unsatisfying, API may change in a near future, and will
+ certainly change in the long term
- - 2: good enough, api sounds good but will probably evolve a bit with more
+ - 2: good enough, API sounds good but will probably evolve a bit with more
hindsight
- - 3: mature, backward incompatible changes unexpected (may still evolve though,
- of course)
+ - 3: mature, backward incompatible changes unexpected (may still evolve
+ though, of course)
:documentation level (DL):
@@ -25,7 +25,7 @@
- 1: poor documentation
- - 2: some valuable documentation but some parts keep uncovered
+ - 2: some valuable documentation but incomplete coverage
- 3: good / complete documentation
@@ -33,191 +33,306 @@
Instance configuration and maintainance
=======================================
-+====================================================================+====+====+
-| FEATURE | CM | DL |
-+====================================================================+====+====+
-| setup - installation | 2 | 3 |
-| setup - environment variables | 3 | 2 |
-| setup - running modes | 2 | 2 |
-| setup - administration tasks | 2 | 2 |
-| setup - configuration file | 2 | 1 |
-+--------------------------------------------------------------------+----+----+
-| configuration - user / groups handling | 3 | 1 |
-| configuration - site configuration | 3 | 1 |
-| configuration - distributed configuration | 2 | 1 |
-+--------------------------------------------------------------------+----+----+
-| multi-sources - capabilities | NA | 0 |
-| multi-sources - configuration | 2 | 0 |
-| multi-sources - ldap integration | 2 | 1 |
-+--------------------------------------------------------------------+----+----+
-| usage - custom ReST markup | 2 | 0 |
-| usage - personal preferences | 2 | 1 |
-+--------------------------------------------------------------------+----+----+
+.. table::
+ +-------------------------------------------------------------------+----+----+
+ | FEATURE | CM | DL |
+ +===============+===================================================+====+====+
+ | setup | installation | 2 | 3 |
+ | +---------------------------------------------------+----+----+
+ | | environment variables | 3 | 2 |
+ | +---------------------------------------------------+----+----+
+ | | running modes | 2 | 2 |
+ | +---------------------------------------------------+----+----+
+ | | administration tasks | 2 | 2 |
+ | +---------------------------------------------------+----+----+
+ | | configuration file | 2 | 1 |
+ +---------------+---------------------------------------------------+----+----+
+ | configuration | user / groups handling | 3 | 1 |
+ | +---------------------------------------------------+----+----+
+ | | site configuration | 3 | 1 |
+ | +---------------------------------------------------+----+----+
+ | | distributed configuration | 2 | 1 |
+ +---------------+---------------------------------------------------+----+----+
+ | multi-sources | capabilities | NA | 0 |
+ | +---------------------------------------------------+----+----+
+ | | configuration | 2 | 0 |
+ | +---------------------------------------------------+----+----+
+ | | ldap integration | 2 | 1 |
+ +---------------+---------------------------------------------------+----+----+
+ | usage | custom ReST markup | 2 | 0 |
+ | +---------------------------------------------------+----+----+
+ | | personal preferences | 2 | 1 |
+ +---------------+---------------------------------------------------+----+----+
Core development
================
-+====================================================================+====+====+
-| FEATURE | CM | DL |
-+====================================================================+====+====+
-| base - concepts | NA | 3 |
-| base - security model | NA | 2 |
-| base - database initialization | 2 | 1 |
-+--------------------------------------------------------------------+----+----+
-| rql - base | 2 | 2 |
-| rql - write | 2 | 2 |
-| rql - function | 2 | 0 |
-| rql - outer joins | 2 | 1 |
-| rql - aggregates | 2 | 1 |
-| rql - subqueries | 2 | 0 |
-+--------------------------------------------------------------------+----+----+
-| schema - base | 2 | 3 |
-| schema - constraints | 3 | 2 |
-| schema - security | 2 | 2 |
-| schema - inheritance | 1 | 1 |
-| schema - customization | 1 | 1 |
-| schema - introspection | 2 | 1 |
-+--------------------------------------------------------------------+----+----+
-| vregistry - appobject | 2 | 2 |
-| vregistry - registration | 2 | 2 |
-| vregistry - selection | 3 | 2 |
-| vregistry - core selectors | 3 | 3 |
-| vregistry - custom selectors | 2 | 1 |
-| vregistry - debugging selection | 2 | 1 |
-+--------------------------------------------------------------------+----+----+
-| entities - interfaces | 2 | ? |
-| entities - customization (`dc_`, ...) | 2 | ? |
-| entities - app logic | 2 | 2 |
-| entities - orm configuration | 2 | 1 |
-| entities - pluggable mixins | 1 | 0 |
-| entities - workflow | 3 | 2 |
-+--------------------------------------------------------------------+----+----+
-| dbapi - connection | 3 | 1 |
-| dbapi - data management | 1 | 1 |
-| dbapi - result set | 3 | 1 |
-| dbapi - transaction, undo | 2 | 0 |
-+--------------------------------------------------------------------+----+----+
-| cube - layout | 2 | 3 |
-| cube - new cube | 2 | 2 |
-+--------------------------------------------------------------------+----+----+
-| migration - context | 2 | 1 |
-| migration - commands | 2 | 2 |
-+--------------------------------------------------------------------+----+----+
-| testlib - CubicWebTC | 2 | 1 |
-| testlib - automatic tests | 2 | 2 |
-+--------------------------------------------------------------------+----+----+
-| i18n - mark string | 3 | 2 |
-| i18n - customize strings from other cubes / cubicweb | 3 | 1 |
-| i18n - update catalog | 3 | 2 |
-+--------------------------------------------------------------------+----+----+
-| more - reloading tips | NA | 0 |
-| more - site_cubicweb | 2 | ? |
-| more - adding options in configuration file | 3 | 0 |
-| more - adding options in site configuration / preferences | 3 | ? |
-| more - optimizing / profiling | 2 | 1 |
-| more - c-c plugins | 3 | 0 |
-| more - crypto services | 0 | 0 |
-| more - massive import | 2 | 0 |
-| more - mime type based conversion | 2 | 0 |
-| more - CWCache | 1 | 0 |
-+--------------------------------------------------------------------+----+----+
+.. table::
+
+ +--------------------------------------------------------------------+----+----+
+ | FEATURE | CM | DL |
+ +===========+========================================================+====+====+
+ | base | concepts | NA | 3 |
+ | +--------------------------------------------------------+----+----+
+ | | security model | NA | 2 |
+ | +--------------------------------------------------------+----+----+
+ | | database initialization | 2 | 1 |
+ +-----------+--------------------------------------------------------+----+----+
+ | rql | base | 2 | 2 |
+ | +--------------------------------------------------------+----+----+
+ | | write | 2 | 2 |
+ | +--------------------------------------------------------+----+----+
+ | | function | 2 | 0 |
+ | +--------------------------------------------------------+----+----+
+ | | outer joins | 2 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | aggregates | 2 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | subqueries | 2 | 0 |
+ +-----------+--------------------------------------------------------+----+----+
+ | schema | base | 2 | 3 |
+ | +--------------------------------------------------------+----+----+
+ | | constraints | 3 | 2 |
+ | +--------------------------------------------------------+----+----+
+ | | security | 2 | 2 |
+ | +--------------------------------------------------------+----+----+
+ | | inheritance | 1 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | customization | 1 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | introspection | 2 | 1 |
+ +-----------+--------------------------------------------------------+----+----+
+ | vregistry | appobject | 2 | 2 |
+ | +--------------------------------------------------------+----+----+
+ | | registration | 2 | 2 |
+ | +--------------------------------------------------------+----+----+
+ | | selection | 3 | 2 |
+ | +--------------------------------------------------------+----+----+
+ | | core selectors | 3 | 3 |
+ | +--------------------------------------------------------+----+----+
+ | | custom selectors | 2 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | debugging selection | 2 | 1 |
+ +-----------+--------------------------------------------------------+----+----+
+ | entities | interfaces | 2 | ? |
+ | +--------------------------------------------------------+----+----+
+ | | customization (`dc_`, ...) | 2 | ? |
+ | +--------------------------------------------------------+----+----+
+ | | app logic | 2 | 2 |
+ | +--------------------------------------------------------+----+----+
+ | | orm configuration | 2 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | pluggable mixins | 1 | 0 |
+ | +--------------------------------------------------------+----+----+
+ | | workflow | 3 | 2 |
+ +-----------+--------------------------------------------------------+----+----+
+ | dbapi | connection | 3 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | data management | 1 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | result set | 3 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | transaction, undo | 2 | 0 |
+ +-----------+--------------------------------------------------------+----+----+
+ | cube | layout | 2 | 3 |
+ | +--------------------------------------------------------+----+----+
+ | | new cube | 2 | 2 |
+ +-----------+--------------------------------------------------------+----+----+
+ | migration | context | 2 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | commands | 2 | 2 |
+ +-----------+--------------------------------------------------------+----+----+
+ | testlib | CubicWebTC | 2 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | automatic tests | 2 | 2 |
+ +-----------+--------------------------------------------------------+----+----+
+ | i18n | mark string | 3 | 2 |
+ | +--------------------------------------------------------+----+----+
+ | | customize strings from other cubes / cubicweb | 3 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | update catalog | 3 | 2 |
+ +-----------+--------------------------------------------------------+----+----+
+ | more | reloading tips | NA | 0 |
+ | +--------------------------------------------------------+----+----+
+ | | site_cubicweb | 2 | ? |
+ | +--------------------------------------------------------+----+----+
+ | | adding options in configuration file | 3 | 0 |
+ | +--------------------------------------------------------+----+----+
+ | | adding options in site configuration / preferences | 3 | ? |
+ | +--------------------------------------------------------+----+----+
+ | | optimizing / profiling | 2 | 1 |
+ | +--------------------------------------------------------+----+----+
+ | | c-c plugins | 3 | 0 |
+ | +--------------------------------------------------------+----+----+
+ | | crypto services | 0 | 0 |
+ | +--------------------------------------------------------+----+----+
+ | | massive import | 2 | 0 |
+ | +--------------------------------------------------------+----+----+
+ | | mime type based conversion | 2 | 0 |
+ | +--------------------------------------------------------+----+----+
+ | | CWCache | 1 | 0 |
+ +-----------+--------------------------------------------------------+----+----+
Web UI development
==================
-+====================================================================+====+====+
-| FEATURE | CM | DL |
-+====================================================================+====+====+
-| base - web request | 2 | 2 |
-| base - exceptions | 2 | 0 |
-| base - session, authentication | 1 | 0 |
-| base - http caching | 2 | 1 |
-| base - external resources | 2 | 2 |
-| base - static files | 2 | ? |
-| base - data sharing | 2 | 2 |
-| base - graphical chart customization | 1 | 1 |
-+--------------------------------------------------------------------+----+----+
-| publishing - cycle | 2 | 2 |
-| publishing - error handling | 2 | 1 |
-| publishing - transactions | NA | ? |
-+--------------------------------------------------------------------+----+----+
-| controller - base | 2 | 2 |
-| controller - view | 2 | 1 |
-| controller - edit | 2 | 1 |
-| controller - json | 2 | 1 |
-+--------------------------------------------------------------------+----+----+
-| views - base | 2 | 2 |
-| views - templates | 2 | 2 |
-| views - boxes | 2 | 1 |
-| views - components | 2 | 1 |
-| views - primary | 2 | 1 |
-| views - tabs | 2 | 1 |
-| views - xml | 2 | 0 |
-| views - text | 2 | 1 |
-| views - table | 2 | 1 |
-| views - plot | 2 | 0 |
-| views - navigation | 2 | 0 |
-| views - calendar, timeline | 2 | 0 |
-| views - index | 2 | 2 |
-| views - breadcrumbs | 2 | 1 |
-| views - actions | 2 | 1 |
-| views - debugging | 2 | 1 |
-+--------------------------------------------------------------------+----+----+
-| form - base | 2 | 1 |
-| form - fields | 2 | 1 |
-| form - widgets | 2 | 1 |
-| form - captcha | 2 | 0 |
-| form - renderers | 2 | 0 |
-| form - validation error handling | 2 | 0 |
-| form - autoform | 2 | 2 |
-| form - reledit | 2 | 0 |
-+--------------------------------------------------------------------+----+----+
-| facets - base | 2 | ? |
-| facets - configuration | 2 | 1 |
-| facets - custom facets | 2 | 0 |
-+--------------------------------------------------------------------+----+----+
-| css - base | 1 | 1 |
-| css - customization | 1 | 1 |
-+--------------------------------------------------------------------+----+----+
-| js - base | 1 | 1 |
-| js - jquery | 1 | 1 |
-| js - base functions | 1 | 0 |
-| js - ajax | 1 | 0 |
-| js - widgets | 1 | 1 |
-+--------------------------------------------------------------------+----+----+
-| other - page template | 0 | 0 |
-| other - inline doc (wdoc) | 2 | 0 |
-| other - magic search | 2 | 0 |
-| other - url mapping | 1 | 1 |
-| other - apache style url rewrite | 1 | 1 |
-| other - sparql | 1 | 0 |
-| other - bookmarks | 2 | 1 |
-+--------------------------------------------------------------------+----+----+
+.. table::
+
+ +--------------------------------------------------------------------+----+----+
+ | FEATURE | CM | DL |
+ +============+=======================================================+====+====+
+ | base | web request | 2 | 2 |
+ | +-------------------------------------------------------+----+----+
+ | | exceptions | 2 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | session, authentication | 1 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | http caching | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | external resources | 2 | 2 |
+ | +-------------------------------------------------------+----+----+
+ | | static files | 2 | ? |
+ | +-------------------------------------------------------+----+----+
+ | | data sharing | 2 | 2 |
+ | +-------------------------------------------------------+----+----+
+ | | graphical chart customization | 1 | 1 |
+ +------------+-------------------------------------------------------+----+----+
+ | publishing | cycle | 2 | 2 |
+ | +-------------------------------------------------------+----+----+
+ | | error handling | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | transactions | NA | ? |
+ +------------+-------------------------------------------------------+----+----+
+ | controller | base | 2 | 2 |
+ | +-------------------------------------------------------+----+----+
+ | | view | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | edit | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | json | 2 | 1 |
+ +------------+-------------------------------------------------------+----+----+
+ | views | base | 2 | 2 |
+ | +-------------------------------------------------------+----+----+
+ | | templates | 2 | 2 |
+ | +-------------------------------------------------------+----+----+
+ | | boxes | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | components | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | primary | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | tabs | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | xml | 2 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | text | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | table | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | plot | 2 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | navigation | 2 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | calendar, timeline | 2 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | index | 2 | 2 |
+ | +-------------------------------------------------------+----+----+
+ | | breadcrumbs | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | actions | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | debugging | 2 | 1 |
+ +------------+-------------------------------------------------------+----+----+
+ | form | base | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | fields | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | widgets | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | captcha | 2 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | renderers | 2 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | validation error handling | 2 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | autoform | 2 | 2 |
+ | +-------------------------------------------------------+----+----+
+ | | reledit | 2 | 0 |
+ +------------+-------------------------------------------------------+----+----+
+ | facets | base | 2 | ? |
+ | +-------------------------------------------------------+----+----+
+ | | configuration | 2 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | custom facets | 2 | 0 |
+ +------------+-------------------------------------------------------+----+----+
+ | css | base | 1 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | customization | 1 | 1 |
+ +------------+-------------------------------------------------------+----+----+
+ | js | base | 1 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | jquery | 1 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | base functions | 1 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | widgets | 1 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | ajax | 1 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | widgets | 1 | 1 |
+ +------------+-------------------------------------------------------+----+----+
+ | other | page template | 0 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | inline doc (wdoc) | 2 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | magic search | 2 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | url mapping | 1 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | apache style url rewrite | 1 | 1 |
+ | +-------------------------------------------------------+----+----+
+ | | sparql | 1 | 0 |
+ | +-------------------------------------------------------+----+----+
+ | | bookmarks | 2 | 1 |
+ +------------+-------------------------------------------------------+----+----+
Repository development
======================
-+====================================================================+====+====+
-| FEATURE | CM | DL |
-+====================================================================+====+====+
-| base - session | 2 | 2 |
-| base - more security control | 2 | 0 |
-| base - debugging | 2 | 0 |
-+--------------------------------------------------------------------+----+----+
-| hooks - development | 2 | 2 |
-| hooks - abstract hooks | 2 | 0 |
-| hooks - core hooks | 2 | 0 |
-| hooks - control | 2 | 0 |
-| hooks - operation | 2 | 2 |
-+--------------------------------------------------------------------+----+----+
-| notification - sending email | 2 | ? |
-| notification - base views | 1 | ? |
-| notification - supervisions | 1 | 0 |
-+--------------------------------------------------------------------+----+----+
-| source - storages | 2 | 0 |
-| source - authentication plugins | 2 | 0 |
-| source - custom sources | 2 | 0 |
-+--------------------------------------------------------------------+----+----+
+.. table::
+
+ +--------------------------------------------------------------------+----+----+
+ | FEATURE | CM | DL |
+ +==============+=====================================================+====+====+
+ | base | session | 2 | 2 |
+ | +-----------------------------------------------------+----+----+
+ | | more security control | 2 | 0 |
+ | +-----------------------------------------------------+----+----+
+ | | debugging | 2 | 0 |
+ +--------------+-----------------------------------------------------+----+----+
+ | hooks | development | 2 | 2 |
+ | +-----------------------------------------------------+----+----+
+ | | abstract hooks | 2 | 0 |
+ | +-----------------------------------------------------+----+----+
+ | | core hooks | 2 | 0 |
+ | +-----------------------------------------------------+----+----+
+ | | control | 2 | 0 |
+ | +-----------------------------------------------------+----+----+
+ | | operation | 2 | 2 |
+ +--------------+-----------------------------------------------------+----+----+
+ | notification | sending email | 2 | ? |
+ | +-----------------------------------------------------+----+----+
+ | | base views | 1 | ? |
+ | +-----------------------------------------------------+----+----+
+ | | supervisions | 1 | 0 |
+ +--------------+-----------------------------------------------------+----+----+
+ | source | storages | 2 | 0 |
+ | +-----------------------------------------------------+----+----+
+ | | authentication plugins | 2 | 0 |
+ | +-----------------------------------------------------+----+----+
+ | | custom sources | 2 | 0 |
+ +--------------+-----------------------------------------------------+----+----+
+
--- a/doc/tools/mode_plan.py Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/tools/mode_plan.py Wed Dec 16 11:23:48 2015 +0100
@@ -23,17 +23,19 @@
rename A010-joe.en.txt to A030-joe.en.txt
accept [y/N]?
"""
+from __future__ import print_function
+
def ren(a,b):
names = glob.glob('%s*'%a)
for name in names :
- print 'rename %s to %s' % (name, name.replace(a,b))
+ print('rename %s to %s' % (name, name.replace(a,b)))
if raw_input('accept [y/N]?').lower() =='y':
for name in names:
os.system('hg mv %s %s' % (name, name.replace(a,b)))
-def ls(): print '\n'.join(sorted(os.listdir('.')))
+def ls(): print('\n'.join(sorted(os.listdir('.'))))
def move():
filenames = []
@@ -47,4 +49,4 @@
for num, name in filenames:
if num >= start:
- print 'hg mv %s %2i%s' %(name,num+1,name[2:])
+ print('hg mv %s %2i%s' %(name,num+1,name[2:]))
--- a/doc/tutorials/base/customizing-the-application.rst Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/tutorials/base/customizing-the-application.rst Wed Dec 16 11:23:48 2015 +0100
@@ -422,7 +422,7 @@
entity = self.cw_rset.get_entity(row, col)
self.w(u'<h1>Welcome to the "%s" community</h1>' % entity.printable_value('name'))
if entity.display_cw_logo():
- self.w(u'<img src="http://www.cubicweb.org/doc/en/_static/cubicweb.png"/>')
+ self.w(u'<img src="https://docs.cubicweb.org/_static/logo-cubicweb-small.svg"/>')
if entity.description:
self.w(u'<p>%s</p>' % entity.printable_value('description'))
@@ -522,7 +522,7 @@
def render_entity_attributes(self, entity):
if entity.display_cw_logo():
- self.w(u'<img src="http://www.cubicweb.org/doc/en/_static/cubicweb.png"/>')
+ self.w(u'<img src="https://docs.cubicweb.org/_static/logo-cubicweb-small.svg"/>')
if entity.description:
self.w(u'<p>%s</p>' % entity.printable_value('description'))
--- a/doc/tutorials/base/discovering-the-ui.rst Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/tutorials/base/discovering-the-ui.rst Wed Dec 16 11:23:48 2015 +0100
@@ -101,16 +101,16 @@
You can achieve the same thing by following the same path as we did for the blog
creation, e.g. by clicking on the `[+]` at the left of the 'Blog entry' link on
-the index page. The diffidence being that since there is no context information,
+the index page. The difference being that since there is no context information,
the 'blog entry of' selector won't be preset to the blog.
If you click on the 'modify' link of the action box, you are back to
the form to edit the entity you just created, except that the form now
has another section with a combo-box entitled 'add relation'. It
-provisos a generic way to edit relations which don't appears in the
+provides a generic way to edit relations which don't appears in the
above form. Choose the relation you want to add and a second combo box
-appears where you can pick existing entities. If there are too many
+appears where you can pick existing entities. If there are too many
of them, you will be offered to navigate to the target entity, that is
go away from the form and go back to it later, once you've selected
the entity you want to link with.
--- a/doc/tutorials/dataimport/diseasome_import.py Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/tutorials/dataimport/diseasome_import.py Wed Dec 16 11:23:48 2015 +0100
@@ -95,7 +95,7 @@
# Perform a first commit, of the entities
store.flush()
kwargs = {}
- for uri, relations in all_relations.iteritems():
+ for uri, relations in all_relations.items():
from_eid = uri_to_eid.get(uri)
# ``subjtype`` should be initialized if ``SQLGenObjectStore`` is used
# and there are inlined relations in the schema.
@@ -108,7 +108,7 @@
kwargs['subjtype'] = uri_to_etype.get(uri)
if not from_eid:
continue
- for rtype, rels in relations.iteritems():
+ for rtype, rels in relations.items():
if rtype in ('classes', 'possible_drugs', 'omim', 'omim_page',
'chromosomal_location', 'same_as', 'gene_id',
'hgnc_id', 'hgnc_page'):
--- a/doc/tutorials/dataimport/diseasome_parser.py Tue Dec 15 19:05:41 2015 +0100
+++ b/doc/tutorials/dataimport/diseasome_parser.py Wed Dec 16 11:23:48 2015 +0100
@@ -97,4 +97,4 @@
entities[subj]['relations'].setdefault(MAPPING_RELS[rel], set())
entities[subj]['relations'][MAPPING_RELS[rel]].add(unicode(obj))
return ((ent.get('attributes'), ent.get('relations'))
- for ent in entities.itervalues())
+ for ent in entities.values())
--- a/entities/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/entities/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,8 +19,12 @@
__docformat__ = "restructuredtext en"
+from warnings import warn
+
+from six import text_type, string_types
from logilab.common.decorators import classproperty
+from logilab.common.deprecation import deprecated
from cubicweb import Unauthorized
from cubicweb.entity import Entity
@@ -35,7 +39,7 @@
@classproperty
def cw_etype(cls):
"""entity type as a unicode string"""
- return unicode(cls.__regid__)
+ return text_type(cls.__regid__)
@classmethod
def cw_create_url(cls, req, **kwargs):
@@ -43,6 +47,7 @@
return req.build_url('add/%s' % cls.__regid__, **kwargs)
@classmethod
+ @deprecated('[3.22] use cw_fti_index_rql_limit instead')
def cw_fti_index_rql_queries(cls, req):
"""return the list of rql queries to fetch entities to FT-index
@@ -60,6 +65,37 @@
return ['Any %s WHERE %s' % (', '.join(selected),
', '.join(restrictions))]
+ @classmethod
+ def cw_fti_index_rql_limit(cls, req, limit=1000):
+ """generate rsets of entities to FT-index
+
+ By default, each successive result set is limited to 1000 entities
+ """
+ if cls.cw_fti_index_rql_queries.__func__ != AnyEntity.cw_fti_index_rql_queries.__func__:
+ warn("[3.22] cw_fti_index_rql_queries is replaced by cw_fti_index_rql_limit",
+ DeprecationWarning)
+ for rql in cls.cw_fti_index_rql_queries(req):
+ yield req.execute(rql)
+ return
+ restrictions = ['X is %s' % cls.__regid__]
+ selected = ['X']
+ start = 0
+ for attrschema in sorted(cls.e_schema.indexable_attributes()):
+ varname = attrschema.type.upper()
+ restrictions.append('X %s %s' % (attrschema, varname))
+ selected.append(varname)
+ while True:
+ q_restrictions = restrictions + ['X eid > %s' % start]
+ rset = req.execute('Any %s ORDERBY X LIMIT %s WHERE %s' %
+ (', '.join(selected),
+ limit,
+ ', '.join(q_restrictions)))
+ if rset:
+ start = rset[-1][0]
+ yield rset
+ else:
+ break
+
# meta data api ###########################################################
def dc_title(self):
@@ -134,7 +170,7 @@
return self.dc_title().lower()
value = self.cw_attr_value(rtype)
# do not restrict to `unicode` because Bytes will return a `str` value
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
return self.printable_value(rtype, format='text/plain').lower()
return value
--- a/entities/adapters.py Tue Dec 15 19:05:41 2015 +0100
+++ b/entities/adapters.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,4 +1,4 @@
-# copyright 2010-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2010-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -18,19 +18,15 @@
"""some basic entity adapter implementations, for interfaces used in the
framework itself.
"""
-
-__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from itertools import chain
-from warnings import warn
from hashlib import md5
from logilab.mtconverter import TransformError
from logilab.common.decorators import cached
-from cubicweb import ValidationError, view, ViolatedConstraint
-from cubicweb.schema import CONSTRAINTS
+from cubicweb import ValidationError, view, ViolatedConstraint, UniqueTogetherError
from cubicweb.predicates import is_instance, relation_possible, match_exception
@@ -63,8 +59,8 @@
NOTE: the dictionary keys should match the list returned by the
`allowed_massmail_keys` method.
"""
- return dict( (attr, getattr(self.entity, attr))
- for attr in self.allowed_massmail_keys() )
+ return dict((attr, getattr(self.entity, attr))
+ for attr in self.allowed_massmail_keys())
class INotifiableAdapter(view.EntityAdapter):
@@ -156,40 +152,46 @@
if role == 'subject':
for entity_ in getattr(entity, rschema.type):
merge_weight_dict(words, entity_.cw_adapt_to('IFTIndexable').get_words())
- else: # if role == 'object':
+ else: # if role == 'object':
for entity_ in getattr(entity, 'reverse_%s' % rschema.type):
merge_weight_dict(words, entity_.cw_adapt_to('IFTIndexable').get_words())
return words
+
def merge_weight_dict(maindict, newdict):
- for weight, words in newdict.iteritems():
+ for weight, words in newdict.items():
maindict.setdefault(weight, []).extend(words)
+
class IDownloadableAdapter(view.EntityAdapter):
"""interface for downloadable entities"""
__regid__ = 'IDownloadable'
__abstract__ = True
- def download_url(self, **kwargs): # XXX not really part of this interface
- """return a URL to download entity's content"""
+ def download_url(self, **kwargs): # XXX not really part of this interface
+ """return a URL to download entity's content
+
+ It should be a unicode object containing url-encoded ASCII.
+ """
raise NotImplementedError
def download_content_type(self):
- """return MIME type of the downloadable content"""
+ """return MIME type (unicode) of the downloadable content"""
raise NotImplementedError
def download_encoding(self):
- """return encoding of the downloadable content"""
+ """return encoding (unicode) of the downloadable content"""
raise NotImplementedError
def download_file_name(self):
- """return file name of the downloadable content"""
+ """return file name (unicode) of the downloadable content"""
raise NotImplementedError
def download_data(self):
- """return actual data of the downloadable content"""
+ """return actual data (bytes) of the downloadable content"""
raise NotImplementedError
+
# XXX should propose to use two different relations for children/parent
class ITreeAdapter(view.EntityAdapter):
"""This adapter provides a tree interface.
@@ -337,7 +339,7 @@
try:
# check we are not jumping to another tree
if (adapter.tree_relation != self.tree_relation or
- adapter.child_role != self.child_role):
+ adapter.child_role != self.child_role):
break
entity = adapter.parent()
adapter = entity.cw_adapt_to('ITree')
@@ -347,9 +349,34 @@
return path
+class ISerializableAdapter(view.EntityAdapter):
+ """Adapter to serialize an entity to a bare python structure that may be
+ directly serialized to e.g. JSON.
+ """
+
+ __regid__ = 'ISerializable'
+ __select__ = is_instance('Any')
+
+ def serialize(self):
+ entity = self.entity
+ entity.complete()
+ data = {
+ 'cw_etype': entity.cw_etype,
+ 'cw_source': entity.cw_metainformation()['source']['uri'],
+ }
+ for rschema, __ in entity.e_schema.attribute_definitions():
+ attr = rschema.type
+ try:
+ value = entity.cw_attr_cache[attr]
+ except KeyError:
+ # Bytes
+ continue
+ data[attr] = value
+ return data
+
+
# error handling adapters ######################################################
-from cubicweb import UniqueTogetherError
class IUserFriendlyError(view.EntityAdapter):
__regid__ = 'IUserFriendlyError'
@@ -380,13 +407,14 @@
__select__ = match_exception(ViolatedConstraint)
def raise_user_exception(self):
- _ = self._cw._
cstrname = self.exc.cstrname
eschema = self.entity.e_schema
for rschema, attrschema in eschema.attribute_definitions():
rdef = rschema.rdef(eschema, attrschema)
for constraint in rdef.constraints:
- if cstrname == 'cstr' + md5(eschema.type + rschema.type + constraint.type() + (constraint.serialize() or '')).hexdigest():
+ if cstrname == 'cstr' + md5(
+ (eschema.type + rschema.type + constraint.type() +
+ (constraint.serialize() or '')).encode('ascii')).hexdigest():
break
else:
continue
--- a/entities/authobjs.py Tue Dec 15 19:05:41 2015 +0100
+++ b/entities/authobjs.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,6 +19,8 @@
__docformat__ = "restructuredtext en"
+from six import string_types
+
from logilab.common.decorators import cached
from cubicweb import Unauthorized
@@ -126,7 +128,7 @@
:type groups: str or iterable(str)
:param groups: a group name or an iterable on group names
"""
- if isinstance(groups, basestring):
+ if isinstance(groups, string_types):
groups = frozenset((groups,))
elif isinstance(groups, (tuple, list)):
groups = frozenset(groups)
--- a/entities/lib.py Tue Dec 15 19:05:41 2015 +0100
+++ b/entities/lib.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,9 +19,10 @@
__docformat__ = "restructuredtext en"
from warnings import warn
+from datetime import datetime
-from urlparse import urlsplit, urlunsplit
-from datetime import datetime
+from six.moves import range
+from six.moves.urllib.parse import urlsplit, urlunsplit
from logilab.mtconverter import xml_escape
@@ -67,7 +68,7 @@
{'y': self.eid})
if skipeids is None:
skipeids = set()
- for i in xrange(len(rset)):
+ for i in range(len(rset)):
eid = rset[i][0]
if eid in skipeids:
continue
@@ -146,4 +147,3 @@
if date:
return date > self.timestamp
return False
-
--- a/entities/sources.py Tue Dec 15 19:05:41 2015 +0100
+++ b/entities/sources.py Wed Dec 16 11:23:48 2015 +0100
@@ -42,7 +42,7 @@
cfg.update(config)
options = SOURCE_TYPES[self.type].options
sconfig = SourceConfiguration(self._cw.vreg.config, options=options)
- for opt, val in cfg.iteritems():
+ for opt, val in cfg.items():
try:
sconfig.set_option(opt, val)
except OptionError:
--- a/entities/test/unittest_base.py Tue Dec 15 19:05:41 2015 +0100
+++ b/entities/test/unittest_base.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -21,6 +21,7 @@
from logilab.common.testlib import unittest_main
from logilab.common.decorators import clear_cache
+from logilab.common.registry import yes
from cubicweb.devtools.testlib import CubicWebTC
@@ -45,13 +46,13 @@
self.assertEqual(entity.dc_creator(), u'member')
def test_type(self):
- #dc_type may be translated
+ # dc_type may be translated
with self.admin_access.client_cnx() as cnx:
member = cnx.entity_from_eid(self.membereid)
self.assertEqual(member.dc_type(), 'CWUser')
def test_cw_etype(self):
- #cw_etype is never translated
+ # cw_etype is never translated
with self.admin_access.client_cnx() as cnx:
member = cnx.entity_from_eid(self.membereid)
self.assertEqual(member.cw_etype, 'CWUser')
@@ -60,16 +61,37 @@
# XXX move to yams
self.assertEqual(self.schema['CWUser'].meta_attributes(), {})
self.assertEqual(dict((str(k), v)
- for k, v in self.schema['State'].meta_attributes().iteritems()),
- {'description_format': ('format', 'description')})
+ for k, v in self.schema['State'].meta_attributes().items()),
+ {'description_format': ('format', 'description')})
def test_fti_rql_method(self):
+ class EmailAddress(AnyEntity):
+ __regid__ = 'EmailAddress'
+ __select__ = AnyEntity.__select__ & yes(2)
+
+ @classmethod
+ def cw_fti_index_rql_queries(cls, req):
+ return ['EmailAddress Y']
+
with self.admin_access.web_request() as req:
+ req.create_entity('EmailAddress', address=u'foo@bar.com')
eclass = self.vreg['etypes'].etype_class('EmailAddress')
+ # deprecated
self.assertEqual(['Any X, ADDRESS, ALIAS WHERE X is EmailAddress, '
'X address ADDRESS, X alias ALIAS'],
eclass.cw_fti_index_rql_queries(req))
+ self.assertEqual(['Any X, ADDRESS, ALIAS ORDERBY X LIMIT 1000 WHERE X is EmailAddress, '
+ 'X address ADDRESS, X alias ALIAS, X eid > 0'],
+ [rset.rql for rset in eclass.cw_fti_index_rql_limit(req)])
+
+ # test backwards compatibility with custom method
+ with self.temporary_appobjects(EmailAddress):
+ self.vreg['etypes'].clear_caches()
+ eclass = self.vreg['etypes'].etype_class('EmailAddress')
+ self.assertEqual(['EmailAddress Y'],
+ [rset.rql for rset in eclass.cw_fti_index_rql_limit(req)])
+
class EmailAddressTC(BaseEntityTC):
@@ -87,14 +109,16 @@
self.assertEqual(email3.prefered.eid, email3.eid)
def test_mangling(self):
+ query = 'INSERT EmailAddress X: X address "maarten.ter.huurne@philips.com"'
with self.admin_access.repo_cnx() as cnx:
- email = cnx.execute('INSERT EmailAddress X: X address "maarten.ter.huurne@philips.com"').get_entity(0, 0)
+ email = cnx.execute(query).get_entity(0, 0)
self.assertEqual(email.display_address(), 'maarten.ter.huurne@philips.com')
self.assertEqual(email.printable_value('address'), 'maarten.ter.huurne@philips.com')
self.vreg.config.global_set_option('mangle-emails', True)
try:
self.assertEqual(email.display_address(), 'maarten.ter.huurne at philips dot com')
- self.assertEqual(email.printable_value('address'), 'maarten.ter.huurne at philips dot com')
+ self.assertEqual(email.printable_value('address'),
+ 'maarten.ter.huurne at philips dot com')
email = cnx.execute('INSERT EmailAddress X: X address "syt"').get_entity(0, 0)
self.assertEqual(email.display_address(), 'syt')
self.assertEqual(email.printable_value('address'), 'syt')
@@ -110,6 +134,7 @@
self.assertEqual(email.printable_value('address', format='text/plain'),
'maarten&ter@philips.com')
+
class CWUserTC(BaseEntityTC):
def test_complete(self):
@@ -147,10 +172,11 @@
with self.admin_access.repo_cnx() as cnx:
e = cnx.execute('CWUser U WHERE U login "member"').get_entity(0, 0)
# Bytes/Password attributes should be omitted
- self.assertEqual(e.cw_adapt_to('IEmailable').allowed_massmail_keys(),
- set(('surname', 'firstname', 'login', 'last_login_time',
- 'creation_date', 'modification_date', 'cwuri', 'eid'))
- )
+ self.assertEqual(
+ e.cw_adapt_to('IEmailable').allowed_massmail_keys(),
+ set(('surname', 'firstname', 'login', 'last_login_time',
+ 'creation_date', 'modification_date', 'cwuri', 'eid'))
+ )
def test_cw_instantiate_object_relation(self):
""" a weird non regression test """
@@ -193,7 +219,7 @@
# no specific class for Subdivisions, the default one should be selected
eclass = self.select_eclass('SubDivision')
self.assertTrue(eclass.__autogenerated__)
- #self.assertEqual(eclass.__bases__, (AnyEntity,))
+ # self.assertEqual(eclass.__bases__, (AnyEntity,))
# build class from most generic to most specific and make
# sure the most specific is always selected
self.vreg._loadedmods[__name__] = {}
@@ -212,5 +238,24 @@
eclass = self.select_eclass('Division')
self.assertEqual(eclass.cw_etype, 'Division')
+
+class ISerializableTC(CubicWebTC):
+
+ def test_serialization(self):
+ with self.admin_access.repo_cnx() as cnx:
+ entity = cnx.create_entity('CWGroup', name=u'tmp')
+ cnx.commit()
+ serializer = entity.cw_adapt_to('ISerializable')
+ expected = {
+ 'cw_etype': u'CWGroup',
+ 'cw_source': 'system',
+ 'cwuri': u'http://testing.fr/cubicweb/%s' % entity.eid,
+ 'creation_date': entity.creation_date,
+ 'modification_date': entity.modification_date,
+ 'name': u'tmp',
+ }
+ self.assertEqual(serializer.serialize(), expected)
+
+
if __name__ == '__main__':
unittest_main()
--- a/entities/test/unittest_wfobjs.py Tue Dec 15 19:05:41 2015 +0100
+++ b/entities/test/unittest_wfobjs.py Wed Dec 16 11:23:48 2015 +0100
@@ -107,7 +107,7 @@
def setup_database(self):
rschema = self.schema['in_state']
- for rdef in rschema.rdefs.itervalues():
+ for rdef in rschema.rdefs.values():
self.assertEqual(rdef.cardinality, '1*')
with self.admin_access.client_cnx() as cnx:
self.member_eid = self.create_user(cnx, 'member').eid
--- a/entities/wfobjs.py Tue Dec 15 19:05:41 2015 +0100
+++ b/entities/wfobjs.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,9 +21,11 @@
* workflow history (TrInfo)
* adapter for workflowable entities (IWorkflowableAdapter)
"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
+from six import text_type, string_types
from logilab.common.decorators import cached, clear_cache
from logilab.common.deprecation import deprecated
@@ -97,7 +99,7 @@
def transition_by_name(self, trname):
rset = self._cw.execute('Any T, TN WHERE T name TN, T name %(n)s, '
'T transition_of WF, WF eid %(wf)s',
- {'n': unicode(trname), 'wf': self.eid})
+ {'n': text_type(trname), 'wf': self.eid})
if rset:
return rset.get_entity(0, 0)
return None
@@ -114,7 +116,7 @@
def add_state(self, name, initial=False, **kwargs):
"""add a state to this workflow"""
- state = self._cw.create_entity('State', name=unicode(name), **kwargs)
+ state = self._cw.create_entity('State', name=text_type(name), **kwargs)
self._cw.execute('SET S state_of WF WHERE S eid %(s)s, WF eid %(wf)s',
{'s': state.eid, 'wf': self.eid})
if initial:
@@ -126,7 +128,7 @@
def _add_transition(self, trtype, name, fromstates,
requiredgroups=(), conditions=(), **kwargs):
- tr = self._cw.create_entity(trtype, name=unicode(name), **kwargs)
+ tr = self._cw.create_entity(trtype, name=text_type(name), **kwargs)
self._cw.execute('SET T transition_of WF '
'WHERE T eid %(t)s, WF eid %(wf)s',
{'t': tr.eid, 'wf': self.eid})
@@ -224,19 +226,19 @@
matches = user.matching_groups(groups)
if matches:
if DBG:
- print 'may_be_fired: %r may fire: user matches %s' % (self.name, groups)
+ print('may_be_fired: %r may fire: user matches %s' % (self.name, groups))
return matches
if 'owners' in groups and user.owns(eid):
if DBG:
- print 'may_be_fired: %r may fire: user is owner' % self.name
+ print('may_be_fired: %r may fire: user is owner' % self.name)
return True
# check one of the rql expression conditions matches if any
if self.condition:
if DBG:
- print ('my_be_fired: %r: %s' %
- (self.name, [(rqlexpr.expression,
+ print('my_be_fired: %r: %s' %
+ (self.name, [(rqlexpr.expression,
rqlexpr.check_expression(self._cw, eid))
- for rqlexpr in self.condition]))
+ for rqlexpr in self.condition]))
for rqlexpr in self.condition:
if rqlexpr.check_expression(self._cw, eid):
return True
@@ -256,13 +258,13 @@
for gname in requiredgroups:
rset = self._cw.execute('SET T require_group G '
'WHERE T eid %(x)s, G name %(gn)s',
- {'x': self.eid, 'gn': unicode(gname)})
+ {'x': self.eid, 'gn': text_type(gname)})
assert rset, '%s is not a known group' % gname
- if isinstance(conditions, basestring):
+ if isinstance(conditions, string_types):
conditions = (conditions,)
for expr in conditions:
- if isinstance(expr, basestring):
- kwargs = {'expr': unicode(expr)}
+ if isinstance(expr, string_types):
+ kwargs = {'expr': text_type(expr)}
else:
assert isinstance(expr, dict)
kwargs = expr
@@ -414,7 +416,7 @@
"""return the default workflow for entities of this type"""
# XXX CWEType method
wfrset = self._cw.execute('Any WF WHERE ET default_workflow WF, '
- 'ET name %(et)s', {'et': unicode(self.entity.cw_etype)})
+ 'ET name %(et)s', {'et': text_type(self.entity.cw_etype)})
if wfrset:
return wfrset.get_entity(0, 0)
self.warning("can't find any workflow for %s", self.entity.cw_etype)
@@ -479,7 +481,7 @@
'Any T,TT, TN WHERE S allowed_transition T, S eid %(x)s, '
'T type TT, T type %(type)s, '
'T name TN, T transition_of WF, WF eid %(wfeid)s',
- {'x': self.current_state.eid, 'type': unicode(type),
+ {'x': self.current_state.eid, 'type': text_type(type),
'wfeid': self.current_workflow.eid})
for tr in rset.entities():
if tr.may_be_fired(self.entity.eid):
@@ -528,7 +530,7 @@
def _get_transition(self, tr):
assert self.current_workflow
- if isinstance(tr, basestring):
+ if isinstance(tr, string_types):
_tr = self.current_workflow.transition_by_name(tr)
assert _tr is not None, 'not a %s transition: %s' % (
self.__regid__, tr)
@@ -549,7 +551,7 @@
tr = self._get_transition(tr)
if any(tr_ for tr_ in self.possible_transitions()
if tr_.eid == tr.eid):
- self.fire_transition(tr)
+ self.fire_transition(tr, comment, commentformat)
def change_state(self, statename, comment=None, commentformat=None, tr=None):
"""change the entity's state to the given state (name or entity) in
--- a/entity.py Tue Dec 15 19:05:41 2015 +0100
+++ b/entity.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,6 +22,9 @@
from warnings import warn
from functools import partial
+from six import text_type, string_types, integer_types
+from six.moves import range
+
from logilab.common.decorators import cached
from logilab.common.deprecation import deprecated
from logilab.common.registry import yes
@@ -57,7 +60,7 @@
"""return True if value can be used at the end of a Rest URL path"""
if value is None:
return False
- value = unicode(value)
+ value = text_type(value)
# the check for ?, /, & are to prevent problems when running
# behind Apache mod_proxy
if value == u'' or u'?' in value or u'/' in value or u'&' in value:
@@ -105,7 +108,7 @@
"""
st = cstr.snippet_rqlst.copy()
# replace relations in ST by eid infos from linkto where possible
- for (info_rtype, info_role), eids in lt_infos.iteritems():
+ for (info_rtype, info_role), eids in lt_infos.items():
eid = eids[0] # NOTE: we currently assume a pruned lt_info with only 1 eid
for rel in st.iget_nodes(RqlRelation):
targetvar = rel_matches(rel, info_rtype, info_role, evar.name)
@@ -132,7 +135,7 @@
def pruned_lt_info(eschema, lt_infos):
pruned = {}
- for (lt_rtype, lt_role), eids in lt_infos.iteritems():
+ for (lt_rtype, lt_role), eids in lt_infos.items():
# we can only use lt_infos describing relation with a cardinality
# of value 1 towards the linked entity
if not len(eids) == 1:
@@ -144,6 +147,7 @@
pruned[(lt_rtype, lt_role)] = eids
return pruned
+
class Entity(AppObject):
"""an entity instance has e_schema automagically set on
the class and instances has access to their issuing cursor.
@@ -279,7 +283,7 @@
select = Select()
mainvar = select.get_variable(mainvar)
select.add_selected(mainvar)
- elif isinstance(mainvar, basestring):
+ elif isinstance(mainvar, string_types):
assert mainvar in select.defined_vars
mainvar = select.get_variable(mainvar)
# eases string -> syntax tree test transition: please remove once stable
@@ -455,7 +459,7 @@
if len(value) == 0:
continue # avoid crash with empty IN clause
elif len(value) == 1:
- value = iter(value).next()
+ value = next(iter(value))
else:
# prepare IN clause
pendingrels.append( (attr, role, value) )
@@ -530,6 +534,7 @@
def __init__(self, req, rset=None, row=None, col=0):
AppObject.__init__(self, req, rset=rset, row=row, col=col)
self._cw_related_cache = {}
+ self._cw_adapters_cache = {}
if rset is not None:
self.eid = rset[row][col]
else:
@@ -545,12 +550,12 @@
raise NotImplementedError('comparison not implemented for %s' % self.__class__)
def __eq__(self, other):
- if isinstance(self.eid, (int, long)):
+ if isinstance(self.eid, integer_types):
return self.eid == other.eid
return self is other
def __hash__(self):
- if isinstance(self.eid, (int, long)):
+ if isinstance(self.eid, integer_types):
return self.eid
return super(Entity, self).__hash__()
@@ -567,10 +572,7 @@
return None if it can not be adapted.
"""
- try:
- cache = self._cw_adapters_cache
- except AttributeError:
- self._cw_adapters_cache = cache = {}
+ cache = self._cw_adapters_cache
try:
return cache[interface]
except KeyError:
@@ -677,8 +679,8 @@
if path is None:
# fallback url: <base-url>/<eid> url is used as cw entities uri,
# prefer it to <base-url>/<etype>/eid/<eid>
- return unicode(value)
- return '%s/%s' % (path, self._cw.url_quote(value))
+ return text_type(value)
+ return u'%s/%s' % (path, self._cw.url_quote(value))
def cw_attr_metadata(self, attr, metadata):
"""return a metadata for an attribute (None if unspecified)"""
@@ -695,7 +697,7 @@
attr = str(attr)
if value is _marker:
value = getattr(self, attr)
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
value = value.strip()
if value is None or value == '': # don't use "not", 0 is an acceptable value
return u''
@@ -849,7 +851,7 @@
if attributes is None:
self._cw_completed = True
varmaker = rqlvar_maker()
- V = varmaker.next()
+ V = next(varmaker)
rql = ['WHERE %s eid %%(x)s' % V]
selected = []
for attr in (attributes or self._cw_to_complete_attributes(skip_bytes, skip_pwd)):
@@ -857,7 +859,7 @@
if attr in self.cw_attr_cache:
continue
# case where attribute must be completed, but is not yet in entity
- var = varmaker.next()
+ var = next(varmaker)
rql.append('%s %s %s' % (V, attr, var))
selected.append((attr, var))
# +1 since this doesn't include the main variable
@@ -876,7 +878,7 @@
# * user has read perm on the relation and on the target entity
assert rschema.inlined
assert role == 'subject'
- var = varmaker.next()
+ var = next(varmaker)
# keep outer join anyway, we don't want .complete to crash on
# missing mandatory relation (see #1058267)
rql.append('%s %s %s?' % (V, rtype, var))
@@ -892,10 +894,10 @@
raise Exception('unable to fetch attributes for entity with eid %s'
% self.eid)
# handle attributes
- for i in xrange(1, lastattr):
+ for i in range(1, lastattr):
self.cw_attr_cache[str(selected[i-1][0])] = rset[i]
# handle relations
- for i in xrange(lastattr, len(rset)):
+ for i in range(lastattr, len(rset)):
rtype, role = selected[i-1][0]
value = rset[i]
if value is None:
@@ -1145,9 +1147,7 @@
self._cw.vreg.solutions(self._cw, select, args)
# insert RQL expressions for schema constraints into the rql syntax tree
if vocabconstraints:
- # RQLConstraint is a subclass for RQLVocabularyConstraint, so they
- # will be included as well
- cstrcls = RQLVocabularyConstraint
+ cstrcls = (RQLVocabularyConstraint, RQLConstraint)
else:
cstrcls = RQLConstraint
lt_infos = pruned_lt_info(self.e_schema, lt_infos or {})
@@ -1236,8 +1236,8 @@
no relation is given
"""
if rtype is None:
- self._cw_related_cache = {}
- self._cw_adapters_cache = {}
+ self._cw_related_cache.clear()
+ self._cw_adapters_cache.clear()
else:
assert role
self._cw_related_cache.pop('%s_%s' % (rtype, role), None)
--- a/etwist/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/etwist/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,4 +18,3 @@
""" CW - nevow/twisted client
"""
-
--- a/etwist/request.py Tue Dec 15 19:05:41 2015 +0100
+++ b/etwist/request.py Wed Dec 16 11:23:48 2015 +0100
@@ -31,7 +31,7 @@
self._twreq = req
super(CubicWebTwistedRequestAdapter, self).__init__(
vreg, https, req.args, headers=req.received_headers)
- for key, name_stream_list in req.files.iteritems():
+ for key, name_stream_list in req.files.items():
for name, stream in name_stream_list:
if name is not None:
name = unicode(name, self.encoding)
--- a/etwist/server.py Tue Dec 15 19:05:41 2015 +0100
+++ b/etwist/server.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,8 +22,10 @@
import select
import traceback
import threading
-from urlparse import urlsplit, urlunsplit
from cgi import FieldStorage, parse_header
+
+from six.moves.urllib.parse import urlsplit, urlunsplit
+
from cubicweb.statsd_logger import statsd_timeit
from twisted.internet import reactor, task, threads
--- a/etwist/service.py Tue Dec 15 19:05:41 2015 +0100
+++ b/etwist/service.py Wed Dec 16 11:23:48 2015 +0100
@@ -15,6 +15,8 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+from __future__ import print_function
+
import os
import sys
@@ -22,7 +24,7 @@
import win32serviceutil
import win32service
except ImportError:
- print 'Win32 extensions for Python are likely not installed.'
+ print('Win32 extensions for Python are likely not installed.')
sys.exit(3)
from os.path import join
--- a/etwist/twctl.py Tue Dec 15 19:05:41 2015 +0100
+++ b/etwist/twctl.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,10 +17,6 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""cubicweb-clt handlers for twisted"""
-from os.path import join
-
-from logilab.common.shellutils import rm
-
from cubicweb.toolsutils import CommandHandler
from cubicweb.web.webctl import WebCreateHandler, WebUpgradeHandler
@@ -36,9 +32,6 @@
def start_server(self, config):
from cubicweb.etwist import server
- config.info('clear ui caches')
- for cachedir in ('uicache', 'uicachehttps'):
- rm(join(config.appdatahome, cachedir, '*'))
return server.run(config)
class TWStopHandler(CommandHandler):
--- a/ext/rest.py Tue Dec 15 19:05:41 2015 +0100
+++ b/ext/rest.py Wed Dec 16 11:23:48 2015 +0100
@@ -38,7 +38,9 @@
from itertools import chain
from logging import getLogger
from os.path import join
-from urlparse import urlsplit
+
+from six import text_type
+from six.moves.urllib.parse import urlsplit
from docutils import statemachine, nodes, utils, io
from docutils.core import Publisher
@@ -172,7 +174,7 @@
rql = params['rql']
if vid is None:
vid = params.get('vid')
- except (ValueError, KeyError), exc:
+ except (ValueError, KeyError) as exc:
msg = inliner.reporter.error('Could not parse bookmark path %s [%s].'
% (bookmark.path, exc), line=lineno)
prb = inliner.problematic(rawtext, rawtext, msg)
@@ -186,7 +188,7 @@
vid = 'noresult'
view = _cw.vreg['views'].select(vid, _cw, rset=rset)
content = view.render()
- except Exception, exc:
+ except Exception as exc:
content = 'An error occurred while interpreting directive bookmark: %r' % exc
set_classes(options)
return [nodes.raw('', content, format='html')], []
@@ -401,7 +403,7 @@
the data formatted as HTML or the original data if an error occurred
"""
req = context._cw
- if isinstance(data, unicode):
+ if isinstance(data, text_type):
encoding = 'unicode'
# remove unprintable characters unauthorized in xml
data = data.translate(ESC_UCAR_TABLE)
@@ -446,8 +448,8 @@
return res
except BaseException:
LOGGER.exception('error while publishing ReST text')
- if not isinstance(data, unicode):
- data = unicode(data, encoding, 'replace')
+ if not isinstance(data, text_type):
+ data = text_type(data, encoding, 'replace')
return xml_escape(req._('error while publishing ReST text')
+ '\n\n' + data)
--- a/ext/tal.py Tue Dec 15 19:05:41 2015 +0100
+++ b/ext/tal.py Wed Dec 16 11:23:48 2015 +0100
@@ -184,7 +184,10 @@
interpreter.execute(self)
except UnicodeError as unierror:
LOGGER.exception(str(unierror))
- raise simpleTALES.ContextContentException("found non-unicode %r string in Context!" % unierror.args[1]), None, sys.exc_info()[-1]
+ exc = simpleTALES.ContextContentException(
+ "found non-unicode %r string in Context!" % unierror.args[1])
+ exc.__traceback__ = sys.exc_info()[-1]
+ raise exc
def compile_template(template):
@@ -203,7 +206,7 @@
:type filepath: str
:param template: path of the file to compile
"""
- fp = file(filepath)
+ fp = open(filepath)
file_content = unicode(fp.read()) # template file should be pure ASCII
fp.close()
return compile_template(file_content)
@@ -232,7 +235,8 @@
result = eval(expr, globals, locals)
except Exception as ex:
ex = ex.__class__('in %r: %s' % (expr, ex))
- raise ex, None, sys.exc_info()[-1]
+ ex.__traceback__ = sys.exc_info()[-1]
+ raise ex
if (isinstance (result, simpleTALES.ContextVariable)):
return result.value()
return result
--- a/ext/test/data/views.py Tue Dec 15 19:05:41 2015 +0100
+++ b/ext/test/data/views.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,4 +21,3 @@
class CustomRsetTableView(tableview.RsetTableView):
__regid__ = 'mytable'
-
--- a/ext/test/unittest_rest.py Tue Dec 15 19:05:41 2015 +0100
+++ b/ext/test/unittest_rest.py Wed Dec 16 11:23:48 2015 +0100
@@ -15,6 +15,8 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+from six import PY3
+
from logilab.common.testlib import unittest_main
from cubicweb.devtools.testlib import CubicWebTC
@@ -87,7 +89,9 @@
context = self.context(req)
out = rest_publish(context, ':rql:`Any X WHERE X is CWUser:toto`')
self.assertTrue(out.startswith("<p>an error occurred while interpreting this "
- "rql directive: ObjectNotFound(u'toto',)</p>"))
+ "rql directive: ObjectNotFound(%s'toto',)</p>" %
+ ('' if PY3 else 'u')),
+ out)
def test_rql_role_without_vid(self):
with self.admin_access.web_request() as req:
@@ -229,7 +233,7 @@
%(rql)s
""" % {'rql': rql,
'colvids': ', '.join(["%d=%s" % (k, v)
- for k, v in colvids.iteritems()])
+ for k, v in colvids.items()])
})
view = self.vreg['views'].select('table', req, rset=req.execute(rql))
view.cellvids = colvids
--- a/gettext.py Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,795 +0,0 @@
-# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
-# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
-#
-# This file is part of CubicWeb.
-#
-# CubicWeb is free software: you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License as published by the Free
-# Software Foundation, either version 2.1 of the License, or (at your option)
-# any later version.
-#
-# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
-# details.
-#
-# You should have received a copy of the GNU Lesser General Public License along
-# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-"""Internationalization and localization support.
-
-This module provides internationalization (I18N) and localization (L10N)
-support for your Python programs by providing an interface to the GNU gettext
-message catalog library.
-
-I18N refers to the operation by which a program is made aware of multiple
-languages. L10N refers to the adaptation of your program, once
-internationalized, to the local language and cultural habits.
-
-"""
-
-# This module represents the integration of work, contributions, feedback, and
-# suggestions from the following people:
-#
-# Martin von Loewis, who wrote the initial implementation of the underlying
-# C-based libintlmodule (later renamed _gettext), along with a skeletal
-# gettext.py implementation.
-#
-# Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
-# which also included a pure-Python implementation to read .mo files if
-# intlmodule wasn't available.
-#
-# James Henstridge, who also wrote a gettext.py module, which has some
-# interesting, but currently unsupported experimental features: the notion of
-# a Catalog class and instances, and the ability to add to a catalog file via
-# a Python API.
-#
-# Barry Warsaw integrated these modules, wrote the .install() API and code,
-# and conformed all C and Python code to Python's coding standards.
-#
-# Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
-# module.
-#
-# J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.
-#
-# TODO:
-# - Lazy loading of .mo files. Currently the entire catalog is loaded into
-# memory, but that's probably bad for large translated programs. Instead,
-# the lexical sort of original strings in GNU .mo files should be exploited
-# to do binary searches and lazy initializations. Or you might want to use
-# the undocumented double-hash algorithm for .mo files with hash tables, but
-# you'll need to study the GNU gettext code to do this.
-#
-# - Support Solaris .mo file formats. Unfortunately, we've been unable to
-# find this format documented anywhere.
-
-
-import locale, copy, os, re, struct, sys
-from errno import ENOENT
-
-
-__all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
- 'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
- 'dgettext', 'dngettext', 'gettext', 'ngettext',
- ]
-
-_default_localedir = os.path.join(sys.prefix, 'share', 'locale')
-
-
-def test(condition, true, false):
- """
- Implements the C expression:
-
- condition ? true : false
-
- Required to correctly interpret plural forms.
- """
- if condition:
- return true
- else:
- return false
-
-
-def c2py(plural):
- """Gets a C expression as used in PO files for plural forms and returns a
- Python lambda function that implements an equivalent expression.
- """
- # Security check, allow only the "n" identifier
- try:
- from cStringIO import StringIO
- except ImportError:
- from StringIO import StringIO
- import token, tokenize
- tokens = tokenize.generate_tokens(StringIO(plural).readline)
- try:
- danger = [x for x in tokens if x[0] == token.NAME and x[1] != 'n']
- except tokenize.TokenError:
- raise ValueError, \
- 'plural forms expression error, maybe unbalanced parenthesis'
- else:
- if danger:
- raise ValueError, 'plural forms expression could be dangerous'
-
- # Replace some C operators by their Python equivalents
- plural = plural.replace('&&', ' and ')
- plural = plural.replace('||', ' or ')
-
- expr = re.compile(r'\!([^=])')
- plural = expr.sub(' not \\1', plural)
-
- # Regular expression and replacement function used to transform
- # "a?b:c" to "test(a,b,c)".
- expr = re.compile(r'(.*?)\?(.*?):(.*)')
- def repl(x):
- return "test(%s, %s, %s)" % (x.group(1), x.group(2),
- expr.sub(repl, x.group(3)))
-
- # Code to transform the plural expression, taking care of parentheses
- stack = ['']
- for c in plural:
- if c == '(':
- stack.append('')
- elif c == ')':
- if len(stack) == 1:
- # Actually, we never reach this code, because unbalanced
- # parentheses get caught in the security check at the
- # beginning.
- raise ValueError, 'unbalanced parenthesis in plural form'
- s = expr.sub(repl, stack.pop())
- stack[-1] += '(%s)' % s
- else:
- stack[-1] += c
- plural = expr.sub(repl, stack.pop())
-
- return eval('lambda n: int(%s)' % plural)
-
-
-
-def _expand_lang(locale):
- from locale import normalize
- locale = normalize(locale)
- COMPONENT_CODESET = 1 << 0
- COMPONENT_TERRITORY = 1 << 1
- COMPONENT_MODIFIER = 1 << 2
- # split up the locale into its base components
- mask = 0
- pos = locale.find('@')
- if pos >= 0:
- modifier = locale[pos:]
- locale = locale[:pos]
- mask |= COMPONENT_MODIFIER
- else:
- modifier = ''
- pos = locale.find('.')
- if pos >= 0:
- codeset = locale[pos:]
- locale = locale[:pos]
- mask |= COMPONENT_CODESET
- else:
- codeset = ''
- pos = locale.find('_')
- if pos >= 0:
- territory = locale[pos:]
- locale = locale[:pos]
- mask |= COMPONENT_TERRITORY
- else:
- territory = ''
- language = locale
- ret = []
- for i in range(mask+1):
- if not (i & ~mask): # if all components for this combo exist ...
- val = language
- if i & COMPONENT_TERRITORY: val += territory
- if i & COMPONENT_CODESET: val += codeset
- if i & COMPONENT_MODIFIER: val += modifier
- ret.append(val)
- ret.reverse()
- return ret
-
-
-
-class NullTranslations:
- def __init__(self, fp=None):
- self._info = {}
- self._charset = None
- self._output_charset = None
- self._fallback = None
- if fp is not None:
- self._parse(fp)
-
- def _parse(self, fp):
- pass
-
- def add_fallback(self, fallback):
- if self._fallback:
- self._fallback.add_fallback(fallback)
- else:
- self._fallback = fallback
-
- def gettext(self, message):
- if self._fallback:
- return self._fallback.gettext(message)
- return message
-
- def pgettext(self, context, message):
- if self._fallback:
- return self._fallback.pgettext(context, message)
- return message
-
- def lgettext(self, message):
- if self._fallback:
- return self._fallback.lgettext(message)
- return message
-
- def lpgettext(self, context, message):
- if self._fallback:
- return self._fallback.lpgettext(context, message)
- return message
-
- def ngettext(self, msgid1, msgid2, n):
- if self._fallback:
- return self._fallback.ngettext(msgid1, msgid2, n)
- if n == 1:
- return msgid1
- else:
- return msgid2
-
- def npgettext(self, context, msgid1, msgid2, n):
- if self._fallback:
- return self._fallback.npgettext(context, msgid1, msgid2, n)
- if n == 1:
- return msgid1
- else:
- return msgid2
-
- def lngettext(self, msgid1, msgid2, n):
- if self._fallback:
- return self._fallback.lngettext(msgid1, msgid2, n)
- if n == 1:
- return msgid1
- else:
- return msgid2
-
- def lnpgettext(self, context, msgid1, msgid2, n):
- if self._fallback:
- return self._fallback.lnpgettext(context, msgid1, msgid2, n)
- if n == 1:
- return msgid1
- else:
- return msgid2
-
- def ugettext(self, message):
- if self._fallback:
- return self._fallback.ugettext(message)
- return unicode(message)
-
- def upgettext(self, context, message):
- if self._fallback:
- return self._fallback.upgettext(context, message)
- return unicode(message)
-
- def ungettext(self, msgid1, msgid2, n):
- if self._fallback:
- return self._fallback.ungettext(msgid1, msgid2, n)
- if n == 1:
- return unicode(msgid1)
- else:
- return unicode(msgid2)
-
- def unpgettext(self, context, msgid1, msgid2, n):
- if self._fallback:
- return self._fallback.unpgettext(context, msgid1, msgid2, n)
- if n == 1:
- return unicode(msgid1)
- else:
- return unicode(msgid2)
-
- def info(self):
- return self._info
-
- def charset(self):
- return self._charset
-
- def output_charset(self):
- return self._output_charset
-
- def set_output_charset(self, charset):
- self._output_charset = charset
-
- def install(self, unicode=False, names=None):
- import __builtin__
- __builtin__.__dict__['_'] = unicode and self.ugettext or self.gettext
- if hasattr(names, "__contains__"):
- if "gettext" in names:
- __builtin__.__dict__['gettext'] = __builtin__.__dict__['_']
- if "pgettext" in names:
- __builtin__.__dict__['pgettext'] = (unicode and self.upgettext
- or self.pgettext)
- if "ngettext" in names:
- __builtin__.__dict__['ngettext'] = (unicode and self.ungettext
- or self.ngettext)
- if "npgettext" in names:
- __builtin__.__dict__['npgettext'] = \
- (unicode and self.unpgettext or self.npgettext)
- if "lgettext" in names:
- __builtin__.__dict__['lgettext'] = self.lgettext
- if "lpgettext" in names:
- __builtin__.__dict__['lpgettext'] = self.lpgettext
- if "lngettext" in names:
- __builtin__.__dict__['lngettext'] = self.lngettext
- if "lnpgettext" in names:
- __builtin__.__dict__['lnpgettext'] = self.lnpgettext
-
-
-class GNUTranslations(NullTranslations):
- # Magic number of .mo files
- LE_MAGIC = 0x950412deL
- BE_MAGIC = 0xde120495L
-
- # The encoding of a msgctxt and a msgid in a .mo file is
- # msgctxt + "\x04" + msgid (gettext version >= 0.15)
- CONTEXT_ENCODING = "%s\x04%s"
-
- def _parse(self, fp):
- """Override this method to support alternative .mo formats."""
- unpack = struct.unpack
- filename = getattr(fp, 'name', '')
- # Parse the .mo file header, which consists of 5 little endian 32
- # bit words.
- self._catalog = catalog = {}
- self.plural = lambda n: int(n != 1) # germanic plural by default
- buf = fp.read()
- buflen = len(buf)
- # Are we big endian or little endian?
- magic = unpack('<I', buf[:4])[0]
- if magic == self.LE_MAGIC:
- version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20])
- ii = '<II'
- elif magic == self.BE_MAGIC:
- version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20])
- ii = '>II'
- else:
- raise IOError(0, 'Bad magic number', filename)
- # Now put all messages from the .mo file buffer into the catalog
- # dictionary.
- for i in xrange(0, msgcount):
- mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
- mend = moff + mlen
- tlen, toff = unpack(ii, buf[transidx:transidx+8])
- tend = toff + tlen
- if mend < buflen and tend < buflen:
- msg = buf[moff:mend]
- tmsg = buf[toff:tend]
- else:
- raise IOError(0, 'File is corrupt', filename)
- # See if we're looking at GNU .mo conventions for metadata
- if mlen == 0:
- # Catalog description
- lastk = k = None
- for item in tmsg.splitlines():
- item = item.strip()
- if not item:
- continue
- if ':' in item:
- k, v = item.split(':', 1)
- k = k.strip().lower()
- v = v.strip()
- self._info[k] = v
- lastk = k
- elif lastk:
- self._info[lastk] += '\n' + item
- if k == 'content-type':
- self._charset = v.split('charset=')[1]
- elif k == 'plural-forms':
- v = v.split(';')
- plural = v[1].split('plural=')[1]
- self.plural = c2py(plural)
- # Note: we unconditionally convert both msgids and msgstrs to
- # Unicode using the character encoding specified in the charset
- # parameter of the Content-Type header. The gettext documentation
- # strongly encourages msgids to be us-ascii, but some appliations
- # require alternative encodings (e.g. Zope's ZCML and ZPT). For
- # traditional gettext applications, the msgid conversion will
- # cause no problems since us-ascii should always be a subset of
- # the charset encoding. We may want to fall back to 8-bit msgids
- # if the Unicode conversion fails.
- if '\x00' in msg:
- # Plural forms
- msgid1, msgid2 = msg.split('\x00')
- tmsg = tmsg.split('\x00')
- if self._charset:
- msgid1 = unicode(msgid1, self._charset)
- tmsg = [unicode(x, self._charset) for x in tmsg]
- for i in range(len(tmsg)):
- catalog[(msgid1, i)] = tmsg[i]
- else:
- if self._charset:
- msg = unicode(msg, self._charset)
- tmsg = unicode(tmsg, self._charset)
- catalog[msg] = tmsg
- # advance to next entry in the seek tables
- masteridx += 8
- transidx += 8
-
- def gettext(self, message):
- missing = object()
- tmsg = self._catalog.get(message, missing)
- if tmsg is missing:
- if self._fallback:
- return self._fallback.gettext(message)
- return message
- # Encode the Unicode tmsg back to an 8-bit string, if possible
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- elif self._charset:
- return tmsg.encode(self._charset)
- return tmsg
-
- def pgettext(self, context, message):
- ctxt_msg_id = self.CONTEXT_ENCODING % (context, message)
- missing = object()
- tmsg = self._catalog.get(ctxt_msg_id, missing)
- if tmsg is missing:
- if self._fallback:
- return self._fallback.pgettext(context, message)
- return message
- # Encode the Unicode tmsg back to an 8-bit string, if possible
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- elif self._charset:
- return tmsg.encode(self._charset)
- return tmsg
-
- def lgettext(self, message):
- missing = object()
- tmsg = self._catalog.get(message, missing)
- if tmsg is missing:
- if self._fallback:
- return self._fallback.lgettext(message)
- return message
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- return tmsg.encode(locale.getpreferredencoding())
-
- def lpgettext(self, context, message):
- ctxt_msg_id = self.CONTEXT_ENCODING % (context, message)
- missing = object()
- tmsg = self._catalog.get(ctxt_msg_id, missing)
- if tmsg is missing:
- if self._fallback:
- return self._fallback.lpgettext(context, message)
- return message
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- return tmsg.encode(locale.getpreferredencoding())
-
- def ngettext(self, msgid1, msgid2, n):
- try:
- tmsg = self._catalog[(msgid1, self.plural(n))]
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- elif self._charset:
- return tmsg.encode(self._charset)
- return tmsg
- except KeyError:
- if self._fallback:
- return self._fallback.ngettext(msgid1, msgid2, n)
- if n == 1:
- return msgid1
- else:
- return msgid2
-
- def npgettext(self, context, msgid1, msgid2, n):
- ctxt_msg_id = self.CONTEXT_ENCODING % (context, msgid1)
- try:
- tmsg = self._catalog[(ctxt_msg_id, self.plural(n))]
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- elif self._charset:
- return tmsg.encode(self._charset)
- return tmsg
- except KeyError:
- if self._fallback:
- return self._fallback.npgettext(context, msgid1, msgid2, n)
- if n == 1:
- return msgid1
- else:
- return msgid2
-
- def lngettext(self, msgid1, msgid2, n):
- try:
- tmsg = self._catalog[(msgid1, self.plural(n))]
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- return tmsg.encode(locale.getpreferredencoding())
- except KeyError:
- if self._fallback:
- return self._fallback.lngettext(msgid1, msgid2, n)
- if n == 1:
- return msgid1
- else:
- return msgid2
-
- def lnpgettext(self, context, msgid1, msgid2, n):
- ctxt_msg_id = self.CONTEXT_ENCODING % (context, msgid1)
- try:
- tmsg = self._catalog[(ctxt_msg_id, self.plural(n))]
- if self._output_charset:
- return tmsg.encode(self._output_charset)
- return tmsg.encode(locale.getpreferredencoding())
- except KeyError:
- if self._fallback:
- return self._fallback.lnpgettext(context, msgid1, msgid2, n)
- if n == 1:
- return msgid1
- else:
- return msgid2
-
- def ugettext(self, message):
- missing = object()
- tmsg = self._catalog.get(message, missing)
- if tmsg is missing:
- if self._fallback:
- return self._fallback.ugettext(message)
- return unicode(message)
- return tmsg
-
- def upgettext(self, context, message):
- ctxt_message_id = self.CONTEXT_ENCODING % (context, message)
- missing = object()
- tmsg = self._catalog.get(ctxt_message_id, missing)
- if tmsg is missing:
- # XXX logilab patch for compat w/ catalog generated by cw < 3.5
- return self.ugettext(message)
- if self._fallback:
- return self._fallback.upgettext(context, message)
- return unicode(message)
- return tmsg
-
- def ungettext(self, msgid1, msgid2, n):
- try:
- tmsg = self._catalog[(msgid1, self.plural(n))]
- except KeyError:
- if self._fallback:
- return self._fallback.ungettext(msgid1, msgid2, n)
- if n == 1:
- tmsg = unicode(msgid1)
- else:
- tmsg = unicode(msgid2)
- return tmsg
-
- def unpgettext(self, context, msgid1, msgid2, n):
- ctxt_message_id = self.CONTEXT_ENCODING % (context, msgid1)
- try:
- tmsg = self._catalog[(ctxt_message_id, self.plural(n))]
- except KeyError:
- if self._fallback:
- return self._fallback.unpgettext(context, msgid1, msgid2, n)
- if n == 1:
- tmsg = unicode(msgid1)
- else:
- tmsg = unicode(msgid2)
- return tmsg
-
-
-# Locate a .mo file using the gettext strategy
-def find(domain, localedir=None, languages=None, all=0):
- # Get some reasonable defaults for arguments that were not supplied
- if localedir is None:
- localedir = _default_localedir
- if languages is None:
- languages = []
- for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
- val = os.environ.get(envar)
- if val:
- languages = val.split(':')
- break
- if 'C' not in languages:
- languages.append('C')
- # now normalize and expand the languages
- nelangs = []
- for lang in languages:
- for nelang in _expand_lang(lang):
- if nelang not in nelangs:
- nelangs.append(nelang)
- # select a language
- if all:
- result = []
- else:
- result = None
- for lang in nelangs:
- if lang == 'C':
- break
- mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
- if os.path.exists(mofile):
- if all:
- result.append(mofile)
- else:
- return mofile
- return result
-
-
-
-# a mapping between absolute .mo file path and Translation object
-_translations = {}
-
-def translation(domain, localedir=None, languages=None,
- class_=None, fallback=False, codeset=None):
- if class_ is None:
- class_ = GNUTranslations
- mofiles = find(domain, localedir, languages, all=1)
- if not mofiles:
- if fallback:
- return NullTranslations()
- raise IOError(ENOENT, 'No translation file found for domain', domain)
- # TBD: do we need to worry about the file pointer getting collected?
- # Avoid opening, reading, and parsing the .mo file after it's been done
- # once.
- result = None
- for mofile in mofiles:
- key = os.path.abspath(mofile)
- t = _translations.get(key)
- if t is None:
- t = _translations.setdefault(key, class_(open(mofile, 'rb')))
- # Copy the translation object to allow setting fallbacks and
- # output charset. All other instance data is shared with the
- # cached object.
- t = copy.copy(t)
- if codeset:
- t.set_output_charset(codeset)
- if result is None:
- result = t
- else:
- result.add_fallback(t)
- return result
-
-
-def install(domain, localedir=None, unicode=False, codeset=None, names=None):
- t = translation(domain, localedir, fallback=True, codeset=codeset)
- t.install(unicode, names)
-
-
-
-# a mapping b/w domains and locale directories
-_localedirs = {}
-# a mapping b/w domains and codesets
-_localecodesets = {}
-# current global domain, `messages' used for compatibility w/ GNU gettext
-_current_domain = 'messages'
-
-
-def textdomain(domain=None):
- global _current_domain
- if domain is not None:
- _current_domain = domain
- return _current_domain
-
-
-def bindtextdomain(domain, localedir=None):
- global _localedirs
- if localedir is not None:
- _localedirs[domain] = localedir
- return _localedirs.get(domain, _default_localedir)
-
-
-def bind_textdomain_codeset(domain, codeset=None):
- global _localecodesets
- if codeset is not None:
- _localecodesets[domain] = codeset
- return _localecodesets.get(domain)
-
-
-def dgettext(domain, message):
- try:
- t = translation(domain, _localedirs.get(domain, None),
- codeset=_localecodesets.get(domain))
- except IOError:
- return message
- return t.gettext(message)
-
-def dpgettext(domain, context, message):
- try:
- t = translation(domain, _localedirs.get(domain, None),
- codeset=_localecodesets.get(domain))
- except IOError:
- return message
- return t.pgettext(context, message)
-
-def ldgettext(domain, message):
- try:
- t = translation(domain, _localedirs.get(domain, None),
- codeset=_localecodesets.get(domain))
- except IOError:
- return message
- return t.lgettext(message)
-
-def ldpgettext(domain, context, message):
- try:
- t = translation(domain, _localedirs.get(domain, None),
- codeset=_localecodesets.get(domain))
- except IOError:
- return message
- return t.lpgettext(context, message)
-
-def dngettext(domain, msgid1, msgid2, n):
- try:
- t = translation(domain, _localedirs.get(domain, None),
- codeset=_localecodesets.get(domain))
- except IOError:
- if n == 1:
- return msgid1
- else:
- return msgid2
- return t.ngettext(msgid1, msgid2, n)
-
-def dnpgettext(domain, context, msgid1, msgid2, n):
- try:
- t = translation(domain, _localedirs.get(domain, None),
- codeset=_localecodesets.get(domain))
- except IOError:
- if n == 1:
- return msgid1
- else:
- return msgid2
- return t.npgettext(context, msgid1, msgid2, n)
-
-def ldngettext(domain, msgid1, msgid2, n):
- try:
- t = translation(domain, _localedirs.get(domain, None),
- codeset=_localecodesets.get(domain))
- except IOError:
- if n == 1:
- return msgid1
- else:
- return msgid2
- return t.lngettext(msgid1, msgid2, n)
-
-def ldnpgettext(domain, context, msgid1, msgid2, n):
- try:
- t = translation(domain, _localedirs.get(domain, None),
- codeset=_localecodesets.get(domain))
- except IOError:
- if n == 1:
- return msgid1
- else:
- return msgid2
- return t.lnpgettext(context, msgid1, msgid2, n)
-
-def gettext(message):
- return dgettext(_current_domain, message)
-
-def pgettext(context, message):
- return dpgettext(_current_domain, context, message)
-
-def lgettext(message):
- return ldgettext(_current_domain, message)
-
-def lpgettext(context, message):
- return ldpgettext(_current_domain, context, message)
-
-def ngettext(msgid1, msgid2, n):
- return dngettext(_current_domain, msgid1, msgid2, n)
-
-def npgettext(context, msgid1, msgid2, n):
- return dnpgettext(_current_domain, context, msgid1, msgid2, n)
-
-def lngettext(msgid1, msgid2, n):
- return ldngettext(_current_domain, msgid1, msgid2, n)
-
-def lnpgettext(context, msgid1, msgid2, n):
- return ldnpgettext(_current_domain, context, msgid1, msgid2, n)
-
-# dcgettext() has been deemed unnecessary and is not implemented.
-
-# James Henstridge's Catalog constructor from GNOME gettext. Documented usage
-# was:
-#
-# import gettext
-# cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
-# _ = cat.gettext
-# print _('Hello World')
-
-# The resulting catalog object currently don't support access through a
-# dictionary API, which was supported (but apparently unused) in GNOME
-# gettext.
-
-Catalog = translation
--- a/hooks/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -33,7 +33,7 @@
# which may cause reloading pb
lifetime = timedelta(days=self.repo.config['keep-transaction-lifetime'])
def cleanup_old_transactions(repo=self.repo, lifetime=lifetime):
- mindate = datetime.now() - lifetime
+ mindate = datetime.utcnow() - lifetime
with repo.internal_cnx() as cnx:
cnx.system_sql(
'DELETE FROM transactions WHERE tx_time < %(time)s',
@@ -52,7 +52,7 @@
def update_feeds(repo):
# take a list to avoid iterating on a dictionary whose size may
# change
- for uri, source in list(repo.sources_by_uri.iteritems()):
+ for uri, source in list(repo.sources_by_uri.items()):
if (uri == 'system'
or not repo.config.source_enabled(source)
or not source.config['synchronize']):
@@ -72,12 +72,12 @@
def __call__(self):
def expire_dataimports(repo=self.repo):
- for uri, source in repo.sources_by_uri.iteritems():
+ for uri, source in repo.sources_by_uri.items():
if (uri == 'system'
or not repo.config.source_enabled(source)):
continue
with repo.internal_cnx() as cnx:
- mindate = datetime.now() - timedelta(seconds=source.config['logs-lifetime'])
+ mindate = datetime.utcnow() - timedelta(seconds=source.config['logs-lifetime'])
cnx.execute('DELETE CWDataImport X WHERE X start_timestamp < %(time)s',
{'time': mindate})
cnx.commit()
--- a/hooks/integrity.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/integrity.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,10 +20,12 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from threading import Lock
+from six import text_type
+
from cubicweb import validation_error, neg_role
from cubicweb.schema import (META_RTYPES, WORKFLOW_RTYPES,
RQLConstraint, RQLUniqueConstraint)
@@ -45,7 +47,7 @@
This lock used to avoid potential integrity pb when checking
RQLUniqueConstraint in two different transactions, as explained in
- http://intranet.logilab.fr/jpl/ticket/36564
+ https://extranet.logilab.fr/3577926
"""
if 'uniquecstrholder' in cnx.transaction_data:
return
@@ -109,9 +111,10 @@
category = 'integrity'
-class EnsureSymmetricRelationsAdd(hook.Hook):
+class _EnsureSymmetricRelationsAdd(hook.Hook):
""" ensure X r Y => Y r X iff r is symmetric """
__regid__ = 'cw.add_ensure_symmetry'
+ __abstract__ = True
category = 'activeintegrity'
events = ('after_add_relation',)
# __select__ is set in the registration callback
@@ -121,9 +124,10 @@
self.rtype, self.eidfrom)
-class EnsureSymmetricRelationsDelete(hook.Hook):
+class _EnsureSymmetricRelationsDelete(hook.Hook):
""" ensure X r Y => Y r X iff r is symmetric """
__regid__ = 'cw.delete_ensure_symmetry'
+ __abstract__ = True
category = 'activeintegrity'
events = ('after_delete_relation',)
# __select__ is set in the registration callback
@@ -247,7 +251,7 @@
def __call__(self):
entity = self.entity
eschema = entity.e_schema
- for attr, val in entity.cw_edited.iteritems():
+ for attr, val in entity.cw_edited.items():
if eschema.subjrels[attr].final and eschema.has_unique_values(attr):
if val is None:
continue
@@ -286,13 +290,13 @@
entity = self.entity
metaattrs = entity.e_schema.meta_attributes()
edited = entity.cw_edited
- for metaattr, (metadata, attr) in metaattrs.iteritems():
+ for metaattr, (metadata, attr) in metaattrs.items():
if metadata == 'format' and attr in edited:
try:
value = edited[attr]
except KeyError:
continue # no text to tidy
- if isinstance(value, unicode): # filter out None and Binary
+ if isinstance(value, text_type): # filter out None and Binary
if getattr(entity, str(metaattr)) == 'text/html':
edited[attr] = soup2xhtml(value, self._cw.encoding)
@@ -335,6 +339,9 @@
vreg.register_all(globals().values(), __name__)
symmetric_rtypes = [rschema.type for rschema in vreg.schema.relations()
if rschema.symmetric]
- EnsureSymmetricRelationsAdd.__select__ = hook.Hook.__select__ & hook.match_rtype(*symmetric_rtypes)
- EnsureSymmetricRelationsDelete.__select__ = hook.Hook.__select__ & hook.match_rtype(*symmetric_rtypes)
-
+ class EnsureSymmetricRelationsAdd(_EnsureSymmetricRelationsAdd):
+ __select__ = _EnsureSymmetricRelationsAdd.__select__ & hook.match_rtype(*symmetric_rtypes)
+ vreg.register(EnsureSymmetricRelationsAdd)
+ class EnsureSymmetricRelationsDelete(_EnsureSymmetricRelationsDelete):
+ __select__ = _EnsureSymmetricRelationsDelete.__select__ & hook.match_rtype(*symmetric_rtypes)
+ vreg.register(EnsureSymmetricRelationsDelete)
--- a/hooks/security.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/security.py Wed Dec 16 11:23:48 2015 +0100
@@ -207,4 +207,3 @@
rdef = rschema.rdef(self._cw.entity_metas(self.eidfrom)['type'],
self._cw.entity_metas(self.eidto)['type'])
rdef.check_perm(self._cw, 'delete', fromeid=self.eidfrom, toeid=self.eidto)
-
--- a/hooks/synccomputed.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/synccomputed.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""Hooks for synchronizing computed attributes"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from collections import defaultdict
@@ -40,7 +40,7 @@
self._container[computed_attribute] = set((eid,))
def precommit_event(self):
- for computed_attribute_rdef, eids in self.get_data().iteritems():
+ for computed_attribute_rdef, eids in self.get_data().items():
attr = computed_attribute_rdef.rtype
formula = computed_attribute_rdef.formula
select = self.cnx.repo.vreg.rqlhelper.parse(formula).children[0]
@@ -110,7 +110,7 @@
def __call__(self):
edited_attributes = frozenset(self.entity.cw_edited)
- for rdef, used_attributes in self.attributes_computed_attributes.iteritems():
+ for rdef, used_attributes in self.attributes_computed_attributes.items():
if edited_attributes.intersection(used_attributes):
# XXX optimize if the modified attributes belong to the same
# entity as the computed attribute
@@ -178,7 +178,7 @@
self.computed_attribute_by_relation[depend_on_rdef].append(rdef)
def generate_entity_creation_hooks(self):
- for etype, computed_attributes in self.computed_attribute_by_etype.iteritems():
+ for etype, computed_attributes in self.computed_attribute_by_etype.items():
regid = 'computed_attribute.%s_created' % etype
selector = hook.is_instance(etype)
yield type('%sCreatedHook' % etype,
@@ -188,7 +188,7 @@
'computed_attributes': computed_attributes})
def generate_relation_change_hooks(self):
- for rdef, computed_attributes in self.computed_attribute_by_relation.iteritems():
+ for rdef, computed_attributes in self.computed_attribute_by_relation.items():
regid = 'computed_attribute.%s_modified' % rdef.rtype
selector = hook.match_rtype(rdef.rtype.type,
frometypes=(rdef.subject.type,),
@@ -206,7 +206,7 @@
'optimized_computed_attributes': optimized_computed_attributes})
def generate_entity_update_hooks(self):
- for etype, attributes_computed_attributes in self.computed_attribute_by_etype_attrs.iteritems():
+ for etype, attributes_computed_attributes in self.computed_attribute_by_etype_attrs.items():
regid = 'computed_attribute.%s_updated' % etype
selector = hook.is_instance(etype)
yield type('%sModifiedHook' % etype,
--- a/hooks/syncschema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/syncschema.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -24,7 +24,7 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from copy import copy
from hashlib import md5
@@ -37,7 +37,7 @@
from cubicweb import validation_error
from cubicweb.predicates import is_instance
from cubicweb.schema import (SCHEMA_TYPES, META_RTYPES, VIRTUAL_RTYPES,
- CONSTRAINTS, ETYPE_NAME_MAP, display_name)
+ CONSTRAINTS, UNIQUE_CONSTRAINTS, ETYPE_NAME_MAP)
from cubicweb.server import hook, schemaserial as ss, schema2sql as y2sql
from cubicweb.server.sqlutils import SQL_PREFIX
from cubicweb.hooks.synccomputed import RecomputeAttributeOperation
@@ -158,24 +158,26 @@
cnx.transaction_data.setdefault('pendingrtypes', set()).add(rtype)
-class DropColumn(hook.Operation):
+class DropColumn(hook.DataOperationMixIn, hook.Operation):
"""actually remove the attribut's column from entity table in the system
database
"""
- table = column = None # make pylint happy
def precommit_event(self):
- cnx, table, column = self.cnx, self.table, self.column
- source = cnx.repo.system_source
- # drop index if any
- source.drop_index(cnx, table, column)
- if source.dbhelper.alter_column_support:
- cnx.system_sql('ALTER TABLE %s DROP COLUMN %s' % (table, column),
- rollback_on_failure=False)
- self.info('dropped column %s from table %s', column, table)
- else:
- # not supported by sqlite for instance
- self.error('dropping column not supported by the backend, handle '
- 'it yourself (%s.%s)', table, column)
+ cnx = self.cnx
+ for etype, attr in self.get_data():
+ table = SQL_PREFIX + etype
+ column = SQL_PREFIX + attr
+ source = cnx.repo.system_source
+ # drop index if any
+ source.drop_index(cnx, table, column)
+ if source.dbhelper.alter_column_support:
+ cnx.system_sql('ALTER TABLE %s DROP COLUMN %s' % (table, column),
+ rollback_on_failure=False)
+ self.info('dropped column %s from table %s', column, table)
+ else:
+ # not supported by sqlite for instance
+ self.error('dropping column not supported by the backend, handle '
+ 'it yourself (%s.%s)', table, column)
# XXX revertprecommit_event
@@ -208,7 +210,7 @@
repo.set_schema(repo.schema)
# CWUser class might have changed, update current session users
cwuser_cls = self.cnx.vreg['etypes'].etype_class('CWUser')
- for session in repo._sessions.itervalues():
+ for session in repo._sessions.values():
session.user.__class__ = cwuser_cls
except Exception:
self.critical('error while setting schema', exc_info=True)
@@ -330,7 +332,7 @@
self.oldvalues = dict( (attr, getattr(rschema, attr)) for attr in self.values)
self.rschema.__dict__.update(self.values)
# then make necessary changes to the system source database
- if not 'inlined' in self.values:
+ if 'inlined' not in self.values:
return # nothing to do
inlined = self.values['inlined']
# check in-lining is possible when inlined
@@ -360,8 +362,7 @@
# drop existant columns
#if cnx.repo.system_source.dbhelper.alter_column_support:
for etype in rschema.subjects():
- DropColumn(cnx, table=SQL_PREFIX + str(etype),
- column=SQL_PREFIX + rtype)
+ DropColumn.get_instance(cnx).add_data((str(etype), rtype))
else:
for etype in rschema.subjects():
try:
@@ -606,10 +607,24 @@
if rset[0][0] == 0 and not cnx.deleted_in_transaction(rdef.subject.eid):
ptypes = cnx.transaction_data.setdefault('pendingrtypes', set())
ptypes.add(rschema.type)
- DropColumn(cnx, table=SQL_PREFIX + str(rdef.subject),
- column=SQL_PREFIX + str(rschema))
+ DropColumn.get_instance(cnx).add_data((str(rdef.subject), str(rschema)))
+ elif rschema.inlined:
+ cnx.system_sql('UPDATE %s%s SET %s%s=NULL WHERE '
+ 'EXISTS(SELECT 1 FROM entities '
+ ' WHERE eid=%s%s AND type=%%(to_etype)s)'
+ % (SQL_PREFIX, rdef.subject, SQL_PREFIX, rdef.rtype,
+ SQL_PREFIX, rdef.rtype),
+ {'to_etype': rdef.object.type})
elif lastrel:
DropRelationTable(cnx, str(rschema))
+ else:
+ cnx.system_sql('DELETE FROM %s_relation WHERE '
+ 'EXISTS(SELECT 1 FROM entities '
+ ' WHERE eid=eid_from AND type=%%(from_etype)s)'
+ ' AND EXISTS(SELECT 1 FROM entities '
+ ' WHERE eid=eid_to AND type=%%(to_etype)s)'
+ % rschema,
+ {'from_etype': rdef.subject.type, 'to_etype': rdef.object.type})
# then update the in-memory schema
if rdef.subject not in ETYPE_NAME_MAP and rdef.object not in ETYPE_NAME_MAP:
rschema.del_relation_def(rdef.subject, rdef.object)
@@ -647,8 +662,7 @@
if 'indexed' in self.values:
syssource.update_rdef_indexed(cnx, rdef)
self.indexed_changed = True
- if 'cardinality' in self.values and (rdef.rtype.final or
- rdef.rtype.inlined) \
+ if 'cardinality' in self.values and rdef.rtype.final \
and self.values['cardinality'][0] != self.oldvalues['cardinality'][0]:
syssource.update_rdef_null_allowed(self.cnx, rdef)
self.null_allowed_changed = True
@@ -717,8 +731,8 @@
syssource.update_rdef_unique(cnx, rdef)
self.unique_changed = True
if cstrtype in ('BoundaryConstraint', 'IntervalBoundConstraint', 'StaticVocabularyConstraint'):
- cstrname = 'cstr' + md5(rdef.subject.type + rdef.rtype.type + cstrtype +
- (self.oldcstr.serialize() or '')).hexdigest()
+ cstrname = 'cstr' + md5((rdef.subject.type + rdef.rtype.type + cstrtype +
+ (self.oldcstr.serialize() or '')).encode('utf-8')).hexdigest()
cnx.system_sql('ALTER TABLE %s%s DROP CONSTRAINT %s' % (SQL_PREFIX, rdef.subject.type, cstrname))
def revertprecommit_event(self):
@@ -749,7 +763,10 @@
return
rdef = self.rdef = cnx.vreg.schema.schema_by_eid(rdefentity.eid)
cstrtype = self.entity.type
- oldcstr = self.oldcstr = rdef.constraint_by_type(cstrtype)
+ if cstrtype in UNIQUE_CONSTRAINTS:
+ oldcstr = self.oldcstr = rdef.constraint_by_type(cstrtype)
+ else:
+ oldcstr = None
newcstr = self.newcstr = CONSTRAINTS[cstrtype].deserialize(self.entity.value)
# in-place modification of in-memory schema first
_set_modifiable_constraints(rdef)
@@ -769,8 +786,8 @@
self.unique_changed = True
if cstrtype in ('BoundaryConstraint', 'IntervalBoundConstraint', 'StaticVocabularyConstraint'):
if oldcstr is not None:
- oldcstrname = 'cstr' + md5(rdef.subject.type + rdef.rtype.type + cstrtype +
- (self.oldcstr.serialize() or '')).hexdigest()
+ oldcstrname = 'cstr' + md5((rdef.subject.type + rdef.rtype.type + cstrtype +
+ (self.oldcstr.serialize() or '')).encode('ascii')).hexdigest()
cnx.system_sql('ALTER TABLE %s%s DROP CONSTRAINT %s' %
(SQL_PREFIX, rdef.subject.type, oldcstrname))
cstrname, check = y2sql.check_constraint(rdef.subject, rdef.object, rdef.rtype.type,
@@ -905,11 +922,6 @@
# duh, schema not found, log error and skip operation
self.warning('no schema for %s', self.eid)
return
- if isinstance(erschema, RelationSchema): # XXX 3.6 migration
- return
- if isinstance(erschema, RelationDefinitionSchema) and \
- self.action in ('delete', 'add'): # XXX 3.6.1 migration
- return
perms = list(erschema.action_permissions(self.action))
if self.group_eid is not None:
perm = self.cnx.entity_from_eid(self.group_eid).name
@@ -972,7 +984,6 @@
raise validation_error(self.entity, {None: _("can't be deleted")})
# delete every entities of this type
if name not in ETYPE_NAME_MAP:
- self._cw.execute('DELETE %s X' % name)
MemSchemaCWETypeDel(self._cw, etype=name)
DropTable(self._cw, table=SQL_PREFIX + name)
@@ -1155,10 +1166,6 @@
else:
rdeftype = 'CWRelation'
pendingrdefs.add((subjschema, rschema, objschema))
- if not (cnx.deleted_in_transaction(subjschema.eid) or
- cnx.deleted_in_transaction(objschema.eid)):
- cnx.execute('DELETE X %s Y WHERE X is %s, Y is %s'
- % (rschema, subjschema, objschema))
RDefDelOp(cnx, rdef=rdef)
--- a/hooks/syncsession.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/syncsession.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""Core hooks: synchronize living session on persistent data changes"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from cubicweb import UnknownProperty, BadConnectionId, validation_error
from cubicweb.predicates import is_instance
@@ -26,7 +26,7 @@
def get_user_sessions(repo, ueid):
- for session in repo._sessions.itervalues():
+ for session in repo._sessions.values():
if ueid == session.user.eid:
yield session
@@ -114,7 +114,7 @@
def __call__(self):
"""modify user permission, need to update users"""
for session in get_user_sessions(self._cw.repo, self.entity.eid):
- _DelUserOp(self._cw, session.id)
+ _DelUserOp(self._cw, session.sessionid)
# CWProperty hooks #############################################################
--- a/hooks/syncsources.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/syncsources.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,7 +17,7 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""hooks for repository sources synchronization"""
-_ = unicode
+from cubicweb import _
from socket import gethostname
@@ -119,7 +119,7 @@
msg = _("You cannot rename the system source")
raise validation_error(self.entity, {('name', 'subject'): msg})
SourceRenamedOp(self._cw, oldname=oldname, newname=newname)
- if 'config' in self.entity.cw_edited:
+ if 'config' in self.entity.cw_edited or 'url' in self.entity.cw_edited:
if self.entity.name == 'system' and self.entity.config:
msg = _("Configuration of the system source goes to "
"the 'sources' file, not in the database")
--- a/hooks/test/data/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/test/data/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,7 +22,7 @@
from cubicweb.schema import ERQLExpression
-_ = unicode
+from cubicweb import _
class friend(RelationDefinition):
subject = ('CWUser', 'CWGroup')
--- a/hooks/test/unittest_hooks.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/test/unittest_hooks.py Wed Dec 16 11:23:48 2015 +0100
@@ -24,9 +24,12 @@
from datetime import datetime
+from six import text_type
+
from cubicweb import ValidationError, AuthenticationError, BadConnectionId
from cubicweb.devtools.testlib import CubicWebTC
+
class CoreHooksTC(CubicWebTC):
def test_inlined(self):
@@ -207,7 +210,7 @@
with self.assertRaises(ValidationError) as cm:
cnx.execute('INSERT CWUser X: X login "admin"')
ex = cm.exception
- ex.translate(unicode)
+ ex.translate(text_type)
self.assertIsInstance(ex.entity, int)
self.assertEqual(ex.errors, {'login-subject': 'the value "admin" is already used, use another one'})
--- a/hooks/test/unittest_synccomputed.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/test/unittest_synccomputed.py Wed Dec 16 11:23:48 2015 +0100
@@ -62,7 +62,7 @@
def test_computed_attribute_by_relation(self):
comp_by_rdef = self.dependencies.computed_attribute_by_relation
self.assertEqual(len(comp_by_rdef), 1)
- key, values = iter(comp_by_rdef.iteritems()).next()
+ key, values = next(iter(comp_by_rdef.items()))
self.assertEqual(key.rtype, 'works_for')
self.assertEqual(len(values), 1)
self.assertEqual(values[0].rtype, 'total_salary')
@@ -73,7 +73,7 @@
values = comp_by_attr['Person']
self.assertEqual(len(values), 2)
values = set((rdef.formula, tuple(v))
- for rdef, v in values.iteritems())
+ for rdef, v in values.items())
self.assertEquals(values,
set((('Any 2014 - D WHERE X birth_year D', tuple(('birth_year',))),
('Any SUM(SA) GROUPBY X WHERE P works_for X, P salary SA', tuple(('salary',)))))
--- a/hooks/test/unittest_syncschema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/test/unittest_syncschema.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,6 +19,7 @@
from logilab.common.testlib import unittest_main
+from yams.constraints import BoundaryConstraint
from cubicweb import ValidationError, Binary
from cubicweb.schema import META_RTYPES
from cubicweb.devtools import startpgcluster, stoppgcluster, PostgresApptestConfiguration
@@ -382,5 +383,23 @@
self.assertEqual(cstr.values, (u'normal', u'auto', u'new'))
cnx.execute('INSERT Transition T: T name "hop", T type "new"')
+ def test_add_constraint(self):
+ with self.admin_access.repo_cnx() as cnx:
+ rdef = self.schema['EmailPart'].rdef('ordernum')
+ cstr = BoundaryConstraint('>=', 0)
+ cnx.execute('INSERT CWConstraint X: X value %(v)s, X cstrtype CT, '
+ 'EDEF constrained_by X WHERE CT name %(ct)s, EDEF eid %(x)s',
+ {'ct': cstr.__class__.__name__, 'v': cstr.serialize(), 'x': rdef.eid})
+ cnx.commit()
+ cstr2 = rdef.constraint_by_type('BoundaryConstraint')
+ self.assertEqual(cstr, cstr2)
+ cstr3 = BoundaryConstraint('<=', 1000)
+ cnx.execute('INSERT CWConstraint X: X value %(v)s, X cstrtype CT, '
+ 'EDEF constrained_by X WHERE CT name %(ct)s, EDEF eid %(x)s',
+ {'ct': cstr3.__class__.__name__, 'v': cstr3.serialize(), 'x': rdef.eid})
+ cnx.commit()
+ self.assertCountEqual(rdef.constraints, [cstr, cstr3])
+
+
if __name__ == '__main__':
unittest_main()
--- a/hooks/test/unittest_syncsession.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/test/unittest_syncsession.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,6 +22,8 @@
syncschema.py hooks are mostly tested in server/test/unittest_migrations.py
"""
+from six import text_type
+
from cubicweb import ValidationError
from cubicweb.devtools.testlib import CubicWebTC
@@ -32,13 +34,13 @@
with self.assertRaises(ValidationError) as cm:
req.execute('INSERT CWProperty X: X pkey "bla.bla", '
'X value "hop", X for_user U')
- cm.exception.translate(unicode)
+ cm.exception.translate(text_type)
self.assertEqual(cm.exception.errors,
{'pkey-subject': 'unknown property key bla.bla'})
with self.assertRaises(ValidationError) as cm:
req.execute('INSERT CWProperty X: X pkey "bla.bla", X value "hop"')
- cm.exception.translate(unicode)
+ cm.exception.translate(text_type)
self.assertEqual(cm.exception.errors,
{'pkey-subject': 'unknown property key bla.bla'})
--- a/hooks/workflow.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/workflow.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""Core hooks: workflow related hooks"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from datetime import datetime
@@ -355,4 +355,3 @@
typewf = entity.cw_adapt_to('IWorkflowable').cwetype_workflow()
if typewf is not None:
_WorkflowChangedOp(self._cw, eid=self.eidfrom, wfeid=typewf.eid)
-
--- a/hooks/zmq.py Tue Dec 15 19:05:41 2015 +0100
+++ b/hooks/zmq.py Wed Dec 16 11:23:48 2015 +0100
@@ -48,5 +48,3 @@
for address in address_sub:
self.repo.app_instances_bus.add_subscriber(address)
self.repo.app_instances_bus.start()
-
-
--- a/i18n.py Tue Dec 15 19:05:41 2015 +0100
+++ b/i18n.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""Some i18n/gettext utilities."""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -24,6 +25,8 @@
from os.path import join, basename, splitext, exists
from glob import glob
+from six import PY2
+
from cubicweb.toolsutils import create_dir
def extract_from_tal(files, output_file):
@@ -39,10 +42,10 @@
def add_msg(w, msgid, msgctx=None):
"""write an empty pot msgid definition"""
- if isinstance(msgid, unicode):
+ if PY2 and isinstance(msgid, unicode):
msgid = msgid.encode('utf-8')
if msgctx:
- if isinstance(msgctx, unicode):
+ if PY2 and isinstance(msgctx, unicode):
msgctx = msgctx.encode('utf-8')
w('msgctxt "%s"\n' % msgctx)
msgid = msgid.replace('"', r'\"').splitlines()
@@ -80,7 +83,7 @@
"""
from subprocess import CalledProcessError
from logilab.common.fileutils import ensure_fs_mode
- print '-> compiling message catalogs to %s' % destdir
+ print('-> compiling message catalogs to %s' % destdir)
errors = []
for lang in langs:
langdir = join(destdir, lang, 'LC_MESSAGES')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/jshintrc Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,115 @@
+{
+ // --------------------------------------------------------------------
+ // JSHint Configuration, Strict Edition
+ // --------------------------------------------------------------------
+ //
+ // This is a options template for [JSHint][1], using [JSHint example][2]
+ // and [Ory Band's example][3] as basis and setting config values to
+ // be most strict:
+ //
+ // * set all enforcing options to true
+ // * set all relaxing options to false
+ // * set all environment options to false, except the browser value
+ // * set all JSLint legacy options to false
+ //
+ // [1]: http://www.jshint.com/
+ // [2]: https://github.com/jshint/node-jshint/blob/master/example/config.json
+ // [3]: https://github.com/oryband/dotfiles/blob/master/jshintrc
+ //
+ // @author http://michael.haschke.biz/
+ // @license http://unlicense.org/
+
+ // == Enforcing Options ===============================================
+ //
+ // These options tell JSHint to be more strict towards your code. Use
+ // them if you want to allow only a safe subset of JavaScript, very
+ // useful when your codebase is shared with a big number of developers
+ // with different skill levels.
+
+ "bitwise" : true, // Prohibit bitwise operators (&, |, ^, etc.).
+ "curly" : true, // Require {} for every new block or scope.
+ "eqeqeq" : true, // Require triple equals i.e. `===`.
+ "forin" : true, // Tolerate `for in` loops without `hasOwnPrototype`.
+ "immed" : true, // Require immediate invocations to be wrapped in parens e.g. `( function(){}() );`
+ "latedef" : true, // Prohibit variable use before definition.
+ "newcap" : true, // Require capitalization of all constructor functions e.g. `new F()`.
+ "noarg" : true, // Prohibit use of `arguments.caller` and `arguments.callee`.
+ "noempty" : true, // Prohibit use of empty blocks.
+ "nonew" : true, // Prohibit use of constructors for side-effects.
+ "plusplus" : true, // Prohibit use of `++` & `--`.
+ "regexp" : true, // Prohibit `.` and `[^...]` in regular expressions.
+ "undef" : true, // Require all non-global variables be declared before they are used.
+ "strict" : true, // Require `use strict` pragma in every file.
+ "trailing" : true, // Prohibit trailing whitespaces.
+
+ // == Relaxing Options ================================================
+ //
+ // These options allow you to suppress certain types of warnings. Use
+ // them only if you are absolutely positive that you know what you are
+ // doing.
+
+ "asi" : false, // Tolerate Automatic Semicolon Insertion (no semicolons).
+ "boss" : false, // Tolerate assignments inside if, for & while. Usually conditions & loops are for comparison, not assignments.
+ "debug" : false, // Allow debugger statements e.g. browser breakpoints.
+ "eqnull" : false, // Tolerate use of `== null`.
+ "es5" : false, // Allow EcmaScript 5 syntax.
+ "esnext" : false, // Allow ES.next specific features such as `const` and `let`.
+ "evil" : false, // Tolerate use of `eval`.
+ "expr" : false, // Tolerate `ExpressionStatement` as Programs.
+ "funcscope" : false, // Tolerate declarations of variables inside of control structures while accessing them later from the outside.
+ "globalstrict" : false, // Allow global "use strict" (also enables 'strict').
+ "iterator" : false, // Allow usage of __iterator__ property.
+ "lastsemic" : false, // Tolerat missing semicolons when the it is omitted for the last statement in a one-line block.
+ "laxbreak" : false, // Tolerate unsafe line breaks e.g. `return [\n] x` without semicolons.
+ "laxcomma" : false, // Suppress warnings about comma-first coding style.
+ "loopfunc" : false, // Allow functions to be defined within loops.
+ "multistr" : false, // Tolerate multi-line strings.
+ "onecase" : false, // Tolerate switches with just one case.
+ "proto" : false, // Tolerate __proto__ property. This property is deprecated.
+ "regexdash" : false, // Tolerate unescaped last dash i.e. `[-...]`.
+ "scripturl" : false, // Tolerate script-targeted URLs.
+ "smarttabs" : false, // Tolerate mixed tabs and spaces when the latter are used for alignmnent only.
+ "shadow" : false, // Allows re-define variables later in code e.g. `var x=1; x=2;`.
+ "sub" : false, // Tolerate all forms of subscript notation besides dot notation e.g. `dict['key']` instead of `dict.key`.
+ "supernew" : false, // Tolerate `new function () { ... };` and `new Object;`.
+ "validthis" : false, // Tolerate strict violations when the code is running in strict mode and you use this in a non-constructor function.
+
+ // == Environments ====================================================
+ //
+ // These options pre-define global variables that are exposed by
+ // popular JavaScript libraries and runtime environments—such as
+ // browser or node.js.
+
+ "browser" : true, // Standard browser globals e.g. `window`, `document`.
+ "couch" : false, // Enable globals exposed by CouchDB.
+ "devel" : false, // Allow development statements e.g. `console.log();`.
+ "dojo" : false, // Enable globals exposed by Dojo Toolkit.
+ "jquery" : false, // Enable globals exposed by jQuery JavaScript library.
+ "mootools" : false, // Enable globals exposed by MooTools JavaScript framework.
+ "node" : false, // Enable globals available when code is running inside of the NodeJS runtime environment.
+ "nonstandard" : false, // Define non-standard but widely adopted globals such as escape and unescape.
+ "prototypejs" : false, // Enable globals exposed by Prototype JavaScript framework.
+ "rhino" : false, // Enable globals available when your code is running inside of the Rhino runtime environment.
+ "wsh" : false, // Enable globals available when your code is running as a script for the Windows Script Host.
+
+ // == JSLint Legacy ===================================================
+ //
+ // These options are legacy from JSLint. Aside from bug fixes they will
+ // not be improved in any way and might be removed at any point.
+
+ "nomen" : false, // Prohibit use of initial or trailing underbars in names.
+ "onevar" : false, // Allow only one `var` statement per function.
+ "passfail" : false, // Stop on first error.
+ "white" : false, // Check against strict whitespace and indentation rules.
+
+ // == Undocumented Options ============================================
+ //
+ // While I've found these options in [example1][2] and [example2][3]
+ // they are not described in the [JSHint Options documentation][4].
+ //
+ // [4]: http://www.jshint.com/options/
+
+ "maxerr" : 100, // Maximum errors before stopping.
+ "predef" : [], // Extra globals.
+ "indent" : 4 // Specify indentation spacing
+}
--- a/mail.py Tue Dec 15 19:05:41 2015 +0100
+++ b/mail.py Wed Dec 16 11:23:48 2015 +0100
@@ -28,16 +28,27 @@
from email.utils import formatdate
from socket import gethostname
+from six import PY2, PY3, text_type
+
+
def header(ustring):
+ if PY3:
+ return Header(ustring, 'utf-8')
return Header(ustring.encode('UTF-8'), 'UTF-8')
def addrheader(uaddr, uname=None):
# even if an email address should be ascii, encode it using utf8 since
# automatic tests may generate non ascii email address
- addr = uaddr.encode('UTF-8')
+ if PY2:
+ addr = uaddr.encode('UTF-8')
+ else:
+ addr = uaddr
if uname:
- return '%s <%s>' % (header(uname).encode(), addr)
- return addr
+ val = '%s <%s>' % (header(uname).encode(), addr)
+ else:
+ val = addr
+ assert isinstance(val, str) # bytes in py2, ascii-encoded unicode in py3
+ return val
def construct_message_id(appid, eid, withtimestamp=True):
@@ -46,7 +57,7 @@
else:
addrpart = 'eid=%s' % eid
# we don't want any equal sign nor trailing newlines
- leftpart = b64encode(addrpart, '.-').rstrip().rstrip('=')
+ leftpart = b64encode(addrpart.encode('ascii'), b'.-').decode('ascii').rstrip().rstrip('=')
return '<%s@%s.%s>' % (leftpart, appid, gethostname())
@@ -75,7 +86,7 @@
to_addrs and cc_addrs are expected to be a list of email address without
name
"""
- assert type(content) is unicode, repr(content)
+ assert isinstance(content, text_type), repr(content)
msg = MIMEText(content.encode('UTF-8'), 'plain', 'UTF-8')
# safety: keep only the first newline
try:
@@ -86,13 +97,13 @@
if uinfo.get('email'):
email = uinfo['email']
elif config and config['sender-addr']:
- email = unicode(config['sender-addr'])
+ email = text_type(config['sender-addr'])
else:
email = u''
if uinfo.get('name'):
name = uinfo['name']
elif config and config['sender-name']:
- name = unicode(config['sender-name'])
+ name = text_type(config['sender-name'])
else:
name = u''
msg['From'] = addrheader(email, name)
--- a/md5crypt.py Tue Dec 15 19:05:41 2015 +0100
+++ b/md5crypt.py Wed Dec 16 11:23:48 2015 +0100
@@ -38,31 +38,37 @@
this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp
"""
-MAGIC = '$1$' # Magic string
-ITOA64 = "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+MAGIC = b'$1$' # Magic string
+ITOA64 = b"./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
from hashlib import md5 # pylint: disable=E0611
+from six import text_type, indexbytes
+from six.moves import range
+
+
def to64 (v, n):
- ret = ''
+ ret = bytearray()
while (n - 1 >= 0):
n = n - 1
- ret = ret + ITOA64[v & 0x3f]
+ ret.append(ITOA64[v & 0x3f])
v = v >> 6
return ret
def crypt(pw, salt):
- if isinstance(pw, unicode):
+ if isinstance(pw, text_type):
pw = pw.encode('utf-8')
+ if isinstance(salt, text_type):
+ salt = salt.encode('ascii')
# Take care of the magic string if present
if salt.startswith(MAGIC):
salt = salt[len(MAGIC):]
# salt can have up to 8 characters:
- salt = salt.split('$', 1)[0]
+ salt = salt.split(b'$', 1)[0]
salt = salt[:8]
ctx = pw + MAGIC + salt
final = md5(pw + salt + pw).digest()
- for pl in xrange(len(pw), 0, -16):
+ for pl in range(len(pw), 0, -16):
if pl > 16:
ctx = ctx + final[:16]
else:
@@ -71,7 +77,7 @@
i = len(pw)
while i:
if i & 1:
- ctx = ctx + chr(0) #if ($i & 1) { $ctx->add(pack("C", 0)); }
+ ctx = ctx + b'\0' #if ($i & 1) { $ctx->add(pack("C", 0)); }
else:
ctx = ctx + pw[0]
i = i >> 1
@@ -79,8 +85,8 @@
# The following is supposed to make
# things run slower.
# my question: WTF???
- for i in xrange(1000):
- ctx1 = ''
+ for i in range(1000):
+ ctx1 = b''
if i & 1:
ctx1 = ctx1 + pw
else:
@@ -95,21 +101,21 @@
ctx1 = ctx1 + pw
final = md5(ctx1).digest()
# Final xform
- passwd = ''
- passwd = passwd + to64((int(ord(final[0])) << 16)
- |(int(ord(final[6])) << 8)
- |(int(ord(final[12]))),4)
- passwd = passwd + to64((int(ord(final[1])) << 16)
- |(int(ord(final[7])) << 8)
- |(int(ord(final[13]))), 4)
- passwd = passwd + to64((int(ord(final[2])) << 16)
- |(int(ord(final[8])) << 8)
- |(int(ord(final[14]))), 4)
- passwd = passwd + to64((int(ord(final[3])) << 16)
- |(int(ord(final[9])) << 8)
- |(int(ord(final[15]))), 4)
- passwd = passwd + to64((int(ord(final[4])) << 16)
- |(int(ord(final[10])) << 8)
- |(int(ord(final[5]))), 4)
- passwd = passwd + to64((int(ord(final[11]))), 2)
+ passwd = b''
+ passwd += to64((indexbytes(final, 0) << 16)
+ |(indexbytes(final, 6) << 8)
+ |(indexbytes(final, 12)),4)
+ passwd += to64((indexbytes(final, 1) << 16)
+ |(indexbytes(final, 7) << 8)
+ |(indexbytes(final, 13)), 4)
+ passwd += to64((indexbytes(final, 2) << 16)
+ |(indexbytes(final, 8) << 8)
+ |(indexbytes(final, 14)), 4)
+ passwd += to64((indexbytes(final, 3) << 16)
+ |(indexbytes(final, 9) << 8)
+ |(indexbytes(final, 15)), 4)
+ passwd += to64((indexbytes(final, 4) << 16)
+ |(indexbytes(final, 10) << 8)
+ |(indexbytes(final, 5)), 4)
+ passwd += to64((indexbytes(final, 11)), 2)
return passwd
--- a/migration.py Tue Dec 15 19:05:41 2015 +0100
+++ b/migration.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""utilities for instances migration"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -25,6 +26,9 @@
import tempfile
from os.path import exists, join, basename, splitext
from itertools import chain
+from warnings import warn
+
+from six import string_types
from logilab.common import IGNORED_EXTENSIONS
from logilab.common.decorators import cached
@@ -49,7 +53,7 @@
assert fromversion <= toversion, (fromversion, toversion)
if not exists(directory):
if not quiet:
- print directory, "doesn't exists, no migration path"
+ print(directory, "doesn't exists, no migration path")
return []
if fromversion == toversion:
return []
@@ -93,9 +97,9 @@
stream = open(scriptpath)
scriptcontent = stream.read()
stream.close()
- print
- print scriptcontent
- print
+ print()
+ print(scriptcontent)
+ print()
else:
return True
@@ -139,9 +143,6 @@
raise
raise AttributeError(name)
- def repo_connect(self):
- return self.config.repository()
-
def migrate(self, vcconf, toupgrade, options):
"""upgrade the given set of cubes
@@ -243,7 +244,7 @@
# avoid '_' to be added to builtins by sys.display_hook
def do_not_add___to_builtins(obj):
if obj is not None:
- print repr(obj)
+ print(repr(obj))
sys.displayhook = do_not_add___to_builtins
local_ctx = self._create_context()
try:
@@ -349,7 +350,16 @@
else:
pyname = splitext(basename(migrscript))[0]
scriptlocals['__name__'] = pyname
- execfile(migrscript, scriptlocals)
+ with open(migrscript, 'rb') as fobj:
+ fcontent = fobj.read()
+ try:
+ code = compile(fcontent, migrscript, 'exec')
+ except SyntaxError:
+ # try without print_function
+ code = compile(fcontent, migrscript, 'exec', 0, True)
+ warn('[3.22] script %r should be updated to work with print_function'
+ % migrscript, DeprecationWarning)
+ exec(code, scriptlocals)
if funcname is not None:
try:
func = scriptlocals[funcname]
@@ -399,7 +409,7 @@
"""modify the list of used cubes in the in-memory config
returns newly inserted cubes, including dependencies
"""
- if isinstance(cubes, basestring):
+ if isinstance(cubes, string_types):
cubes = (cubes,)
origcubes = self.config.cubes()
newcubes = [p for p in self.config.expand_cubes(cubes)
@@ -454,6 +464,10 @@
def version_strictly_lower(a, b):
+ if a is None:
+ return True
+ if b is None:
+ return False
if a:
a = Version(a)
if b:
@@ -491,8 +505,8 @@
self.dependencies[cube] = dict(self.config.cube_dependencies(cube))
self.dependencies[cube]['cubicweb'] = self.config.cube_depends_cubicweb_version(cube)
# compute reverse dependencies
- for cube, dependencies in self.dependencies.iteritems():
- for name, constraint in dependencies.iteritems():
+ for cube, dependencies in self.dependencies.items():
+ for name, constraint in dependencies.items():
self.reverse_dependencies.setdefault(name,set())
if constraint:
try:
@@ -522,9 +536,9 @@
elif op == None:
continue
else:
- print ('unable to handle %s in %s, set to `%s %s` '
- 'but currently up to `%s %s`' %
- (cube, source, oper, version, op, ver))
+ print('unable to handle %s in %s, set to `%s %s` '
+ 'but currently up to `%s %s`' %
+ (cube, source, oper, version, op, ver))
# "solve" constraint satisfaction problem
if cube not in self.cubes:
self.errors.append( ('add', cube, version, source) )
@@ -536,4 +550,4 @@
elif oper is None:
pass # no constraint on version
else:
- print 'unknown operator', oper
+ print('unknown operator', oper)
--- a/misc/cwfs/cwfs.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/cwfs/cwfs.py Wed Dec 16 11:23:48 2015 +0100
@@ -80,17 +80,17 @@
self._restrictions = []
def parse(self) :
- self._entity = self._components.next()
+ self._entity = next(self._components)
try:
self.process_entity()
except StopIteration :
pass
def process_entity(self) :
- _next = self._components.next()
+ _next = next(self._components)
if _next in self.schema.get_attrs(self._entity) :
self._attr = _next
- _next = self._components.next()
+ _next = next(self._components)
self._restrictions.append( (self._entity, self._attr, _next) )
self._attr = None
self._rel = None
@@ -136,7 +136,7 @@
def parse(self):
self._var = self._alphabet.pop(0)
- self._e_type = self._components.next()
+ self._e_type = next(self._components)
e_type = self._e_type.capitalize()
self._restrictions.append('%s is %s' % (self._var, e_type))
try:
@@ -146,11 +146,11 @@
return 'Any %s WHERE %s' % (self._var, ', '.join(self._restrictions))
def process_entity(self) :
- _next = self._components.next()
+ _next = next(self._components)
if _next in self.schema.get_attrs(self._e_type) :
attr = _next
try:
- _next = self._components.next()
+ _next = next(self._components)
self._restrictions.append('%s %s %s' % (self._var, attr, _next))
except StopIteration:
a_var = self._alphabet.pop(0)
@@ -163,7 +163,7 @@
self._restrictions.append('%s %s %s' % (self._var, rel, r_var))
self._var = r_var
try:
- _next = self._components.next()
+ _next = next(self._components)
self._restrictions.append('%s is %s' % (r_var, _next.capitalize()))
except StopIteration:
raise
@@ -173,4 +173,3 @@
def to_rql(path) :
p = SytPathParser(SCHEMA,path)
return p.parse()
-
--- a/misc/cwfs/cwfs_test.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/cwfs/cwfs_test.py Wed Dec 16 11:23:48 2015 +0100
@@ -30,7 +30,7 @@
sections = []
buffer = ""
in_section = False
- for line in file(filename) :
+ for line in open(filename) :
if line.startswith('Test::'):
in_section = True
buffer = ""
--- a/misc/cwzope/cwzope.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/cwzope/cwzope.py Wed Dec 16 11:23:48 2015 +0100
@@ -48,4 +48,3 @@
cnx = connect(user, password, host, database, group)
CNX_CACHE[key] = cnx
return cnx
-
--- a/misc/migration/3.10.0_Any.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/migration/3.10.0_Any.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,3 +1,5 @@
+from six import text_type
+
from cubicweb.server.session import hooks_control
for uri, cfg in config.read_sources_file().items():
@@ -24,7 +26,7 @@
repo.sources_by_uri.pop(uri)
config = u'\n'.join('%s=%s' % (key, value) for key, value in cfg.items()
if key != 'adapter' and value is not None)
- create_entity('CWSource', name=unicode(uri), type=unicode(cfg['adapter']),
+ create_entity('CWSource', name=text_type(uri), type=text_type(cfg['adapter']),
config=config)
commit()
@@ -33,4 +35,3 @@
'X pkey ~= "boxes.%" OR '
'X pkey ~= "contentnavigation.%"').entities():
x.cw_set(pkey=u'ctxcomponents.' + x.pkey.split('.', 1)[1])
-
--- a/misc/migration/3.14.0_Any.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/migration/3.14.0_Any.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
config['rql-cache-size'] = config['rql-cache-size'] * 10
add_entity_type('CWDataImport')
@@ -10,4 +12,4 @@
mainvars = guess_rrqlexpr_mainvars(expression)
yamscstr = CONSTRAINTS[rqlcstr.type](expression, mainvars)
rqlcstr.cw_set(value=yamscstr.serialize())
- print 'updated', rqlcstr.type, rqlcstr.value.strip()
+ print('updated', rqlcstr.type, rqlcstr.value.strip())
--- a/misc/migration/3.15.4_Any.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/migration/3.15.4_Any.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
from logilab.common.shellutils import generate_password
from cubicweb.server.utils import crypt_password
@@ -5,7 +7,7 @@
salt = user.upassword.getvalue()
if crypt_password('', salt) == salt:
passwd = generate_password()
- print 'setting random password for user %s' % user.login
+ print('setting random password for user %s' % user.login)
user.set_attributes(upassword=passwd)
commit()
--- a/misc/migration/3.21.0_Any.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/migration/3.21.0_Any.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
from cubicweb.schema import PURE_VIRTUAL_RTYPES
from cubicweb.server.schema2sql import rschema_has_table
@@ -27,7 +29,7 @@
' SELECT eid FROM entities) AS eids' % args,
ask_confirm=False)[0][0]
if count:
- print '%s references %d unknown entities, deleting' % (rschema, count)
+ print('%s references %d unknown entities, deleting' % (rschema, count))
sql('DELETE FROM %(r)s_relation '
'WHERE eid_from IN (SELECT eid_from FROM %(r)s_relation EXCEPT SELECT eid FROM entities)' % args)
sql('DELETE FROM %(r)s_relation '
@@ -65,14 +67,14 @@
broken_eids = sql('SELECT cw_eid FROM cw_%(e)s WHERE cw_%(r)s IS NULL' % args,
ask_confirm=False)
if broken_eids:
- print 'Required relation %(e)s.%(r)s missing' % args
+ print('Required relation %(e)s.%(r)s missing' % args)
args['eids'] = ', '.join(str(eid) for eid, in broken_eids)
rql('DELETE %(e)s X WHERE X eid IN (%(eids)s)' % args)
broken_eids = sql('SELECT cw_eid FROM cw_%(e)s WHERE cw_%(r)s IN (SELECT cw_%(r)s FROM cw_%(e)s '
'EXCEPT SELECT eid FROM entities)' % args,
ask_confirm=False)
if broken_eids:
- print 'Required relation %(e)s.%(r)s references unknown objects, deleting subject entities' % args
+ print('Required relation %(e)s.%(r)s references unknown objects, deleting subject entities' % args)
args['eids'] = ', '.join(str(eid) for eid, in broken_eids)
rql('DELETE %(e)s X WHERE X eid IN (%(eids)s)' % args)
else:
@@ -81,7 +83,7 @@
' EXCEPT'
' SELECT eid FROM entities) AS eids' % args,
ask_confirm=False)[0][0]:
- print '%(e)s.%(r)s references unknown entities, deleting relation' % args
+ print('%(e)s.%(r)s references unknown entities, deleting relation' % args)
sql('UPDATE cw_%(e)s SET cw_%(r)s = NULL WHERE cw_%(r)s IS NOT NULL AND cw_%(r)s IN '
'(SELECT cw_%(r)s FROM cw_%(e)s EXCEPT SELECT eid FROM entities)' % args)
@@ -104,7 +106,7 @@
' EXCEPT'
' SELECT eid FROM entities) AS eids' % args,
ask_confirm=False)[0][0]:
- print '%(e)s has nonexistent entities, deleting' % args
+ print('%(e)s has nonexistent entities, deleting' % args)
sql('DELETE FROM cw_%(e)s WHERE cw_eid IN '
'(SELECT cw_eid FROM cw_%(e)s EXCEPT SELECT eid FROM entities)' % args)
args['c'] = 'cw_%(e)s_cw_eid_fkey' % args
--- a/misc/migration/3.8.5_Any.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/migration/3.8.5_Any.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
def migrate_varchar_to_nvarchar():
dbdriver = config.system_source_config['db-driver']
if dbdriver != "sqlserver2005":
@@ -52,7 +54,7 @@
for statement in generated_statements:
- print statement
+ print(statement)
sql(statement, ask_confirm=False)
commit()
--- a/misc/migration/bootstrapmigration_repository.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/migration/bootstrapmigration_repository.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,6 +19,9 @@
it should only include low level schema changes
"""
+from __future__ import print_function
+
+from six import text_type
from cubicweb import ConfigurationError
from cubicweb.server.session import hooks_control
@@ -77,8 +80,8 @@
sql('ALTER TABLE "entities" DROP COLUMN "mtime"')
sql('ALTER TABLE "entities" DROP COLUMN "source"')
except: # programming error, already migrated
- print "Failed to drop mtime or source database columns"
- print "'entities' table of the database has probably been already updated"
+ print("Failed to drop mtime or source database columns")
+ print("'entities' table of the database has probably been already updated")
commit()
@@ -101,7 +104,7 @@
driver = config.system_source_config['db-driver']
if not (driver == 'postgres' or driver.startswith('sqlserver')):
import sys
- print >>sys.stderr, 'This migration is not supported for backends other than sqlserver or postgres (yet).'
+ print('This migration is not supported for backends other than sqlserver or postgres (yet).', file=sys.stderr)
sys.exit(1)
add_relation_definition('CWAttribute', 'add_permission', 'CWGroup')
@@ -148,7 +151,7 @@
default = yams.DATE_FACTORY_MAP[atype](default)
else:
assert atype == 'String', atype
- default = unicode(default)
+ default = text_type(default)
return Binary.zpickle(default)
dbh = repo.system_source.dbhelper
@@ -196,7 +199,7 @@
(rschema.type, ','.join(subjects))))
if martians:
martians = ','.join(martians)
- print 'deleting broken relations %s for eids %s' % (rschema.type, martians)
+ print('deleting broken relations %s for eids %s' % (rschema.type, martians))
sql('DELETE FROM %s_relation WHERE eid_from IN (%s) OR eid_to IN (%s)' % (rschema.type, martians, martians))
with session.deny_all_hooks_but():
rql('SET X %(r)s Y WHERE Y %(r)s X, NOT X %(r)s Y' % {'r': rschema.type})
@@ -219,20 +222,20 @@
if driver == 'postgres':
for indexname, in sql('select indexname from pg_indexes'):
if indexname.startswith('unique_'):
- print 'dropping index', indexname
+ print('dropping index', indexname)
sql('DROP INDEX %s' % indexname)
commit()
elif driver.startswith('sqlserver'):
for viewname, in sql('select name from sys.views'):
if viewname.startswith('utv_'):
- print 'dropping view (index should be cascade-deleted)', viewname
+ print('dropping view (index should be cascade-deleted)', viewname)
sql('DROP VIEW %s' % viewname)
commit()
# recreate the constraints, hook will lead to low-level recreation
for eschema in sorted(schema.entities()):
if eschema._unique_together:
- print 'recreate unique indexes for', eschema
+ print('recreate unique indexes for', eschema)
rql_args = schemaserial.uniquetogether2rqls(eschema)
for rql, args in rql_args:
args['x'] = eschema.eid
@@ -243,10 +246,10 @@
for rschema in sorted(schema.relations()):
if rschema.final:
if rschema.type in fsschema:
- print 'sync perms for', rschema.type
+ print('sync perms for', rschema.type)
sync_schema_props_perms(rschema.type, syncprops=False, ask_confirm=False, commit=False)
else:
- print 'WARNING: attribute %s missing from fs schema' % rschema.type
+ print('WARNING: attribute %s missing from fs schema' % rschema.type)
commit()
if applcubicwebversion < (3, 17, 0) and cubicwebversion >= (3, 17, 0):
@@ -298,7 +301,7 @@
with hooks_control(session, session.HOOKS_ALLOW_ALL, 'integrity'):
for rschema in repo.schema.relations():
rpermsdict = permsdict.get(rschema.eid, {})
- for rdef in rschema.rdefs.itervalues():
+ for rdef in rschema.rdefs.values():
for action in rdef.ACTIONS:
actperms = []
for something in rpermsdict.get(action == 'update' and 'add' or action, ()):
--- a/misc/migration/postcreate.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/migration/postcreate.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,13 +16,19 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""cubicweb post creation script, set user's workflow"""
+from __future__ import print_function
+
+from six import text_type
+
+from cubicweb import _
+
# insert versions
create_entity('CWProperty', pkey=u'system.version.cubicweb',
- value=unicode(config.cubicweb_version()))
+ value=text_type(config.cubicweb_version()))
for cube in config.cubes():
create_entity('CWProperty', pkey=u'system.version.%s' % cube.lower(),
- value=unicode(config.cube_version(cube)))
+ value=text_type(config.cube_version(cube)))
# some entities have been added before schema entities, fix the 'is' and
# 'is_instance_of' relations
@@ -30,8 +36,8 @@
sql('INSERT INTO %s_relation '
'SELECT X.eid, ET.cw_eid FROM entities as X, cw_CWEType as ET '
'WHERE X.type=ET.cw_name AND NOT EXISTS('
- ' SELECT 1 from is_relation '
- ' WHERE eid_from=X.eid AND eid_to=ET.cw_eid)' % rtype)
+ ' SELECT 1 from %s_relation '
+ ' WHERE eid_from=X.eid AND eid_to=ET.cw_eid)' % (rtype, rtype))
# user workflow
userwf = add_workflow(_('default user workflow'), 'CWUser')
@@ -46,11 +52,11 @@
if hasattr(config, 'anonymous_user'):
anonlogin, anonpwd = config.anonymous_user()
if anonlogin == session.user.login:
- print 'you are using a manager account as anonymous user.'
- print 'Hopefully this is not a production instance...'
+ print('you are using a manager account as anonymous user.')
+ print('Hopefully this is not a production instance...')
elif anonlogin:
from cubicweb.server import create_user
- create_user(session, unicode(anonlogin), anonpwd, u'guests')
+ create_user(session, text_type(anonlogin), anonpwd, u'guests')
# need this since we already have at least one user in the database (the default admin)
for user in rql('Any X WHERE X is CWUser').entities():
--- a/misc/scripts/cwuser_ldap2system.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/scripts/cwuser_ldap2system.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
import base64
from cubicweb.server.utils import crypt_password
@@ -20,10 +22,10 @@
rset = sql("SELECT eid,type,source,extid,mtime FROM entities WHERE source!='system'", ask_confirm=False)
for eid, type, source, extid, mtime in rset:
if type != 'CWUser':
- print "don't know what to do with entity type", type
+ print("don't know what to do with entity type", type)
continue
if not source.lower().startswith('ldap'):
- print "don't know what to do with source type", source
+ print("don't know what to do with source type", source)
continue
extid = base64.decodestring(extid)
ldapinfos = [x.strip().split('=') for x in extid.split(',')]
@@ -33,7 +35,7 @@
args = dict(eid=eid, type=type, source=source, login=login,
firstname=firstname, surname=surname, mtime=mtime,
pwd=dbhelper.binary_value(crypt_password('toto')))
- print args
+ print(args)
sql(insert, args)
sql(update, args)
--- a/misc/scripts/detect_cycle.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/scripts/detect_cycle.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,9 +1,10 @@
+from __future__ import print_function
try:
rtype, = __args__
except ValueError:
- print 'USAGE: cubicweb-ctl shell <instance> detect_cycle.py -- <relation type>'
- print
+ print('USAGE: cubicweb-ctl shell <instance> detect_cycle.py -- <relation type>')
+ print()
graph = {}
for fromeid, toeid in rql('Any X,Y WHERE X %s Y' % rtype):
@@ -12,4 +13,4 @@
from logilab.common.graph import get_cycles
for cycle in get_cycles(graph):
- print 'cycle', '->'.join(str(n) for n in cycle)
+ print('cycle', '->'.join(str(n) for n in cycle))
--- a/misc/scripts/ldap_change_base_dn.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/scripts/ldap_change_base_dn.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,10 +1,12 @@
+from __future__ import print_function
+
from base64 import b64decode, b64encode
try:
uri, newdn = __args__
except ValueError:
- print 'USAGE: cubicweb-ctl shell <instance> ldap_change_base_dn.py -- <ldap source uri> <new dn>'
- print
- print 'you should not have updated your sources file yet'
+ print('USAGE: cubicweb-ctl shell <instance> ldap_change_base_dn.py -- <ldap source uri> <new dn>')
+ print()
+ print('you should not have updated your sources file yet')
olddn = repo.sources_by_uri[uri].config['user-base-dn']
@@ -16,9 +18,9 @@
olduserdn = b64decode(extid)
newuserdn = olduserdn.replace(olddn, newdn)
if newuserdn != olduserdn:
- print olduserdn, '->', newuserdn
+ print(olduserdn, '->', newuserdn)
sql("UPDATE entities SET extid='%s' WHERE eid=%s" % (b64encode(newuserdn), eid))
commit()
-print 'you can now update the sources file to the new dn and restart the instance'
+print('you can now update the sources file to the new dn and restart the instance')
--- a/misc/scripts/ldapuser2ldapfeed.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/scripts/ldapuser2ldapfeed.py Wed Dec 16 11:23:48 2015 +0100
@@ -2,6 +2,8 @@
Once this script is run, execute c-c db-check to cleanup relation tables.
"""
+from __future__ import print_function
+
import sys
from collections import defaultdict
from logilab.common.shellutils import generate_password
@@ -14,12 +16,12 @@
' on the command line)')
sys.exit(1)
except KeyError:
- print '%s is not an active source' % source_name
+ print('%s is not an active source' % source_name)
sys.exit(1)
# check source is reachable before doing anything
if not source.get_connection().cnx:
- print '%s is not reachable. Fix this before running this script' % source_name
+ print('%s is not reachable. Fix this before running this script' % source_name)
sys.exit(1)
raw_input('Ensure you have shutdown all instances of this application before continuing.'
@@ -31,7 +33,7 @@
from cubicweb.server.edition import EditedEntity
-print '******************** backport entity content ***************************'
+print('******************** backport entity content ***************************')
todelete = defaultdict(list)
extids = set()
@@ -39,17 +41,17 @@
for entity in rql('Any X WHERE X cw_source S, S eid %(s)s', {'s': source.eid}).entities():
etype = entity.cw_etype
if not source.support_entity(etype):
- print "source doesn't support %s, delete %s" % (etype, entity.eid)
+ print("source doesn't support %s, delete %s" % (etype, entity.eid))
todelete[etype].append(entity)
continue
try:
entity.complete()
except Exception:
- print '%s %s much probably deleted, delete it (extid %s)' % (
- etype, entity.eid, entity.cw_metainformation()['extid'])
+ print('%s %s much probably deleted, delete it (extid %s)' % (
+ etype, entity.eid, entity.cw_metainformation()['extid']))
todelete[etype].append(entity)
continue
- print 'get back', etype, entity.eid
+ print('get back', etype, entity.eid)
entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache)
if not entity.creation_date:
entity.cw_edited['creation_date'] = datetime.now()
@@ -61,7 +63,7 @@
if not entity.cwuri:
entity.cw_edited['cwuri'] = '%s/?dn=%s' % (
source.urls[0], extid.decode('utf-8', 'ignore'))
- print entity.cw_edited
+ print(entity.cw_edited)
if extid in extids:
duplicates.append(extid)
continue
@@ -73,13 +75,13 @@
# only cleanup entities table, remaining stuff should be cleaned by a c-c
# db-check to be run after this script
if duplicates:
- print 'found %s duplicate entries' % len(duplicates)
+ print('found %s duplicate entries' % len(duplicates))
from pprint import pprint
pprint(duplicates)
-print len(todelete), 'entities will be deleted'
-for etype, entities in todelete.iteritems():
- print 'deleting', etype, [e.login for e in entities]
+print(len(todelete), 'entities will be deleted')
+for etype, entities in todelete.items():
+ print('deleting', etype, [e.login for e in entities])
system_source.delete_info_multi(session, entities, source_name)
@@ -89,9 +91,8 @@
if raw_input('Commit?') in 'yY':
- print 'committing'
+ print('committing')
commit()
else:
rollback()
- print 'rolled back'
-
+ print('rolled back')
--- a/misc/scripts/pyroforge2datafeed.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/scripts/pyroforge2datafeed.py Wed Dec 16 11:23:48 2015 +0100
@@ -2,6 +2,8 @@
Once this script is run, execute c-c db-check to cleanup relation tables.
"""
+from __future__ import print_function
+
import sys
try:
@@ -12,14 +14,14 @@
' on the command line)')
sys.exit(1)
except KeyError:
- print '%s is not an active source' % source_name
+ print('%s is not an active source' % source_name)
sys.exit(1)
# check source is reachable before doing anything
try:
source.get_connection()._repo
except AttributeError:
- print '%s is not reachable. Fix this before running this script' % source_name
+ print('%s is not reachable. Fix this before running this script' % source_name)
sys.exit(1)
raw_input('Ensure you have shutdown all instances of this application before continuing.'
@@ -39,7 +41,7 @@
))
-print '******************** backport entity content ***************************'
+print('******************** backport entity content ***************************')
from cubicweb.server import debugged
todelete = {}
@@ -47,20 +49,20 @@
for entity in rql('Any X WHERE X cw_source S, S eid %(s)s', {'s': source.eid}).entities():
etype = entity.cw_etype
if not source.support_entity(etype):
- print "source doesn't support %s, delete %s" % (etype, entity.eid)
+ print("source doesn't support %s, delete %s" % (etype, entity.eid))
elif etype in DONT_GET_BACK_ETYPES:
- print 'ignore %s, delete %s' % (etype, entity.eid)
+ print('ignore %s, delete %s' % (etype, entity.eid))
else:
try:
entity.complete()
if not host in entity.cwuri:
- print 'SKIP foreign entity', entity.cwuri, source.config['base-url']
+ print('SKIP foreign entity', entity.cwuri, source.config['base-url'])
continue
except Exception:
- print '%s %s much probably deleted, delete it (extid %s)' % (
- etype, entity.eid, entity.cw_metainformation()['extid'])
+ print('%s %s much probably deleted, delete it (extid %s)' % (
+ etype, entity.eid, entity.cw_metainformation()['extid']))
else:
- print 'get back', etype, entity.eid
+ print('get back', etype, entity.eid)
entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache)
system_source.add_entity(session, entity)
sql("UPDATE entities SET asource=%(asource)s, source='system', extid=%(extid)s "
@@ -72,11 +74,11 @@
# only cleanup entities table, remaining stuff should be cleaned by a c-c
# db-check to be run after this script
-for entities in todelete.itervalues():
+for entities in todelete.values():
system_source.delete_info_multi(session, entities, source_name)
-print '******************** backport mapping **********************************'
+print('******************** backport mapping **********************************')
session.disable_hook_categories('cw.sources')
mapping = []
for mappart in rql('Any X,SCH WHERE X cw_schema SCH, X cw_for_source S, S eid %(s)s',
@@ -85,13 +87,13 @@
if schemaent.cw_etype != 'CWEType':
assert schemaent.cw_etype == 'CWRType'
sch = schema._eid_index[schemaent.eid]
- for rdef in sch.rdefs.itervalues():
+ for rdef in sch.rdefs.values():
if not source.support_entity(rdef.subject) \
or not source.support_entity(rdef.object):
continue
if rdef.subject in DONT_GET_BACK_ETYPES \
and rdef.object in DONT_GET_BACK_ETYPES:
- print 'dont map', rdef
+ print('dont map', rdef)
continue
if rdef.subject in DONT_GET_BACK_ETYPES:
options = u'action=link\nlinkattr=name'
@@ -105,7 +107,7 @@
roles = 'object',
else:
roles = 'subject',
- print 'map', rdef, options, roles
+ print('map', rdef, options, roles)
for role in roles:
mapping.append( (
(str(rdef.subject), str(rdef.rtype), str(rdef.object)),
--- a/misc/scripts/repair_file_1-9_migration.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/scripts/repair_file_1-9_migration.py Wed Dec 16 11:23:48 2015 +0100
@@ -4,13 +4,14 @@
* on our intranet on July 07 2010
* on our extranet on July 16 2010
"""
+from __future__ import print_function
try:
backupinstance, = __args__
except ValueError:
- print 'USAGE: cubicweb-ctl shell <instance> repair_file_1-9_migration.py -- <backup instance id>'
- print
- print 'you should restored the backup on a new instance, accessible through pyro'
+ print('USAGE: cubicweb-ctl shell <instance> repair_file_1-9_migration.py -- <backup instance id>')
+ print()
+ print('you should restored the backup on a new instance, accessible through pyro')
from cubicweb import cwconfig, dbapi
from cubicweb.server.session import hooks_control
@@ -32,20 +33,20 @@
'XX from_entity YY, YY name "File")'):
if rtype in ('is', 'is_instance_of'):
continue
- print rtype
+ print(rtype)
for feid, xeid in backupcu.execute('Any F,X WHERE F %s X, F is IN (File,Image)' % rtype):
- print 'restoring relation %s between file %s and %s' % (rtype, feid, xeid),
- print rql('SET F %s X WHERE F eid %%(f)s, X eid %%(x)s, NOT F %s X' % (rtype, rtype),
- {'f': feid, 'x': xeid})
+ print('restoring relation %s between file %s and %s' % (rtype, feid, xeid), end=' ')
+ print(rql('SET F %s X WHERE F eid %%(f)s, X eid %%(x)s, NOT F %s X' % (rtype, rtype),
+ {'f': feid, 'x': xeid}))
for rtype, in backupcu.execute('DISTINCT Any RTN WHERE X relation_type RT, RT name RTN,'
'X to_entity Y, Y name "Image", X is CWRelation, '
'EXISTS(XX is CWRelation, XX relation_type RT, '
'XX to_entity YY, YY name "File")'):
- print rtype
+ print(rtype)
for feid, xeid in backupcu.execute('Any F,X WHERE X %s F, F is IN (File,Image)' % rtype):
- print 'restoring relation %s between %s and file %s' % (rtype, xeid, feid),
- print rql('SET X %s F WHERE F eid %%(f)s, X eid %%(x)s, NOT X %s F' % (rtype, rtype),
- {'f': feid, 'x': xeid})
+ print('restoring relation %s between %s and file %s' % (rtype, xeid, feid), end=' ')
+ print(rql('SET X %s F WHERE F eid %%(f)s, X eid %%(x)s, NOT X %s F' % (rtype, rtype),
+ {'f': feid, 'x': xeid}))
commit()
--- a/misc/scripts/repair_splitbrain_ldapuser_source.py Tue Dec 15 19:05:41 2015 +0100
+++ b/misc/scripts/repair_splitbrain_ldapuser_source.py Wed Dec 16 11:23:48 2015 +0100
@@ -14,6 +14,7 @@
deciding to apply it for you. And then ADAPT it tou your needs.
"""
+from __future__ import print_function
import base64
from collections import defaultdict
@@ -28,12 +29,12 @@
' on the command line)')
sys.exit(1)
except KeyError:
- print '%s is not an active source' % source_name
+ print('%s is not an active source' % source_name)
sys.exit(1)
# check source is reachable before doing anything
if not source.get_connection().cnx:
- print '%s is not reachable. Fix this before running this script' % source_name
+ print('%s is not reachable. Fix this before running this script' % source_name)
sys.exit(1)
def find_dupes():
@@ -52,11 +53,11 @@
CWUser = schema['CWUser']
for extid, eids in dupes.items():
newest = eids.pop() # we merge everything on the newest
- print 'merging ghosts of', extid, 'into', newest
+ print('merging ghosts of', extid, 'into', newest)
# now we merge pairwise into the newest
for old in eids:
subst = {'old': old, 'new': newest}
- print ' merging', old
+ print(' merging', old)
gone_eids.append(old)
for rschema in CWUser.subject_relations():
if rschema.final or rschema == 'identity':
@@ -83,24 +84,24 @@
rollback()
return
commit() # XXX flushing operations is wanted rather than really committing
- print 'clean up entities table'
+ print('clean up entities table')
sql('DELETE FROM entities WHERE eid IN (%s)' % (', '.join(str(x) for x in gone_eids)))
commit()
def main():
dupes = find_dupes()
if not dupes:
- print 'No duplicate user'
+ print('No duplicate user')
return
- print 'Found %s duplicate user instances' % len(dupes)
+ print('Found %s duplicate user instances' % len(dupes))
while True:
- print 'Fix or dry-run? (f/d) ... or Ctrl-C to break out'
+ print('Fix or dry-run? (f/d) ... or Ctrl-C to break out')
answer = raw_input('> ')
if answer.lower() not in 'fd':
continue
- print 'Please STOP THE APPLICATION INSTANCES (service or interactive), and press Return when done.'
+ print('Please STOP THE APPLICATION INSTANCES (service or interactive), and press Return when done.')
raw_input('<I swear all running instances and workers of the application are stopped>')
with hooks_control(session, session.HOOKS_DENY_ALL):
merge_dupes(dupes, docommit=answer=='f')
--- a/multipart.py Tue Dec 15 19:05:41 2015 +0100
+++ b/multipart.py Wed Dec 16 11:23:48 2015 +0100
@@ -41,14 +41,13 @@
from wsgiref.headers import Headers
import re, sys
try:
- from urlparse import parse_qs
-except ImportError: # pragma: no cover (fallback for Python 2.5)
- from cgi import parse_qs
-try:
from io import BytesIO
except ImportError: # pragma: no cover (fallback for Python 2.5)
from StringIO import StringIO as BytesIO
+from six import PY3, text_type
+from six.moves.urllib.parse import parse_qs
+
##############################################################################
################################ Helper & Misc ################################
##############################################################################
@@ -63,7 +62,7 @@
""" A dict that remembers old values for each key """
def __init__(self, *a, **k):
self.dict = dict()
- for k, v in dict(*a, **k).iteritems():
+ for k, v in dict(*a, **k).items():
self[k] = v
def __len__(self): return len(self.dict)
@@ -84,12 +83,12 @@
return self.dict[key][index]
def iterallitems(self):
- for key, values in self.dict.iteritems():
+ for key, values in self.dict.items():
for value in values:
yield key, value
def tob(data, enc='utf8'): # Convert strings to bytes (py2 and py3)
- return data.encode(enc) if isinstance(data, unicode) else data
+ return data.encode(enc) if isinstance(data, text_type) else data
def copy_file(stream, target, maxread=-1, buffer_size=2*16):
''' Read from :stream and write to :target until :maxread or EOF. '''
@@ -397,17 +396,21 @@
'application/x-url-encoded'):
mem_limit = kw.get('mem_limit', 2**20)
if content_length > mem_limit:
- raise MultipartError("Request to big. Increase MAXMEM.")
+ raise MultipartError("Request too big. Increase MAXMEM.")
data = stream.read(mem_limit)
if stream.read(1): # These is more that does not fit mem_limit
- raise MultipartError("Request to big. Increase MAXMEM.")
+ raise MultipartError("Request too big. Increase MAXMEM.")
+ if PY3:
+ data = data.decode('ascii')
data = parse_qs(data, keep_blank_values=True)
- for key, values in data.iteritems():
+ for key, values in data.items():
for value in values:
- forms[key] = value.decode(charset)
+ if PY3:
+ forms[key] = value
+ else:
+ forms[key.decode(charset)] = value.decode(charset)
else:
raise MultipartError("Unsupported content type.")
except MultipartError:
if strict: raise
return forms, files
-
--- a/predicates.py Tue Dec 15 19:05:41 2015 +0100
+++ b/predicates.py Wed Dec 16 11:23:48 2015 +0100
@@ -24,6 +24,9 @@
from warnings import warn
from operator import eq
+from six import string_types, integer_types
+from six.moves import range
+
from logilab.common.deprecation import deprecated
from logilab.common.registry import Predicate, objectify_predicate, yes
@@ -106,7 +109,7 @@
if accept_none is None:
accept_none = self.accept_none
if not accept_none and \
- any(rset[i][col] is None for i in xrange(len(rset))):
+ any(row[col] is None for row in rset):
return 0
etypes = rset.column_types(col)
else:
@@ -332,7 +335,7 @@
# on rset containing several entity types, each row may be
# individually adaptable, while the whole rset won't be if the
# same adapter can't be used for each type
- for row in xrange(len(kwargs['rset'])):
+ for row in range(len(kwargs['rset'])):
kwargs.setdefault('col', 0)
_score = super(adaptable, self).__call__(cls, req, row=row, **kwargs)
if not _score:
@@ -489,10 +492,13 @@
page_size = kwargs.get('page_size')
if page_size is None:
page_size = req.form.get('page_size')
+ if page_size is not None:
+ try:
+ page_size = int(page_size)
+ except ValueError:
+ page_size = None
if page_size is None:
page_size = req.property_value('navigation.page-size')
- else:
- page_size = int(page_size)
if len(rset) <= (page_size*self.nbpages):
return 0
return self.nbpages
@@ -611,7 +617,7 @@
super(is_instance, self).__init__(**kwargs)
self.expected_etypes = expected_etypes
for etype in self.expected_etypes:
- assert isinstance(etype, basestring), etype
+ assert isinstance(etype, string_types), etype
def __str__(self):
return '%s(%s)' % (self.__class__.__name__,
@@ -671,7 +677,7 @@
score = scorefunc(*args, **kwargs)
if not score:
return 0
- if isinstance(score, (int, long)):
+ if isinstance(score, integer_types):
return score
return 1
self.score_entity = intscore
@@ -828,7 +834,7 @@
class has_related_entities(EntityPredicate):
"""Return 1 if entity support the specified relation and has some linked
- entities by this relation , optionaly filtered according to the specified
+ entities by this relation , optionally filtered according to the specified
target type.
The relation is specified by the following initializer arguments:
@@ -1091,7 +1097,7 @@
"""
if from_state_name is not None:
warn("on_fire_transition's from_state_name argument is unused", DeprecationWarning)
- if isinstance(tr_names, basestring):
+ if isinstance(tr_names, string_types):
tr_names = set((tr_names,))
def match_etype_and_transition(trinfo):
# take care trinfo.transition is None when calling change_state
@@ -1291,7 +1297,7 @@
raise ValueError("match_form_params() can't be called with both "
"positional and named arguments")
if expected:
- if len(expected) == 1 and not isinstance(expected[0], basestring):
+ if len(expected) == 1 and not isinstance(expected[0], string_types):
raise ValueError("match_form_params() positional arguments "
"must be strings")
super(match_form_params, self).__init__(*expected)
--- a/pylintext.py Tue Dec 15 19:05:41 2015 +0100
+++ b/pylintext.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,7 +17,7 @@
def cubicweb_transform(module):
# handle objectify_predicate decorator (and its former name until bw compat
# is kept). Only look at module level functions, should be enough.
- for assnodes in module.locals.itervalues():
+ for assnodes in module.locals.values():
for node in assnodes:
if isinstance(node, scoped_nodes.Function) and node.decorators:
for decorator in node.decorators.nodes:
@@ -48,4 +48,3 @@
def register(linter):
"""called when loaded by pylint --load-plugins, nothing to do here"""
MANAGER.register_transform(nodes.Module, cubicweb_transform)
-
--- a/repoapi.py Tue Dec 15 19:05:41 2015 +0100
+++ b/repoapi.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,21 +17,17 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""Official API to access the content of a repository
"""
+from warnings import warn
+
+from six import add_metaclass
+
from logilab.common.deprecation import class_deprecated
from cubicweb.utils import parse_repo_uri
-from cubicweb import ConnectionError, AuthenticationError
+from cubicweb import AuthenticationError
from cubicweb.server.session import Connection
-### private function for specific method ############################
-
-def _get_inmemory_repo(config, vreg=None):
- from cubicweb.server.repository import Repository
- from cubicweb.server.utils import TasksManager
- return Repository(config, TasksManager(), vreg=vreg)
-
-
### public API ######################################################
def get_repository(uri=None, config=None, vreg=None):
@@ -41,16 +37,11 @@
The returned repository may be an in-memory repository or a proxy object
using a specific RPC method, depending on the given URI.
"""
- if uri is None:
- return _get_inmemory_repo(config, vreg)
-
- protocol, hostport, appid = parse_repo_uri(uri)
+ if uri is not None:
+ warn('[3.22] get_repository only wants a config')
- if protocol == 'inmemory':
- # me may have been called with a dummy 'inmemory://' uri ...
- return _get_inmemory_repo(config, vreg)
-
- raise ConnectionError('unknown protocol: `%s`' % protocol)
+ assert config is not None, 'get_repository(config=config)'
+ return config.repository(vreg)
def connect(repo, login, **kwargs):
"""Take credential and return associated Connection.
@@ -75,6 +66,6 @@
return connect(repo, anon_login, password=anon_password)
+@add_metaclass(class_deprecated)
class ClientConnection(Connection):
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.20] %(cls)s is deprecated, use Connection instead'
--- a/req.py Tue Dec 15 19:05:41 2015 +0100
+++ b/req.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,10 +20,10 @@
__docformat__ = "restructuredtext en"
from warnings import warn
-from urlparse import urlsplit, urlunsplit
-from urllib import quote as urlquote, unquote as urlunquote
from datetime import time, datetime, timedelta
-from cgi import parse_qs, parse_qsl
+
+from six import PY2, PY3, text_type
+from six.moves.urllib.parse import parse_qs, parse_qsl, quote as urlquote, unquote as urlunquote, urlsplit, urlunsplit
from logilab.common.decorators import cached
from logilab.common.deprecation import deprecated
@@ -73,7 +73,7 @@
# connection
self.user = None
self.local_perm_cache = {}
- self._ = unicode
+ self._ = text_type
def _set_user(self, orig_user):
"""set the user for this req_session_base
@@ -219,7 +219,7 @@
parts.append(
'%(varname)s %(attr)s X, '
'%(varname)s eid %%(reverse_%(attr)s)s'
- % {'attr': attr, 'varname': varmaker.next()})
+ % {'attr': attr, 'varname': next(varmaker)})
else:
assert attr in eschema.subjrels, \
'%s not in %s subject relations' % (attr, eschema)
@@ -300,7 +300,7 @@
def build_url_params(self, **kwargs):
"""return encoded params to incorporate them in a URL"""
args = []
- for param, values in kwargs.iteritems():
+ for param, values in kwargs.items():
if not isinstance(values, (list, tuple)):
values = (values,)
for value in values:
@@ -313,7 +313,7 @@
necessary encoding / decoding. Also it's designed to quote each
part of a url path and so the '/' character will be encoded as well.
"""
- if isinstance(value, unicode):
+ if PY2 and isinstance(value, unicode):
quoted = urlquote(value.encode(self.encoding), safe=safe)
return unicode(quoted, self.encoding)
return urlquote(str(value), safe=safe)
@@ -324,6 +324,8 @@
decoding is based on `self.encoding` which is the encoding
used in `url_quote`
"""
+ if PY3:
+ return urlunquote(quoted)
if isinstance(quoted, unicode):
quoted = quoted.encode(self.encoding)
try:
@@ -333,6 +335,10 @@
def url_parse_qsl(self, querystring):
"""return a list of (key, val) found in the url quoted query string"""
+ if PY3:
+ for key, val in parse_qsl(querystring):
+ yield key, val
+ return
if isinstance(querystring, unicode):
querystring = querystring.encode(self.encoding)
for key, val in parse_qsl(querystring):
@@ -348,12 +354,12 @@
newparams may only be mono-valued.
"""
- if isinstance(url, unicode):
+ if PY2 and isinstance(url, unicode):
url = url.encode(self.encoding)
schema, netloc, path, query, fragment = urlsplit(url)
query = parse_qs(query)
# sort for testing predictability
- for key, val in sorted(newparams.iteritems()):
+ for key, val in sorted(newparams.items()):
query[key] = (self.url_quote(val),)
query = '&'.join(u'%s=%s' % (param, value)
for param, values in sorted(query.items())
--- a/rqlrewrite.py Tue Dec 15 19:05:41 2015 +0100
+++ b/rqlrewrite.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,6 +22,8 @@
"""
__docformat__ = "restructuredtext en"
+from six import text_type, string_types
+
from rql import nodes as n, stmts, TypeResolverException
from rql.utils import common_parent
@@ -54,7 +56,7 @@
eschema = schema.eschema
allpossibletypes = {}
for solution in solutions:
- for varname, etype in solution.iteritems():
+ for varname, etype in solution.items():
# XXX not considering aliases by design, right ?
if varname not in newroot.defined_vars or eschema(etype).final:
continue
@@ -92,7 +94,7 @@
for etype in sorted(possibletypes):
node.append(n.Constant(etype, 'etype'))
else:
- etype = iter(possibletypes).next()
+ etype = next(iter(possibletypes))
node = n.Constant(etype, 'etype')
comp = mytyperel.children[1]
comp.replace(comp.children[0], node)
@@ -286,7 +288,7 @@
if fnode.name == 'FTIRANK':
# we've to fetch the has_text relation as well
var = fnode.children[0].variable
- rel = iter(var.stinfo['ftirels']).next()
+ rel = next(iter(var.stinfo['ftirels']))
assert not rel.ored(), 'unsupported'
newselect.add_restriction(rel.copy(newselect))
# remove relation from the orig select and
@@ -330,7 +332,7 @@
union.replace(select, newselect)
elif not () in localchecks:
union.remove(select)
- for lcheckdef, lchecksolutions in localchecks.iteritems():
+ for lcheckdef, lchecksolutions in localchecks.items():
if not lcheckdef:
continue
myrqlst = select.copy(solutions=lchecksolutions)
@@ -427,7 +429,7 @@
def insert_varmap_snippets(self, varmap, rqlexprs, varexistsmap):
try:
self.init_from_varmap(varmap, varexistsmap)
- except VariableFromSubQuery, ex:
+ except VariableFromSubQuery as ex:
# variable may have been moved to a newly inserted subquery
# we should insert snippet in that subquery
subquery = self.select.aliases[ex.variable].query
@@ -548,7 +550,7 @@
'cant check security of %s, ambigous type for %s in %s',
stmt, varname, key[0]) # key[0] == the rql expression
raise Unauthorized()
- etype = iter(ptypes).next()
+ etype = next(iter(ptypes))
eschema = self.schema.eschema(etype)
if not eschema.has_perm(self.session, action):
rqlexprs = eschema.get_rqlexprs(action)
@@ -621,7 +623,7 @@
while argname in self.kwargs:
argname = subselect.allocate_varname()
subselect.add_constant_restriction(subselect.get_variable(self.u_varname),
- 'eid', unicode(argname), 'Substitute')
+ 'eid', text_type(argname), 'Substitute')
self.kwargs[argname] = self.session.user.eid
add_types_restriction(self.schema, subselect, subselect,
solutions=self.solutions)
@@ -646,7 +648,7 @@
# insert "is" where necessary
varexistsmap = {}
self.removing_ambiguity = True
- for (erqlexpr, varmap, oldvarname), etype in variantes[0].iteritems():
+ for (erqlexpr, varmap, oldvarname), etype in variantes[0].items():
varname = self.rewritten[(erqlexpr, varmap, oldvarname)]
var = self.select.defined_vars[varname]
exists = var.references()[0].scope
@@ -655,7 +657,7 @@
# insert ORED exists where necessary
for variante in variantes[1:]:
self.insert_snippets(snippets, varexistsmap)
- for key, etype in variante.iteritems():
+ for key, etype in variante.items():
varname = self.rewritten[key]
try:
var = self.select.defined_vars[varname]
@@ -674,7 +676,7 @@
variantes = set()
for sol in newsolutions:
variante = []
- for key, newvar in self.rewritten.iteritems():
+ for key, newvar in self.rewritten.items():
variante.append( (key, sol[newvar]) )
variantes.add(tuple(variante))
# rebuild variantes as dict
@@ -682,7 +684,7 @@
# remove variable which have always the same type
for key in self.rewritten:
it = iter(variantes)
- etype = it.next()[key]
+ etype = next(it)[key]
for variante in it:
if variante[key] != etype:
break
@@ -700,7 +702,7 @@
# no more references, undefine the variable
del self.select.defined_vars[vref.name]
removed.add(vref.name)
- for key, newvar in self.rewritten.items(): # I mean items we alter it
+ for key, newvar in list(self.rewritten.items()):
if newvar in removed:
del self.rewritten[key]
@@ -760,7 +762,7 @@
# insert "U eid %(u)s"
stmt.add_constant_restriction(
stmt.get_variable(self.u_varname),
- 'eid', unicode(argname), 'Substitute')
+ 'eid', text_type(argname), 'Substitute')
self.kwargs[argname] = self.session.user.eid
return self.u_varname
key = (self.current_expr, self.varmap, vname)
@@ -883,7 +885,7 @@
return n.Constant(vi['const'], 'Int')
return n.VariableRef(stmt.get_variable(selectvar))
vname_or_term = self._get_varname_or_term(node.name)
- if isinstance(vname_or_term, basestring):
+ if isinstance(vname_or_term, string_types):
return n.VariableRef(stmt.get_variable(vname_or_term))
# shared term
return vname_or_term.copy(stmt)
--- a/rset.py Tue Dec 15 19:05:41 2015 +0100
+++ b/rset.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,11 +16,13 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""The `ResultSet` class which is returned as result of an rql query"""
-
__docformat__ = "restructuredtext en"
from warnings import warn
+from six import PY3
+from six.moves import range
+
from logilab.common import nullobject
from logilab.common.decorators import cached, clear_cache, copy_cache
from rql import nodes, stmts
@@ -101,7 +103,7 @@
if self._rsetactions is None:
self._rsetactions = {}
if kwargs:
- key = tuple(sorted(kwargs.iteritems()))
+ key = tuple(sorted(kwargs.items()))
else:
key = None
try:
@@ -120,10 +122,6 @@
"""returns the ith element of the result set"""
return self.rows[i] #ResultSetRow(self.rows[i])
- def __getslice__(self, i, j):
- """returns slice [i:j] of the result set"""
- return self.rows[i:j]
-
def __iter__(self):
"""Returns an iterator over rows"""
return iter(self.rows)
@@ -186,7 +184,7 @@
"""
rows, descr = [], []
rset = self.copy(rows, descr)
- for i in xrange(len(self)):
+ for i in range(len(self)):
if not filtercb(self.get_entity(i, col)):
continue
rows.append(self.rows[i])
@@ -215,10 +213,10 @@
rset = self.copy(rows, descr)
if col >= 0:
entities = sorted(enumerate(self.entities(col)),
- key=lambda (i, e): keyfunc(e), reverse=reverse)
+ key=lambda t: keyfunc(t[1]), reverse=reverse)
else:
entities = sorted(enumerate(self),
- key=lambda (i, e): keyfunc(e), reverse=reverse)
+ key=lambda t: keyfunc(t[1]), reverse=reverse)
for index, _ in entities:
rows.append(self.rows[index])
descr.append(self.description[index])
@@ -311,7 +309,7 @@
newselect.limit = limit
newselect.offset = offset
aliases = [nodes.VariableRef(newselect.get_variable(chr(65+i), i))
- for i in xrange(len(rqlst.children[0].selection))]
+ for i in range(len(rqlst.children[0].selection))]
for vref in aliases:
newselect.append_selected(nodes.VariableRef(vref.variable))
newselect.set_with([nodes.SubQuery(aliases, rqlst)], check=False)
@@ -322,7 +320,7 @@
return rql
def limit(self, limit, offset=0, inplace=False):
- """limit the result set to the given number of rows optionaly starting
+ """limit the result set to the given number of rows optionally starting
from an index different than 0
:type limit: int
@@ -373,6 +371,8 @@
warn('[3.21] the "encoded" argument is deprecated', DeprecationWarning)
encoding = self.req.encoding
rqlstr = self.syntax_tree().as_string(kwargs=self.args)
+ if PY3:
+ return rqlstr
# sounds like we get encoded or unicode string due to a bug in as_string
if not encoded:
if isinstance(rqlstr, unicode):
@@ -387,7 +387,7 @@
def entities(self, col=0):
"""iter on entities with eid in the `col` column of the result set"""
- for i in xrange(len(self)):
+ for i in range(len(self)):
# may have None values in case of outer join (or aggregat on eid
# hacks)
if self.rows[i][col] is not None:
@@ -507,9 +507,9 @@
eschema = entity.e_schema
eid_col, attr_cols, rel_cols = self._rset_structure(eschema, col)
entity.eid = rowvalues[eid_col]
- for attr, col_idx in attr_cols.iteritems():
+ for attr, col_idx in attr_cols.items():
entity.cw_attr_cache[attr] = rowvalues[col_idx]
- for (rtype, role), col_idx in rel_cols.iteritems():
+ for (rtype, role), col_idx in rel_cols.items():
value = rowvalues[col_idx]
if value is None:
if role == 'subject':
@@ -606,7 +606,7 @@
except AttributeError:
# not a variable
continue
- for i in xrange(len(select.selection)):
+ for i in range(len(select.selection)):
if i == col:
continue
coletype = self.description[row][i]
--- a/rtags.py Tue Dec 15 19:05:41 2015 +0100
+++ b/rtags.py Wed Dec 16 11:23:48 2015 +0100
@@ -40,6 +40,8 @@
import logging
from warnings import warn
+from six import string_types
+
from logilab.common.logging_ext import set_log_methods
from logilab.common.registry import RegistrableInstance, yes
@@ -95,7 +97,7 @@
def init(self, schema, check=True):
# XXX check existing keys against schema
if check:
- for (stype, rtype, otype, tagged), value in self._tagdefs.items():
+ for (stype, rtype, otype, tagged), value in list(self._tagdefs.items()):
for ertype in (stype, rtype, otype):
if ertype != '*' and not ertype in schema:
self.warning('removing rtag %s: %s, %s undefined in schema',
@@ -145,7 +147,7 @@
return tag
def _tag_etype_attr(self, etype, attr, desttype='*', *args, **kwargs):
- if isinstance(attr, basestring):
+ if isinstance(attr, string_types):
attr, role = attr, 'subject'
else:
attr, role = attr
--- a/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,15 +16,18 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""classes to define schemas for CubicWeb"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
-_ = unicode
import re
from os.path import join, basename
from logging import getLogger
from warnings import warn
+from six import PY2, text_type, string_types, add_metaclass
+from six.moves import range
+
from logilab.common import tempattr
from logilab.common.decorators import cached, clear_cache, monkeypatch, cachedproperty
from logilab.common.logging_ext import set_log_methods
@@ -45,7 +48,7 @@
from rql.analyze import ETypeResolver
import cubicweb
-from cubicweb import ETYPE_NAME_MAP, ValidationError, Unauthorized
+from cubicweb import ETYPE_NAME_MAP, ValidationError, Unauthorized, _
try:
from cubicweb import server
@@ -102,6 +105,9 @@
INTERNAL_TYPES = set(('CWProperty', 'CWCache', 'ExternalUri', 'CWDataImport',
'CWSource', 'CWSourceHostConfig', 'CWSourceSchemaConfig'))
+UNIQUE_CONSTRAINTS = ('SizeConstraint', 'FormatConstraint',
+ 'StaticVocabularyConstraint',
+ 'RQLVocabularyConstraint')
_LOGGER = getLogger('cubicweb.schemaloader')
@@ -142,7 +148,10 @@
suppressing and reinserting an expression if only a space has been
added/removed for instance)
"""
- return u', '.join(' '.join(expr.split()) for expr in rqlstring.split(','))
+ union = parse(u'Any 1 WHERE %s' % rqlstring).as_string()
+ if PY2 and isinstance(union, str):
+ union = union.decode('utf-8')
+ return union.split(' WHERE ', 1)[1]
def _check_valid_formula(rdef, formula_rqlst):
@@ -204,7 +213,7 @@
"""
self.eid = eid # eid of the entity representing this rql expression
assert mainvars, 'bad mainvars %s' % mainvars
- if isinstance(mainvars, basestring):
+ if isinstance(mainvars, string_types):
mainvars = set(splitstrip(mainvars))
elif not isinstance(mainvars, set):
mainvars = set(mainvars)
@@ -246,6 +255,9 @@
return self.expression == other.expression
return False
+ def __ne__(self, other):
+ return not (self == other)
+
def __hash__(self):
return hash(self.expression)
@@ -271,7 +283,7 @@
def transform_has_permission(self):
found = None
rqlst = self.rqlst
- for var in rqlst.defined_vars.itervalues():
+ for var in rqlst.defined_vars.values():
for varref in var.references():
rel = varref.relation()
if rel is None:
@@ -319,7 +331,7 @@
"""
creating = kwargs.get('creating')
if not creating and self.eid is not None:
- key = (self.eid, tuple(sorted(kwargs.iteritems())))
+ key = (self.eid, tuple(sorted(kwargs.items())))
try:
return _cw.local_perm_cache[key]
except KeyError:
@@ -363,7 +375,7 @@
get_eschema = _cw.vreg.schema.eschema
try:
for eaction, col in has_perm_defs:
- for i in xrange(len(rset)):
+ for i in range(len(rset)):
eschema = get_eschema(rset.description[i][col])
eschema.check_perm(_cw, eaction, eid=rset[i][col])
if self.eid is not None:
@@ -400,13 +412,35 @@
return self._check(_cw, x=eid, **kwargs)
return self._check(_cw, **kwargs)
-def constraint_by_eid(self, eid):
- for cstr in self.constraints:
- if cstr.eid == eid:
- return cstr
- raise ValueError('No constraint with eid %d' % eid)
-RelationDefinitionSchema.constraint_by_eid = constraint_by_eid
+
+class CubicWebRelationDefinitionSchema(RelationDefinitionSchema):
+ def constraint_by_eid(self, eid):
+ for cstr in self.constraints:
+ if cstr.eid == eid:
+ return cstr
+ raise ValueError('No constraint with eid %d' % eid)
+
+ def rql_expression(self, expression, mainvars=None, eid=None):
+ """rql expression factory"""
+ if self.rtype.final:
+ return ERQLExpression(expression, mainvars, eid)
+ return RRQLExpression(expression, mainvars, eid)
+ def check_permission_definitions(self):
+ super(CubicWebRelationDefinitionSchema, self).check_permission_definitions()
+ schema = self.subject.schema
+ for action, groups in self.permissions.items():
+ for group_or_rqlexpr in groups:
+ if action == 'read' and \
+ isinstance(group_or_rqlexpr, RQLExpression):
+ msg = "can't use rql expression for read permission of %s"
+ raise BadSchemaDefinition(msg % self)
+ if self.final and isinstance(group_or_rqlexpr, RRQLExpression):
+ msg = "can't use RRQLExpression on %s, use an ERQLExpression"
+ raise BadSchemaDefinition(msg % self)
+ if not self.final and isinstance(group_or_rqlexpr, ERQLExpression):
+ msg = "can't use ERQLExpression on %s, use a RRQLExpression"
+ raise BadSchemaDefinition(msg % self)
def vargraph(rqlst):
""" builds an adjacency graph of variables from the rql syntax tree, e.g:
@@ -522,7 +556,7 @@
if not deps:
eschemas.append(eschema)
del graph[eschema]
- for deps in graph.itervalues():
+ for deps in graph.values():
try:
deps.remove(eschema)
except KeyError:
@@ -548,9 +582,9 @@
key = key + '_' + form
# ensure unicode
if context is not None:
- return unicode(req.pgettext(context, key))
+ return text_type(req.pgettext(context, key))
else:
- return unicode(req._(key))
+ return text_type(req._(key))
# Schema objects definition ###################################################
@@ -576,7 +610,7 @@
assert action in self.ACTIONS, action
#assert action in self._groups, '%s %s' % (self, action)
try:
- return frozenset(g for g in self.permissions[action] if isinstance(g, basestring))
+ return frozenset(g for g in self.permissions[action] if isinstance(g, string_types))
except KeyError:
return ()
PermissionMixIn.get_groups = get_groups
@@ -595,7 +629,7 @@
assert action in self.ACTIONS, action
#assert action in self._rqlexprs, '%s %s' % (self, action)
try:
- return tuple(g for g in self.permissions[action] if not isinstance(g, basestring))
+ return tuple(g for g in self.permissions[action] if not isinstance(g, string_types))
except KeyError:
return ()
PermissionMixIn.get_rqlexprs = get_rqlexprs
@@ -665,7 +699,7 @@
groups = self.get_groups(action)
if _cw.user.matching_groups(groups):
if DBG:
- print ('check_perm: %r %r: user matches %s' % (action, _self_str, groups))
+ print('check_perm: %r %r: user matches %s' % (action, _self_str, groups))
return
# if 'owners' in allowed groups, check if the user actually owns this
# object, if so that's enough
@@ -676,14 +710,14 @@
kwargs.get('creating')
or ('eid' in kwargs and _cw.user.owns(kwargs['eid']))):
if DBG:
- print ('check_perm: %r %r: user is owner or creation time' %
- (action, _self_str))
+ print('check_perm: %r %r: user is owner or creation time' %
+ (action, _self_str))
return
# else if there is some rql expressions, check them
if DBG:
- print ('check_perm: %r %r %s' %
- (action, _self_str, [(rqlexpr, kwargs, rqlexpr.check(_cw, **kwargs))
- for rqlexpr in self.get_rqlexprs(action)]))
+ print('check_perm: %r %r %s' %
+ (action, _self_str, [(rqlexpr, kwargs, rqlexpr.check(_cw, **kwargs))
+ for rqlexpr in self.get_rqlexprs(action)]))
if any(rqlexpr.check(_cw, **kwargs)
for rqlexpr in self.get_rqlexprs(action)):
return
@@ -691,35 +725,10 @@
PermissionMixIn.check_perm = check_perm
-RelationDefinitionSchema._RPROPERTIES['eid'] = None
+CubicWebRelationDefinitionSchema._RPROPERTIES['eid'] = None
# remember rproperties defined at this point. Others will have to be serialized in
# CWAttribute.extra_props
-KNOWN_RPROPERTIES = RelationDefinitionSchema.ALL_PROPERTIES()
-
-def rql_expression(self, expression, mainvars=None, eid=None):
- """rql expression factory"""
- if self.rtype.final:
- return ERQLExpression(expression, mainvars, eid)
- return RRQLExpression(expression, mainvars, eid)
-RelationDefinitionSchema.rql_expression = rql_expression
-
-orig_check_permission_definitions = RelationDefinitionSchema.check_permission_definitions
-def check_permission_definitions(self):
- orig_check_permission_definitions(self)
- schema = self.subject.schema
- for action, groups in self.permissions.iteritems():
- for group_or_rqlexpr in groups:
- if action == 'read' and \
- isinstance(group_or_rqlexpr, RQLExpression):
- msg = "can't use rql expression for read permission of %s"
- raise BadSchemaDefinition(msg % self)
- if self.final and isinstance(group_or_rqlexpr, RRQLExpression):
- msg = "can't use RRQLExpression on %s, use an ERQLExpression"
- raise BadSchemaDefinition(msg % self)
- if not self.final and isinstance(group_or_rqlexpr, ERQLExpression):
- msg = "can't use ERQLExpression on %s, use a RRQLExpression"
- raise BadSchemaDefinition(msg % self)
-RelationDefinitionSchema.check_permission_definitions = check_permission_definitions
+KNOWN_RPROPERTIES = CubicWebRelationDefinitionSchema.ALL_PROPERTIES()
class CubicWebEntitySchema(EntitySchema):
@@ -763,7 +772,7 @@
def check_permission_definitions(self):
super(CubicWebEntitySchema, self).check_permission_definitions()
- for groups in self.permissions.itervalues():
+ for groups in self.permissions.values():
for group_or_rqlexpr in groups:
if isinstance(group_or_rqlexpr, RRQLExpression):
msg = "can't use RRQLExpression on %s, use an ERQLExpression"
@@ -870,6 +879,7 @@
class CubicWebRelationSchema(PermissionMixIn, RelationSchema):
permissions = {}
ACTIONS = ()
+ rdef_class = CubicWebRelationDefinitionSchema
def __init__(self, schema=None, rdef=None, eid=None, **kwargs):
if rdef is not None:
@@ -906,7 +916,7 @@
if rdef.may_have_permission(action, req):
return True
else:
- for rdef in self.rdefs.itervalues():
+ for rdef in self.rdefs.values():
if rdef.may_have_permission(action, req):
return True
return False
@@ -948,7 +958,7 @@
if not rdef.has_perm(_cw, action, **kwargs):
return False
else:
- for rdef in self.rdefs.itervalues():
+ for rdef in self.rdefs.values():
if not rdef.has_perm(_cw, action, **kwargs):
return False
return True
@@ -986,7 +996,7 @@
etype_name_re = r'[A-Z][A-Za-z0-9]*[a-z]+[A-Za-z0-9]*$'
def add_entity_type(self, edef):
- edef.name = edef.name.encode()
+ edef.name = str(edef.name)
edef.name = bw_normalize_etype(edef.name)
if not re.match(self.etype_name_re, edef.name):
raise BadSchemaDefinition(
@@ -1011,7 +1021,7 @@
raise BadSchemaDefinition(
'%r is not a valid name for a relation type. It should be '
'lower cased' % rdef.name)
- rdef.name = rdef.name.encode()
+ rdef.name = str(rdef.name)
rschema = super(CubicWebSchema, self).add_relation_type(rdef)
self._eid_index[rschema.eid] = rschema
return rschema
@@ -1071,7 +1081,7 @@
def iter_computed_attributes(self):
for relation in self.relations():
- for rdef in relation.rdefs.itervalues():
+ for rdef in relation.rdefs.values():
if rdef.final and rdef.formula is not None:
yield rdef
@@ -1198,11 +1208,11 @@
return ';%s;%s\n%s' % (','.join(sorted(self.mainvars)), self.expression,
self.msg or '')
+ @classmethod
def deserialize(cls, value):
value, msg = value.split('\n', 1)
_, mainvars, expression = value.split(';', 2)
return cls(expression, mainvars, msg)
- deserialize = classmethod(deserialize)
def repo_check(self, session, eidfrom, rtype, eidto=None):
"""raise ValidationError if the relation doesn't satisfy the constraint
@@ -1245,7 +1255,7 @@
return _cw.execute(rql, args, build_descr=False)
-class RQLConstraint(RepoEnforcedRQLConstraintMixIn, RQLVocabularyConstraint):
+class RQLConstraint(RepoEnforcedRQLConstraintMixIn, BaseRQLConstraint):
"""the rql constraint is similar to the RQLVocabularyConstraint but
are also enforced at the repository level
"""
@@ -1287,12 +1297,13 @@
make_workflowable(cls)
return cls
+
+@add_metaclass(workflowable_definition)
class WorkflowableEntityType(ybo.EntityType):
"""Use this base class instead of :class:`EntityType` to have workflow
relations (i.e. `in_state`, `wf_info_for` and `custom_workflow`) on your
entity type.
"""
- __metaclass__ = workflowable_definition
__abstract__ = True
--- a/schemas/Bookmark.py Tue Dec 15 19:05:41 2015 +0100
+++ b/schemas/Bookmark.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,7 +19,7 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from yams.buildobjs import EntityType, RelationType, SubjectRelation, String
from cubicweb.schema import RRQLExpression
--- a/schemas/base.py Tue Dec 15 19:05:41 2015 +0100
+++ b/schemas/base.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""core CubicWeb schema, but not necessary at bootstrap time"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from yams.buildobjs import (EntityType, RelationType, RelationDefinition,
SubjectRelation,
@@ -379,5 +379,3 @@
'add': ('managers', RRQLExpression('U has_update_permission S'),),
'delete': ('managers', RRQLExpression('U has_update_permission S'),),
}
-
-
--- a/schemas/bootstrap.py Tue Dec 15 19:05:41 2015 +0100
+++ b/schemas/bootstrap.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,7 +19,7 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from yams.buildobjs import (EntityType, RelationType, RelationDefinition, Bytes,
SubjectRelation, RichString, String, Boolean, Int)
--- a/schemas/workflow.py Tue Dec 15 19:05:41 2015 +0100
+++ b/schemas/workflow.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,7 +19,7 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from yams.buildobjs import (EntityType, RelationType, RelationDefinition,
SubjectRelation,
@@ -273,7 +273,7 @@
"""indicate the current state of an entity"""
__permissions__ = RO_REL_PERMS
- # not inlined intentionnaly since when using ldap sources, user'state
+ # not inlined intentionnally since when using ldap sources, user'state
# has to be stored outside the CWUser table
inlined = False
--- a/selectors.py Tue Dec 15 19:05:41 2015 +0100
+++ b/selectors.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,6 +18,8 @@
from warnings import warn
+from six import string_types
+
from logilab.common.deprecation import deprecated, class_renamed
from cubicweb.predicates import *
@@ -84,7 +86,7 @@
See `EntityPredicate` documentation for behaviour when row is not specified.
- :param *etypes: entity types (`basestring`) which should be refused
+ :param *etypes: entity types (`string_types`) which should be refused
"""
def __init__(self, *etypes):
super(_but_etype, self).__init__()
--- a/server/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,6 +20,7 @@
The server module contains functions to initialize a new repository.
"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -28,6 +29,9 @@
from glob import glob
from contextlib import contextmanager
+from six import text_type, string_types
+from six.moves import filter
+
from logilab.common.modutils import LazyObject
from logilab.common.textutils import splitstrip
from logilab.common.registry import yes
@@ -138,7 +142,7 @@
if not debugmode:
DEBUG = 0
return
- if isinstance(debugmode, basestring):
+ if isinstance(debugmode, string_types):
for mode in splitstrip(debugmode, sep='|'):
DEBUG |= globals()[mode]
else:
@@ -196,7 +200,7 @@
user = session.create_entity('CWUser', login=login, upassword=pwd)
for group in groups:
session.execute('SET U in_group G WHERE U eid %(u)s, G name %(group)s',
- {'u': user.eid, 'group': unicode(group)})
+ {'u': user.eid, 'group': text_type(group)})
return user
def init_repository(config, interactive=True, drop=False, vreg=None,
@@ -225,67 +229,72 @@
sourcescfg = config.read_sources_file()
source = sourcescfg['system']
driver = source['db-driver']
- sqlcnx = repo.system_source.get_connection()
- sqlcursor = sqlcnx.cursor()
- execute = sqlcursor.execute
- if drop:
- helper = database.get_db_helper(driver)
- dropsql = sql_drop_all_user_tables(helper, sqlcursor)
- # We may fail dropping some tables because of table dependencies, in a first pass.
- # So, we try a second drop sequence to drop remaining tables if needed.
- # Note that 2 passes is an arbitrary choice as it seems enougth for our usecases.
- # (looping may induce infinite recursion when user have no right for example)
- # Here we try to keep code simple and backend independant. That why we don't try to
- # distinguish remaining tables (wrong right, dependencies, ...).
- failed = sqlexec(dropsql, execute, cnx=sqlcnx,
- pbtitle='-> dropping tables (first pass)')
+ with repo.internal_cnx() as cnx:
+ sqlcnx = cnx.cnxset.cnx
+ sqlcursor = cnx.cnxset.cu
+ execute = sqlcursor.execute
+ if drop:
+ helper = database.get_db_helper(driver)
+ dropsql = sql_drop_all_user_tables(helper, sqlcursor)
+ # We may fail dropping some tables because of table dependencies, in a first pass.
+ # So, we try a second drop sequence to drop remaining tables if needed.
+ # Note that 2 passes is an arbitrary choice as it seems enough for our usecases
+ # (looping may induce infinite recursion when user have no rights for example).
+ # Here we try to keep code simple and backend independent. That's why we don't try to
+ # distinguish remaining tables (missing privileges, dependencies, ...).
+ failed = sqlexec(dropsql, execute, cnx=sqlcnx,
+ pbtitle='-> dropping tables (first pass)')
+ if failed:
+ failed = sqlexec(failed, execute, cnx=sqlcnx,
+ pbtitle='-> dropping tables (second pass)')
+ remainings = list(filter(drop_filter, helper.list_tables(sqlcursor)))
+ assert not remainings, 'Remaining tables: %s' % ', '.join(remainings)
+ handler = config.migration_handler(schema, interactive=False, repo=repo, cnx=cnx)
+ # install additional driver specific sql files
+ handler.cmd_install_custom_sql_scripts()
+ for cube in reversed(config.cubes()):
+ handler.cmd_install_custom_sql_scripts(cube)
+ _title = '-> creating tables '
+ print(_title, end=' ')
+ # schema entities and relations tables
+ # can't skip entities table even if system source doesn't support them,
+ # they are used sometimes by generated sql. Keeping them empty is much
+ # simpler than fixing this...
+ schemasql = sqlschema(schema, driver)
+ #skip_entities=[str(e) for e in schema.entities()
+ # if not repo.system_source.support_entity(str(e))])
+ failed = sqlexec(schemasql, execute, pbtitle=_title, delimiter=';;')
if failed:
- failed = sqlexec(failed, execute, cnx=sqlcnx,
- pbtitle='-> dropping tables (second pass)')
- remainings = filter(drop_filter, helper.list_tables(sqlcursor))
- assert not remainings, 'Remaining tables: %s' % ', '.join(remainings)
- _title = '-> creating tables '
- print _title,
- # schema entities and relations tables
- # can't skip entities table even if system source doesn't support them,
- # they are used sometimes by generated sql. Keeping them empty is much
- # simpler than fixing this...
- schemasql = sqlschema(schema, driver)
- #skip_entities=[str(e) for e in schema.entities()
- # if not repo.system_source.support_entity(str(e))])
- failed = sqlexec(schemasql, execute, pbtitle=_title, delimiter=';;')
- if failed:
- print 'The following SQL statements failed. You should check your schema.'
- print failed
- raise Exception('execution of the sql schema failed, you should check your schema')
- sqlcursor.close()
- sqlcnx.commit()
- sqlcnx.close()
+ print('The following SQL statements failed. You should check your schema.')
+ print(failed)
+ raise Exception('execution of the sql schema failed, you should check your schema')
+ sqlcursor.close()
+ sqlcnx.commit()
with repo.internal_cnx() as cnx:
# insert entity representing the system source
ssource = cnx.create_entity('CWSource', type=u'native', name=u'system')
repo.system_source.eid = ssource.eid
cnx.execute('SET X cw_source X WHERE X eid %(x)s', {'x': ssource.eid})
# insert base groups and default admin
- print '-> inserting default user and default groups.'
+ print('-> inserting default user and default groups.')
try:
- login = unicode(sourcescfg['admin']['login'])
+ login = text_type(sourcescfg['admin']['login'])
pwd = sourcescfg['admin']['password']
except KeyError:
if interactive:
msg = 'enter login and password of the initial manager account'
login, pwd = manager_userpasswd(msg=msg, confirm=True)
else:
- login, pwd = unicode(source['db-user']), source['db-password']
+ login, pwd = text_type(source['db-user']), source['db-password']
# sort for eid predicatability as expected in some server tests
for group in sorted(BASE_GROUPS):
- cnx.create_entity('CWGroup', name=unicode(group))
+ cnx.create_entity('CWGroup', name=text_type(group))
admin = create_user(cnx, login, pwd, u'managers')
cnx.execute('SET X owned_by U WHERE X is IN (CWGroup,CWSource), U eid %(u)s',
{'u': admin.eid})
cnx.commit()
repo.shutdown()
- # reloging using the admin user
+ # re-login using the admin user
config._cubes = None # avoid assertion error
repo = get_repository(config=config)
with connect(repo, login, password=pwd) as cnx:
@@ -293,10 +302,6 @@
repo.system_source.eid = ssource.eid # redo this manually
handler = config.migration_handler(schema, interactive=False,
cnx=cnx, repo=repo)
- # install additional driver specific sql files
- handler.cmd_install_custom_sql_scripts()
- for cube in reversed(config.cubes()):
- handler.cmd_install_custom_sql_scripts(cube)
# serialize the schema
initialize_schema(config, schema, handler)
# yoo !
@@ -310,7 +315,7 @@
# (drop instance attribute to get back to class attribute)
del config.cubicweb_appobject_path
del config.cube_appobject_path
- print '-> database for instance %s initialized.' % config.appid
+ print('-> database for instance %s initialized.' % config.appid)
def initialize_schema(config, schema, mhandler, event='create'):
--- a/server/checkintegrity.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/checkintegrity.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,6 +20,8 @@
* integrity of a CubicWeb repository. Hum actually only the system database is
checked.
"""
+from __future__ import print_function
+
__docformat__ = "restructuredtext en"
import sys
@@ -27,7 +29,7 @@
from logilab.common.shellutils import ProgressBar
-from cubicweb.schema import PURE_VIRTUAL_RTYPES, VIRTUAL_RTYPES
+from cubicweb.schema import PURE_VIRTUAL_RTYPES, VIRTUAL_RTYPES, UNIQUE_CONSTRAINTS
from cubicweb.server.sqlutils import SQL_PREFIX
def notify_fixed(fix):
@@ -90,11 +92,11 @@
dbhelper = repo.system_source.dbhelper
cursor = cnx.cnxset.cu
if not dbhelper.has_fti_table(cursor):
- print 'no text index table'
+ print('no text index table')
dbhelper.init_fti(cursor)
repo.system_source.do_fti = True # ensure full-text indexation is activated
if etypes is None:
- print 'Reindexing entities'
+ print('Reindexing entities')
etypes = set()
for eschema in schema.entities():
if eschema.final:
@@ -107,8 +109,8 @@
# clear fti table first
cnx.system_sql('DELETE FROM %s' % dbhelper.fti_table)
else:
- print 'Reindexing entities of type %s' % \
- ', '.join(sorted(str(e) for e in etypes))
+ print('Reindexing entities of type %s' % \
+ ', '.join(sorted(str(e) for e in etypes)))
# clear fti table first. Use subquery for sql compatibility
cnx.system_sql("DELETE FROM %s WHERE EXISTS(SELECT 1 FROM ENTITIES "
"WHERE eid=%s AND type IN (%s))" % (
@@ -122,8 +124,7 @@
source = repo.system_source
for eschema in etypes:
etype_class = cnx.vreg['etypes'].etype_class(str(eschema))
- for fti_rql in etype_class.cw_fti_index_rql_queries(cnx):
- rset = cnx.execute(fti_rql)
+ for rset in etype_class.cw_fti_index_rql_limit(cnx):
source.fti_index_entities(cnx, rset.entities())
# clear entity cache to avoid high memory consumption on big tables
cnx.drop_entity_cache()
@@ -135,10 +136,7 @@
def check_schema(schema, cnx, eids, fix=1):
"""check serialized schema"""
- print 'Checking serialized schema'
- unique_constraints = ('SizeConstraint', 'FormatConstraint',
- 'VocabularyConstraint',
- 'RQLVocabularyConstraint')
+ print('Checking serialized schema')
rql = ('Any COUNT(X),RN,SN,ON,CTN GROUPBY RN,SN,ON,CTN ORDERBY 1 '
'WHERE X is CWConstraint, R constrained_by X, '
'R relation_type RT, RT name RN, R from_entity ST, ST name SN, '
@@ -146,17 +144,17 @@
for count, rn, sn, on, cstrname in cnx.execute(rql):
if count == 1:
continue
- if cstrname in unique_constraints:
- print "ERROR: got %s %r constraints on relation %s.%s.%s" % (
- count, cstrname, sn, rn, on)
+ if cstrname in UNIQUE_CONSTRAINTS:
+ print("ERROR: got %s %r constraints on relation %s.%s.%s" % (
+ count, cstrname, sn, rn, on))
if fix:
- print 'dunno how to fix, do it yourself'
+ print('dunno how to fix, do it yourself')
def check_text_index(schema, cnx, eids, fix=1):
"""check all entities registered in the text index"""
- print 'Checking text index'
+ print('Checking text index')
msg = ' Entity with eid %s exists in the text index but in no source (autofix will remove from text index)'
cursor = cnx.system_sql('SELECT uid FROM appears;')
for row in cursor.fetchall():
@@ -170,7 +168,7 @@
def check_entities(schema, cnx, eids, fix=1):
"""check all entities registered in the repo system table"""
- print 'Checking entities system table'
+ print('Checking entities system table')
# system table but no source
msg = ' Entity %s with eid %s exists in the system table but in no source (autofix will delete the entity)'
cursor = cnx.system_sql('SELECT eid,type FROM entities;')
@@ -228,7 +226,7 @@
'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_instance_of_relation as cs '
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid)')
notify_fixed(True)
- print 'Checking entities tables'
+ print('Checking entities tables')
msg = ' Entity with eid %s exists in the %s table but not in the system table (autofix will delete the entity)'
for eschema in schema.entities():
if eschema.final:
@@ -263,7 +261,7 @@
"""check that eids referenced by relations are registered in the repo system
table
"""
- print 'Checking relations'
+ print('Checking relations')
for rschema in schema.relations():
if rschema.final or rschema.type in PURE_VIRTUAL_RTYPES:
continue
@@ -287,7 +285,7 @@
cursor = cnx.system_sql('SELECT eid_from FROM %s_relation;' % rschema)
except Exception as ex:
# usually because table doesn't exist
- print 'ERROR', ex
+ print('ERROR', ex)
continue
for row in cursor.fetchall():
eid = row[0]
@@ -310,14 +308,14 @@
def check_mandatory_relations(schema, cnx, eids, fix=1):
"""check entities missing some mandatory relation"""
- print 'Checking mandatory relations'
+ print('Checking mandatory relations')
msg = '%s #%s is missing mandatory %s relation %s (autofix will delete the entity)'
for rschema in schema.relations():
if rschema.final or rschema in PURE_VIRTUAL_RTYPES or rschema in ('is', 'is_instance_of'):
continue
smandatory = set()
omandatory = set()
- for rdef in rschema.rdefs.itervalues():
+ for rdef in rschema.rdefs.values():
if rdef.cardinality[0] in '1+':
smandatory.add(rdef.subject)
if rdef.cardinality[1] in '1+':
@@ -340,12 +338,12 @@
"""check for entities stored in the system source missing some mandatory
attribute
"""
- print 'Checking mandatory attributes'
+ print('Checking mandatory attributes')
msg = '%s #%s is missing mandatory attribute %s (autofix will delete the entity)'
for rschema in schema.relations():
if not rschema.final or rschema in VIRTUAL_RTYPES:
continue
- for rdef in rschema.rdefs.itervalues():
+ for rdef in rschema.rdefs.values():
if rdef.cardinality[0] in '1+':
rql = 'Any X WHERE X %s NULL, X is %s, X cw_source S, S name "system"' % (
rschema, rdef.subject)
@@ -361,7 +359,7 @@
FIXME: rewrite using RQL queries ?
"""
- print 'Checking metadata'
+ print('Checking metadata')
cursor = cnx.system_sql("SELECT DISTINCT type FROM entities;")
eidcolumn = SQL_PREFIX + 'eid'
msg = ' %s with eid %s has no %s (autofix will set it to now)'
@@ -403,9 +401,9 @@
if fix:
cnx.commit()
else:
- print
+ print()
if not fix:
- print 'WARNING: Diagnostic run, nothing has been corrected'
+ print('WARNING: Diagnostic run, nothing has been corrected')
if reindex:
cnx.rollback()
reindex_entities(repo.schema, cnx, withpb=withpb)
--- a/server/cwzmq.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/cwzmq.py Wed Dec 16 11:23:48 2015 +0100
@@ -65,7 +65,7 @@
def add_subscriber(self, address):
subscriber = Subscriber(self.ioloop, address)
- for topic, callback in self._topics.iteritems():
+ for topic, callback in self._topics.items():
subscriber.subscribe(topic, callback)
self._subscribers.append(subscriber)
--- a/server/edition.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/edition.py Wed Dec 16 11:23:48 2015 +0100
@@ -38,7 +38,7 @@
class EditedEntity(dict):
"""encapsulate entities attributes being written by an RQL query"""
def __init__(self, entity, **kwargs):
- dict.__init__(self, **kwargs)
+ super(EditedEntity, self).__init__(**kwargs)
self.entity = entity
self.skip_security = set()
self.querier_pending_relations = {}
@@ -50,15 +50,18 @@
def __lt__(self, other):
# we don't want comparison by value inherited from dict
- return id(self) < id(other)
+ raise NotImplementedError
def __eq__(self, other):
- return id(self) == id(other)
+ return self is other
+
+ def __ne__(self, other):
+ return not (self == other)
def __setitem__(self, attr, value):
assert attr != 'eid'
# don't add attribute into skip_security if already in edited
- # attributes, else we may accidentaly skip a desired security check
+ # attributes, else we may accidentally skip a desired security check
if attr not in self:
self.skip_security.add(attr)
self.edited_attribute(attr, value)
@@ -83,7 +86,7 @@
def setdefault(self, attr, default):
assert attr != 'eid'
# don't add attribute into skip_security if already in edited
- # attributes, else we may accidentaly skip a desired security check
+ # attributes, else we may accidentally skip a desired security check
if attr not in self:
self[attr] = default
return self[attr]
@@ -93,7 +96,7 @@
setitem = self.__setitem__
else:
setitem = self.edited_attribute
- for attr, value in values.iteritems():
+ for attr, value in values.items():
setitem(attr, value)
def edited_attribute(self, attr, value):
--- a/server/hook.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/hook.py Wed Dec 16 11:23:48 2015 +0100
@@ -248,6 +248,8 @@
.. autoclass:: cubicweb.server.hook.LateOperation
.. autoclass:: cubicweb.server.hook.DataOperationMixIn
"""
+from __future__ import print_function
+
__docformat__ = "restructuredtext en"
from warnings import warn
@@ -331,7 +333,7 @@
with cnx.running_hooks_ops():
for hook in hooks:
if debug:
- print event, _kwargs, hook
+ print(event, _kwargs, hook)
hook()
def get_pruned_hooks(self, cnx, event, entities, eids_from_to, kwargs):
@@ -370,7 +372,7 @@
pruned = set()
cnx.pruned_hooks_cache[cache_key] = pruned
if look_for_selector is not None:
- for id, hooks in self.iteritems():
+ for id, hooks in self.items():
for hook in hooks:
enabled_cat, main_filter = hook.filterable_selectors()
if enabled_cat is not None:
@@ -382,14 +384,14 @@
(main_filter.frometypes is not None or \
main_filter.toetypes is not None):
continue
- first_kwargs = _iter_kwargs(entities, eids_from_to, kwargs).next()
+ first_kwargs = next(_iter_kwargs(entities, eids_from_to, kwargs))
if not main_filter(hook, cnx, **first_kwargs):
pruned.add(hook)
return pruned
def filtered_possible_objects(self, pruned, *args, **kwargs):
- for appobjects in self.itervalues():
+ for appobjects in self.values():
if pruned:
filtered_objects = [obj for obj in appobjects if obj not in pruned]
if not filtered_objects:
@@ -636,7 +638,7 @@
# to set in concrete class (mandatory)
subject_relations = None
object_relations = None
- # to set in concrete class (optionaly)
+ # to set in concrete class (optionally)
skip_subject_relations = ()
skip_object_relations = ()
@@ -713,7 +715,7 @@
the transaction has been either rolled back either:
- * intentionaly
+ * intentionally
* a 'precommit' event failed, in which case all operations are rolled back
once 'revertprecommit'' has been called
--- a/server/hooksmanager.py Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,22 +0,0 @@
-# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
-# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
-#
-# This file is part of CubicWeb.
-#
-# CubicWeb is free software: you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License as published by the Free
-# Software Foundation, either version 2.1 of the License, or (at your option)
-# any later version.
-#
-# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
-# details.
-#
-# You should have received a copy of the GNU Lesser General Public License along
-# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-from logilab.common.deprecation import class_renamed, class_moved
-from cubicweb.server import hook
-
-SystemHook = class_renamed('SystemHook', hook.Hook)
-Hook = class_moved(hook.Hook)
--- a/server/migractions.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/migractions.py Wed Dec 16 11:23:48 2015 +0100
@@ -26,6 +26,8 @@
* add an entity
* execute raw RQL queries
"""
+from __future__ import print_function
+
__docformat__ = "restructuredtext en"
import sys
@@ -40,6 +42,8 @@
from warnings import warn
from contextlib import contextmanager
+from six import PY2, text_type
+
from logilab.common.deprecation import deprecated
from logilab.common.decorators import cached, clear_cache
@@ -93,7 +97,7 @@
self.repo = repo
self.session = cnx.session
elif connect:
- self.repo_connect()
+ self.repo = config.repository()
self.set_cnx()
else:
self.session = None
@@ -134,30 +138,24 @@
try:
self.cnx = repoapi.connect(self.repo, login, password=pwd)
if not 'managers' in self.cnx.user.groups:
- print 'migration need an account in the managers group'
+ print('migration need an account in the managers group')
else:
break
except AuthenticationError:
- print 'wrong user/password'
+ print('wrong user/password')
except (KeyboardInterrupt, EOFError):
- print 'aborting...'
+ print('aborting...')
sys.exit(0)
try:
login, pwd = manager_userpasswd()
except (KeyboardInterrupt, EOFError):
- print 'aborting...'
+ print('aborting...')
sys.exit(0)
self.session = self.repo._get_session(self.cnx.sessionid)
-
- @cached
- def repo_connect(self):
- self.repo = repoapi.get_repository(config=self.config)
- return self.repo
-
def cube_upgraded(self, cube, version):
self.cmd_set_property('system.version.%s' % cube.lower(),
- unicode(version))
+ text_type(version))
self.commit()
def shutdown(self):
@@ -191,7 +189,7 @@
def backup_database(self, backupfile=None, askconfirm=True, format='native'):
config = self.config
- repo = self.repo_connect()
+ repo = self.repo
# paths
timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
instbkdir = osp.join(config.appdatahome, 'backup')
@@ -202,13 +200,13 @@
# check backup has to be done
if osp.exists(backupfile) and not \
self.confirm('Backup file %s exists, overwrite it?' % backupfile):
- print '-> no backup done.'
+ print('-> no backup done.')
return
elif askconfirm and not self.confirm('Backup %s database?' % config.appid):
- print '-> no backup done.'
+ print('-> no backup done.')
return
open(backupfile,'w').close() # kinda lock
- os.chmod(backupfile, 0600)
+ os.chmod(backupfile, 0o600)
# backup
source = repo.system_source
tmpdir = tempfile.mkdtemp()
@@ -217,7 +215,7 @@
try:
source.backup(osp.join(tmpdir, source.uri), self.confirm, format=format)
except Exception as ex:
- print '-> error trying to backup %s [%s]' % (source.uri, ex)
+ print('-> error trying to backup %s [%s]' % (source.uri, ex))
if not self.confirm('Continue anyway?', default='n'):
raise SystemExit(1)
else:
@@ -226,7 +224,7 @@
format_file.write('%s\n' % format)
with open(osp.join(tmpdir, 'versions.txt'), 'w') as version_file:
versions = repo.get_versions()
- for cube, version in versions.iteritems():
+ for cube, version in versions.items():
version_file.write('%s %s\n' % (cube, version))
if not failed:
bkup = tarfile.open(backupfile, 'w|gz')
@@ -236,7 +234,7 @@
# call hooks
repo.hm.call_hooks('server_backup', repo=repo, timestamp=timestamp)
# done
- print '-> backup file', backupfile
+ print('-> backup file', backupfile)
finally:
shutil.rmtree(tmpdir)
@@ -268,19 +266,19 @@
if written_format in ('portable', 'native'):
format = written_format
self.config.init_cnxset_pool = False
- repo = self.repo_connect()
+ repo = self.repo = self.config.repository()
source = repo.system_source
try:
source.restore(osp.join(tmpdir, source.uri), self.confirm, drop, format)
except Exception as exc:
- print '-> error trying to restore %s [%s]' % (source.uri, exc)
+ print('-> error trying to restore %s [%s]' % (source.uri, exc))
if not self.confirm('Continue anyway?', default='n'):
raise SystemExit(1)
shutil.rmtree(tmpdir)
# call hooks
repo.init_cnxset_pool()
repo.hm.call_hooks('server_restore', repo=repo, timestamp=backupfile)
- print '-> database restored.'
+ print('-> database restored.')
def commit(self):
self.cnx.commit()
@@ -362,11 +360,11 @@
directory = osp.join(self.config.cube_dir(cube), 'schema')
sql_scripts = glob(osp.join(directory, '*.%s.sql' % driver))
for fpath in sql_scripts:
- print '-> installing', fpath
+ print('-> installing', fpath)
failed = sqlexec(open(fpath).read(), self.cnx.system_sql, False,
delimiter=';;')
if failed:
- print '-> ERROR, skipping', fpath
+ print('-> ERROR, skipping', fpath)
# schema synchronization internals ########################################
@@ -424,7 +422,7 @@
{'x': expreid}, ask_confirm=False)
else:
newexprs.pop(expression)
- for expression in newexprs.itervalues():
+ for expression in newexprs.values():
expr = expression.expression
if not confirm or self.confirm('Add %s expression for %s permission of %s?'
% (expr, action, erschema)):
@@ -460,7 +458,10 @@
assert reporschema.eid, reporschema
self.rqlexecall(ss.updaterschema2rql(rschema, reporschema.eid),
ask_confirm=self.verbosity>=2)
- if syncrdefs:
+ if rschema.rule:
+ if syncperms:
+ self._synchronize_permissions(rschema, reporschema.eid)
+ elif syncrdefs:
for subj, obj in rschema.rdefs:
if (subj, obj) not in reporschema.rdefs:
continue
@@ -552,12 +553,12 @@
for name in cols:
rschema = repoeschema.subjrels.get(name)
if rschema is None:
- print 'dont add %s unique constraint on %s, missing %s' % (
- ','.join(cols), eschema, name)
+ print('dont add %s unique constraint on %s, missing %s' % (
+ ','.join(cols), eschema, name))
return False
if not (rschema.final or rschema.inlined):
- print 'dont add %s unique constraint on %s, %s is neither final nor inlined' % (
- ','.join(cols), eschema, name)
+ print('dont add %s unique constraint on %s, %s is neither final nor inlined' % (
+ ','.join(cols), eschema, name))
return False
return True
@@ -574,6 +575,7 @@
against its current definition:
* order and other properties
* constraints
+ * permissions
"""
subjtype, objtype = str(subjtype), str(objtype)
rschema = self.fs_schema.rschema(rtype)
@@ -743,8 +745,8 @@
rschema = self.repo.schema.rschema(attrname)
attrtype = rschema.objects(etype)[0]
except KeyError:
- print 'warning: attribute %s %s is not known, skip deletion' % (
- etype, attrname)
+ print('warning: attribute %s %s is not known, skip deletion' % (
+ etype, attrname))
else:
self.cmd_drop_relation_definition(etype, attrname, attrtype,
commit=commit)
@@ -781,7 +783,7 @@
instschema = self.repo.schema
eschema = self.fs_schema.eschema(etype)
if etype in instschema and not (eschema.final and eschema.eid is None):
- print 'warning: %s already known, skip addition' % etype
+ print('warning: %s already known, skip addition' % etype)
return
confirm = self.verbosity >= 2
groupmap = self.group_mapping()
@@ -899,9 +901,11 @@
self.commit()
def cmd_drop_entity_type(self, etype, commit=True):
- """unregister an existing entity type
+ """Drop an existing entity type.
- This will trigger deletion of necessary relation types and definitions
+ This will trigger deletion of necessary relation types and definitions.
+ Note that existing entities of the given type will be deleted without
+ any hooks called.
"""
# XXX what if we delete an entity type which is specialized by other types
# unregister the entity from CWEType
@@ -918,7 +922,7 @@
"""
schema = self.repo.schema
if oldname not in schema:
- print 'warning: entity type %s is unknown, skip renaming' % oldname
+ print('warning: entity type %s is unknown, skip renaming' % oldname)
return
# if merging two existing entity types
if newname in schema:
@@ -997,7 +1001,7 @@
# elif simply renaming an entity type
else:
self.rqlexec('SET ET name %(newname)s WHERE ET is CWEType, ET name %(on)s',
- {'newname' : unicode(newname), 'on' : oldname},
+ {'newname' : text_type(newname), 'on' : oldname},
ask_confirm=False)
if commit:
self.commit()
@@ -1017,8 +1021,8 @@
rschema = self.fs_schema.rschema(rtype)
execute = self.cnx.execute
if rtype in reposchema:
- print 'warning: relation type %s is already known, skip addition' % (
- rtype)
+ print('warning: relation type %s is already known, skip addition' % (
+ rtype))
elif rschema.rule:
gmap = self.group_mapping()
ss.execschemarql(execute, rschema, ss.crschema2rql(rschema, gmap))
@@ -1060,7 +1064,11 @@
self.commit()
def cmd_drop_relation_type(self, rtype, commit=True):
- """unregister an existing relation type"""
+ """Drop an existing relation type.
+
+ Note that existing relations of the given type will be deleted without
+ any hooks called.
+ """
self.rqlexec('DELETE CWRType X WHERE X name %r' % rtype,
ask_confirm=self.verbosity>=2)
self.rqlexec('DELETE CWComputedRType X WHERE X name %r' % rtype,
@@ -1098,8 +1106,8 @@
if not rtype in self.repo.schema:
self.cmd_add_relation_type(rtype, addrdef=False, commit=True)
if (subjtype, objtype) in self.repo.schema.rschema(rtype).rdefs:
- print 'warning: relation %s %s %s is already known, skip addition' % (
- subjtype, rtype, objtype)
+ print('warning: relation %s %s %s is already known, skip addition' % (
+ subjtype, rtype, objtype))
return
rdef = self._get_rdef(rschema, subjtype, objtype)
ss.execschemarql(self.cnx.execute, rdef,
@@ -1120,7 +1128,11 @@
return rdef
def cmd_drop_relation_definition(self, subjtype, rtype, objtype, commit=True):
- """unregister an existing relation definition"""
+ """Drop an existing relation definition.
+
+ Note that existing relations of the given definition will be deleted
+ without any hooks called.
+ """
rschema = self.repo.schema.rschema(rtype)
if rschema.rule:
raise ExecutionError('Cannot drop a relation definition for a '
@@ -1200,7 +1212,7 @@
values = []
for k, v in kwargs.items():
values.append('X %s %%(%s)s' % (k, k))
- if isinstance(v, str):
+ if PY2 and isinstance(v, str):
kwargs[k] = unicode(v)
rql = 'SET %s WHERE %s' % (','.join(values), ','.join(restriction))
self.rqlexec(rql, kwargs, ask_confirm=self.verbosity>=2)
@@ -1233,7 +1245,7 @@
self.rqlexec('SET C value %%(v)s WHERE X from_entity S, X relation_type R,'
'X constrained_by C, C cstrtype CT, CT name "SizeConstraint",'
'S name "%s", R name "%s"' % (etype, rtype),
- {'v': unicode(SizeConstraint(size).serialize())},
+ {'v': text_type(SizeConstraint(size).serialize())},
ask_confirm=self.verbosity>=2)
else:
self.rqlexec('DELETE X constrained_by C WHERE X from_entity S, X relation_type R,'
@@ -1270,7 +1282,7 @@
:rtype: `Workflow`
"""
- wf = self.cmd_create_entity('Workflow', name=unicode(name),
+ wf = self.cmd_create_entity('Workflow', name=text_type(name),
**kwargs)
if not isinstance(wfof, (list, tuple)):
wfof = (wfof,)
@@ -1278,19 +1290,19 @@
return 'missing workflow relations, see make_workflowable(%s)' % etype
for etype in wfof:
eschema = self.repo.schema[etype]
- etype = unicode(etype)
+ etype = text_type(etype)
if ensure_workflowable:
assert 'in_state' in eschema.subjrels, _missing_wf_rel(etype)
assert 'custom_workflow' in eschema.subjrels, _missing_wf_rel(etype)
assert 'wf_info_for' in eschema.objrels, _missing_wf_rel(etype)
rset = self.rqlexec(
'SET X workflow_of ET WHERE X eid %(x)s, ET name %(et)s',
- {'x': wf.eid, 'et': unicode(etype)}, ask_confirm=False)
+ {'x': wf.eid, 'et': text_type(etype)}, ask_confirm=False)
assert rset, 'unexistant entity type %s' % etype
if default:
self.rqlexec(
'SET ET default_workflow X WHERE X eid %(x)s, ET name %(et)s',
- {'x': wf.eid, 'et': unicode(etype)}, ask_confirm=False)
+ {'x': wf.eid, 'et': text_type(etype)}, ask_confirm=False)
if commit:
self.commit()
return wf
@@ -1321,13 +1333,13 @@
To set a user specific property value, use appropriate method on CWUser
instance.
"""
- value = unicode(value)
+ value = text_type(value)
try:
prop = self.rqlexec(
'CWProperty X WHERE X pkey %(k)s, NOT X for_user U',
- {'k': unicode(pkey)}, ask_confirm=False).get_entity(0, 0)
+ {'k': text_type(pkey)}, ask_confirm=False).get_entity(0, 0)
except Exception:
- self.cmd_create_entity('CWProperty', pkey=unicode(pkey), value=value)
+ self.cmd_create_entity('CWProperty', pkey=text_type(pkey), value=value)
else:
prop.cw_set(value=value)
@@ -1351,7 +1363,7 @@
# remove from entity cache to avoid memory exhaustion
del entity.cw_attr_cache[attribute]
pb.update()
- print
+ print()
source.set_storage(etype, attribute, storage)
def cmd_create_entity(self, etype, commit=False, **kwargs):
@@ -1564,12 +1576,14 @@
else:
raise StopIteration
- def next(self):
+ def __next__(self):
if self._rsetit is not None:
- return self._rsetit.next()
+ return next(self._rsetit)
rset = self._get_rset()
self._rsetit = iter(rset)
- return self._rsetit.next()
+ return next(self._rsetit)
+
+ next = __next__
def entities(self):
try:
--- a/server/querier.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/querier.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,10 +18,15 @@
"""Helper classes to execute RQL queries on a set of sources, performing
security checking and data aggregation.
"""
+from __future__ import print_function
+
__docformat__ = "restructuredtext en"
from itertools import repeat
+from six import text_type, string_types, integer_types
+from six.moves import range
+
from rql import RQLSyntaxError, CoercionError
from rql.stmts import Union
from rql.nodes import ETYPE_PYOBJ_MAP, etype_from_pyobj, Relation, Exists, Not
@@ -61,7 +66,7 @@
def check_no_password_selected(rqlst):
"""check that Password entities are not selected"""
for solution in rqlst.solutions:
- for var, etype in solution.iteritems():
+ for var, etype in solution.items():
if etype == 'Password':
raise Unauthorized('Password selection is not allowed (%s)' % var)
@@ -103,13 +108,13 @@
solution, args))
if not user.matching_groups(rdef.get_groups('read')):
if DBG:
- print ('check_read_access: %s %s does not match %s' %
- (rdef, user.groups, rdef.get_groups('read')))
+ print('check_read_access: %s %s does not match %s' %
+ (rdef, user.groups, rdef.get_groups('read')))
# XXX rqlexpr not allowed
raise Unauthorized('read', rel.r_type)
if DBG:
- print ('check_read_access: %s %s matches %s' %
- (rdef, user.groups, rdef.get_groups('read')))
+ print('check_read_access: %s %s matches %s' %
+ (rdef, user.groups, rdef.get_groups('read')))
def get_local_checks(cnx, rqlst, solution):
"""Check that the given user has credentials to access data read by the
@@ -138,8 +143,8 @@
ex = Unauthorized('read', solution[varname])
ex.var = varname
if DBG:
- print ('check_read_access: %s %s %s %s' %
- (varname, eschema, user.groups, eschema.get_groups('read')))
+ print('check_read_access: %s %s %s %s' %
+ (varname, eschema, user.groups, eschema.get_groups('read')))
raise ex
# don't insert security on variable only referenced by 'NOT X relation Y' or
# 'NOT EXISTS(X relation Y)'
@@ -265,7 +270,7 @@
# which have a known eid
varkwargs = {}
if not cnx.transaction_data.get('security-rqlst-cache'):
- for var in rqlst.defined_vars.itervalues():
+ for var in rqlst.defined_vars.values():
if var.stinfo['constnode'] is not None:
eid = var.stinfo['constnode'].eval(self.args)
varkwargs[var.name] = int(eid)
@@ -285,7 +290,7 @@
newsolutions.append(solution)
# try to benefit of rqlexpr.check cache for entities which
# are specified by eid in query'args
- for varname, eid in varkwargs.iteritems():
+ for varname, eid in varkwargs.items():
try:
rqlexprs = localcheck.pop(varname)
except KeyError:
@@ -303,7 +308,7 @@
# mark variables protected by an rql expression
restricted_vars.update(localcheck)
# turn local check into a dict key
- localcheck = tuple(sorted(localcheck.iteritems()))
+ localcheck = tuple(sorted(localcheck.items()))
localchecks.setdefault(localcheck, []).append(solution)
# raise Unautorized exception if the user can't access to any solution
if not newsolutions:
@@ -334,7 +339,7 @@
def __init__(self, querier, rqlst, args, cnx):
ExecutionPlan.__init__(self, querier, rqlst, args, cnx)
- # save originaly selected variable, we may modify this
+ # save originally selected variable, we may modify this
# dictionary for substitution (query parameters)
self.selected = rqlst.selection
# list of rows of entities definition (ssplanner.EditedEntity)
@@ -414,7 +419,7 @@
def relation_defs(self):
"""return the list for relation definitions to insert"""
- for rdefs in self._expanded_r_defs.itervalues():
+ for rdefs in self._expanded_r_defs.values():
for rdef in rdefs:
yield rdef
for rdef in self.r_defs:
@@ -446,13 +451,13 @@
relations = {}
for subj, rtype, obj in self.relation_defs():
# if a string is given into args instead of an int, we get it here
- if isinstance(subj, basestring):
+ if isinstance(subj, string_types):
subj = int(subj)
- elif not isinstance(subj, (int, long)):
+ elif not isinstance(subj, integer_types):
subj = subj.entity.eid
- if isinstance(obj, basestring):
+ if isinstance(obj, string_types):
obj = int(obj)
- elif not isinstance(obj, (int, long)):
+ elif not isinstance(obj, integer_types):
obj = obj.entity.eid
if repo.schema.rschema(rtype).inlined:
if subj not in edited_entities:
@@ -468,7 +473,7 @@
else:
relations[rtype] = [(subj, obj)]
repo.glob_add_relations(cnx, relations)
- for edited in edited_entities.itervalues():
+ for edited in edited_entities.values():
repo.glob_update_entity(cnx, edited)
@@ -507,7 +512,7 @@
def parse(self, rql, annotate=False):
"""return a rql syntax tree for the given rql"""
try:
- return self._parse(unicode(rql), annotate=annotate)
+ return self._parse(text_type(rql), annotate=annotate)
except UnicodeError:
raise RQLSyntaxError(rql)
@@ -539,8 +544,8 @@
"""
if server.DEBUG & (server.DBG_RQL | server.DBG_SQL):
if server.DEBUG & (server.DBG_MORE | server.DBG_SQL):
- print '*'*80
- print 'querier input', repr(rql), repr(args)
+ print('*'*80)
+ print('querier input', repr(rql), repr(args))
# parse the query and binds variables
cachekey = (rql,)
try:
@@ -601,7 +606,7 @@
if args:
# different SQL generated when some argument is None or not (IS
# NULL). This should be considered when computing sql cache key
- cachekey += tuple(sorted([k for k, v in args.iteritems()
+ cachekey += tuple(sorted([k for k, v in args.items()
if v is None]))
# make an execution plan
plan = self.plan_factory(rqlst, args, cnx)
@@ -641,7 +646,7 @@
# so compute description manually even if there is only
# one solution
basedescr = [None] * len(plan.selected)
- todetermine = zip(xrange(len(plan.selected)), repeat(False))
+ todetermine = list(zip(range(len(plan.selected)), repeat(False)))
descr = _build_descr(cnx, results, basedescr, todetermine)
# FIXME: get number of affected entities / relations on non
# selection queries ?
@@ -668,7 +673,7 @@
unstables = rqlst.get_variable_indices()
basedescr = []
todetermine = []
- for i in xrange(len(rqlst.children[0].selection)):
+ for i in range(len(rqlst.children[0].selection)):
ttype = _selection_idx_type(i, rqlst, args)
if ttype is None or ttype == 'Any':
ttype = None
--- a/server/repository.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/repository.py Wed Dec 16 11:23:48 2015 +0100
@@ -25,15 +25,18 @@
point to a cubicweb instance.
* handles session management
"""
+from __future__ import print_function
+
__docformat__ = "restructuredtext en"
import threading
-import Queue
from warnings import warn
from itertools import chain
from time import time, localtime, strftime
from contextlib import contextmanager
+from six.moves import range, queue
+
from logilab.common.decorators import cached, clear_cache
from logilab.common.deprecation import deprecated
@@ -186,18 +189,18 @@
# registry hook to fix user class on registry reload
@onevent('after-registry-reload', self)
def fix_user_classes(self):
- # After registery reload the 'CWUser' class used for CWEtype
- # changed. To any existing user object have a different class than
+ # After registry reload the 'CWUser' class used for CWEtype
+ # changed. So any existing user object have a different class than
# the new loaded one. We are hot fixing this.
usercls = self.vreg['etypes'].etype_class('CWUser')
- for session in self._sessions.itervalues():
+ for session in self._sessions.values():
if not isinstance(session.user, InternalManager):
session.user.__class__ = usercls
def init_cnxset_pool(self):
"""should be called bootstrap_repository, as this is what it does"""
config = self.config
- self._cnxsets_pool = Queue.Queue()
+ self._cnxsets_pool = queue.Queue()
# 0. init a cnxset that will be used to fetch bootstrap information from
# the database
self._cnxsets_pool.put_nowait(self.system_source.wrapped_connection())
@@ -240,7 +243,7 @@
# proper initialization
self._get_cnxset().close(True)
self.cnxsets = [] # list of available cnxsets (can't iterate on a Queue)
- for i in xrange(config['connections-pool-size']):
+ for i in range(config['connections-pool-size']):
self.cnxsets.append(self.system_source.wrapped_connection())
self._cnxsets_pool.put_nowait(self.cnxsets[-1])
@@ -308,7 +311,7 @@
else:
self.vreg._set_schema(schema)
self.querier.set_schema(schema)
- for source in self.sources_by_uri.itervalues():
+ for source in self.sources_by_uri.values():
source.set_schema(schema)
self.schema = schema
@@ -377,7 +380,7 @@
def _get_cnxset(self):
try:
return self._cnxsets_pool.get(True, timeout=5)
- except Queue.Empty:
+ except queue.Empty:
raise Exception('no connections set available after 5 secs, probably either a '
'bug in code (too many uncommited/rolled back '
'connections) or too much load on the server (in '
@@ -387,13 +390,6 @@
def _free_cnxset(self, cnxset):
self._cnxsets_pool.put_nowait(cnxset)
- def pinfo(self):
- # XXX: session.cnxset is accessed from a local storage, would be interesting
- # to see if there is a cnxset set in any thread specific data)
- return '%s: %s (%s)' % (self._cnxsets_pool.qsize(),
- ','.join(session.user.login for session in self._sessions.itervalues()
- if session.cnxset),
- threading.currentThread())
def shutdown(self):
"""called on server stop event to properly close opened sessions and
connections
@@ -441,7 +437,7 @@
"""
# iter on sources_by_uri then check enabled source since sources doesn't
# contain copy based sources
- for source in self.sources_by_uri.itervalues():
+ for source in self.sources_by_uri.values():
if self.config.source_enabled(source) and source.support_entity('CWUser'):
try:
return source.authenticate(cnx, login, **authinfo)
@@ -575,7 +571,7 @@
"""
sources = {}
# remove sensitive information
- for uri, source in self.sources_by_uri.iteritems():
+ for uri, source in self.sources_by_uri.items():
sources[uri] = source.public_config
return sources
@@ -623,7 +619,7 @@
raise Exception('bad input for find_user')
with self.internal_cnx() as cnx:
varmaker = rqlvar_maker()
- vars = [(attr, varmaker.next()) for attr in fetch_attrs]
+ vars = [(attr, next(varmaker)) for attr in fetch_attrs]
rql = 'Any %s WHERE X is CWUser, ' % ','.join(var[1] for var in vars)
rql += ','.join('X %s %s' % (var[0], var[1]) for var in vars) + ','
rset = cnx.execute(rql + ','.join('X %s %%(%s)s' % (attr, attr)
@@ -779,6 +775,7 @@
args[key] = int(args[key])
return tuple(cachekey)
+ @deprecated('[3.22] use the new store API')
def extid2eid(self, source, extid, etype, cnx, insert=True,
sourceparams=None):
"""Return eid from a local id. If the eid is a negative integer, that
@@ -919,7 +916,7 @@
# set caches asap
extid = self.init_entity_caches(cnx, entity, source)
if server.DEBUG & server.DBG_REPO:
- print 'ADD entity', self, entity.cw_etype, entity.eid, edited
+ print('ADD entity', self, entity.cw_etype, entity.eid, edited)
prefill_entity_caches(entity)
self.hm.call_hooks('before_add_entity', cnx, entity=entity)
relations = preprocess_inlined_relations(cnx, entity)
@@ -950,8 +947,8 @@
"""
entity = edited.entity
if server.DEBUG & server.DBG_REPO:
- print 'UPDATE entity', entity.cw_etype, entity.eid, \
- entity.cw_attr_cache, edited
+ print('UPDATE entity', entity.cw_etype, entity.eid,
+ entity.cw_attr_cache, edited)
hm = self.hm
eschema = entity.e_schema
cnx.set_entity_cache(entity)
@@ -1043,9 +1040,9 @@
except KeyError:
data_by_etype[etype] = [entity]
source = self.system_source
- for etype, entities in data_by_etype.iteritems():
+ for etype, entities in data_by_etype.items():
if server.DEBUG & server.DBG_REPO:
- print 'DELETE entities', etype, [entity.eid for entity in entities]
+ print('DELETE entities', etype, [entity.eid for entity in entities])
self.hm.call_hooks('before_delete_entity', cnx, entities=entities)
self._delete_cascade_multi(cnx, entities)
source.delete_entities(cnx, entities)
@@ -1067,10 +1064,10 @@
subjects_by_types = {}
objects_by_types = {}
activintegrity = cnx.is_hook_category_activated('activeintegrity')
- for rtype, eids_subj_obj in relations.iteritems():
+ for rtype, eids_subj_obj in relations.items():
if server.DEBUG & server.DBG_REPO:
for subjeid, objeid in eids_subj_obj:
- print 'ADD relation', subjeid, rtype, objeid
+ print('ADD relation', subjeid, rtype, objeid)
for subjeid, objeid in eids_subj_obj:
if rtype in relations_by_rtype:
relations_by_rtype[rtype].append((subjeid, objeid))
@@ -1105,22 +1102,22 @@
objects[objeid] = len(relations_by_rtype[rtype])
continue
objects[objeid] = len(relations_by_rtype[rtype])
- for rtype, source_relations in relations_by_rtype.iteritems():
+ for rtype, source_relations in relations_by_rtype.items():
self.hm.call_hooks('before_add_relation', cnx,
rtype=rtype, eids_from_to=source_relations)
- for rtype, source_relations in relations_by_rtype.iteritems():
+ for rtype, source_relations in relations_by_rtype.items():
source.add_relations(cnx, rtype, source_relations)
rschema = self.schema.rschema(rtype)
for subjeid, objeid in source_relations:
cnx.update_rel_cache_add(subjeid, rtype, objeid, rschema.symmetric)
- for rtype, source_relations in relations_by_rtype.iteritems():
+ for rtype, source_relations in relations_by_rtype.items():
self.hm.call_hooks('after_add_relation', cnx,
rtype=rtype, eids_from_to=source_relations)
def glob_delete_relation(self, cnx, subject, rtype, object):
"""delete a relation from the repository"""
if server.DEBUG & server.DBG_REPO:
- print 'DELETE relation', subject, rtype, object
+ print('DELETE relation', subject, rtype, object)
source = self.system_source
self.hm.call_hooks('before_delete_relation', cnx,
eidfrom=subject, rtype=rtype, eidto=object)
--- a/server/rqlannotation.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/rqlannotation.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,6 +18,7 @@
"""Functions to add additional annotations on a rql syntax tree to ease later
code generation.
"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -33,7 +34,7 @@
#if server.DEBUG:
# print '-------- sql annotate', repr(rqlst)
getrschema = annotator.schema.rschema
- for var in rqlst.defined_vars.itervalues():
+ for var in rqlst.defined_vars.values():
stinfo = var.stinfo
if stinfo.get('ftirels'):
has_text_query = True
@@ -144,7 +145,7 @@
stinfo['invariant'] = False
# see unittest_rqlannotation. test_has_text_security_cache_bug
# XXX probably more to do, but yet that work without more...
- for col_alias in rqlst.aliases.itervalues():
+ for col_alias in rqlst.aliases.values():
if col_alias.stinfo.get('ftirels'):
has_text_query = True
return has_text_query
@@ -194,7 +195,7 @@
# if DISTINCT query, can use variable from a different scope as principal
# since introduced duplicates will be removed
if scope.stmt.distinct and diffscope_rels:
- return iter(_sort(diffscope_rels)).next()
+ return next(iter(_sort(diffscope_rels)))
# XXX could use a relation from a different scope if it can't generate
# duplicates, so we should have to check cardinality
raise CantSelectPrincipal()
@@ -231,7 +232,7 @@
for select in union.children:
for subquery in select.with_:
set_qdata(getrschema, subquery.query, noinvariant)
- for var in select.defined_vars.itervalues():
+ for var in select.defined_vars.values():
if var.stinfo['invariant']:
if var in noinvariant and not var.stinfo['principal'].r_type == 'has_text':
var._q_invariant = False
@@ -317,7 +318,7 @@
def compute(self, rqlst):
# set domains for each variable
- for varname, var in rqlst.defined_vars.iteritems():
+ for varname, var in rqlst.defined_vars.items():
if var.stinfo['uidrel'] is not None or \
self.eschema(rqlst.solutions[0][varname]).final:
ptypes = var.stinfo['possibletypes']
@@ -354,9 +355,9 @@
continue
def _debug_print(self):
- print 'varsols', dict((x, sorted(str(v) for v in values))
- for x, values in self.varsols.iteritems())
- print 'ambiguous vars', sorted(self.ambiguousvars)
+ print('varsols', dict((x, sorted(str(v) for v in values))
+ for x, values in self.varsols.items()))
+ print('ambiguous vars', sorted(self.ambiguousvars))
def set_rel_constraint(self, term, rel, etypes_func):
if isinstance(term, VariableRef) and self.is_ambiguous(term.variable):
--- a/server/schema2sql.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/schema2sql.py Wed Dec 16 11:23:48 2015 +0100
@@ -162,8 +162,8 @@
def check_constraint(eschema, aschema, attr, constraint, dbhelper, prefix=''):
# XXX should find a better name
- cstrname = 'cstr' + md5(eschema.type + attr + constraint.type() +
- (constraint.serialize() or '')).hexdigest()
+ cstrname = 'cstr' + md5((eschema.type + attr + constraint.type() +
+ (constraint.serialize() or '')).encode('ascii')).hexdigest()
if constraint.type() == 'BoundaryConstraint':
value = as_sql(constraint.boundary, dbhelper, prefix)
return cstrname, '%s%s %s %s' % (prefix, attr, constraint.operator, value)
--- a/server/schemaserial.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/schemaserial.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""functions for schema / permissions (de)serialization using RQL"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -23,6 +24,8 @@
import json
import sys
+from six import PY2, text_type, string_types
+
from logilab.common.shellutils import ProgressBar, DummyProgressBar
from yams import BadSchemaDefinition, schema as schemamod, buildobjs as ybo
@@ -49,11 +52,11 @@
return res
missing = [g for g in ('owners', 'managers', 'users', 'guests') if not g in res]
if missing:
- print 'some native groups are missing but the following groups have been found:'
- print '\n'.join('* %s (%s)' % (n, eid) for n, eid in res.items())
- print
- print 'enter the eid of a to group to map to each missing native group'
- print 'or just type enter to skip permissions granted to a group'
+ print('some native groups are missing but the following groups have been found:')
+ print('\n'.join('* %s (%s)' % (n, eid) for n, eid in res.items()))
+ print()
+ print('enter the eid of a to group to map to each missing native group')
+ print('or just type enter to skip permissions granted to a group')
for group in missing:
while True:
value = raw_input('eid for group %s: ' % group).strip()
@@ -62,13 +65,13 @@
try:
eid = int(value)
except ValueError:
- print 'eid should be an integer'
+ print('eid should be an integer')
continue
for eid_ in res.values():
if eid == eid_:
break
else:
- print 'eid is not a group eid'
+ print('eid is not a group eid')
continue
res[name] = eid
break
@@ -146,7 +149,7 @@
{'x': etype, 'n': netype})
cnx.commit(False)
tocleanup = [eid]
- tocleanup += (eid for eid, cached in repo._type_source_cache.iteritems()
+ tocleanup += (eid for eid, cached in repo._type_source_cache.items()
if etype == cached[0])
repo.clear_caches(tocleanup)
cnx.commit(False)
@@ -240,7 +243,7 @@
'order', 'description', 'indexed', 'fulltextindexed',
'internationalizable', 'default', 'formula'), values))
typeparams = extra_props.get(attrs['rdefeid'])
- attrs.update(json.load(typeparams) if typeparams else {})
+ attrs.update(json.loads(typeparams.getvalue().decode('ascii')) if typeparams else {})
default = attrs['default']
if default is not None:
if isinstance(default, Binary):
@@ -281,7 +284,7 @@
else:
rtype = str(rel)
relations[1].append(rtype)
- for eschema, unique_together in unique_togethers.itervalues():
+ for eschema, unique_together in unique_togethers.values():
eschema._unique_together.append(tuple(sorted(unique_together)))
schema.infer_specialization_rules()
cnx.commit()
@@ -331,7 +334,7 @@
thispermsdict = permsidx[erschema.eid]
except KeyError:
return
- for action, somethings in thispermsdict.iteritems():
+ for action, somethings in thispermsdict.items():
erschema.permissions[action] = tuple(
isinstance(p, tuple) and erschema.rql_expression(*p) or p
for p in somethings)
@@ -344,7 +347,7 @@
current schema
"""
_title = '-> storing the schema in the database '
- print _title,
+ print(_title, end=' ')
execute = cnx.execute
eschemas = schema.entities()
pb_size = (len(eschemas + schema.relations())
@@ -366,7 +369,7 @@
cstrtypemap = {}
rql = 'INSERT CWConstraintType X: X name %(ct)s'
for cstrtype in CONSTRAINTS:
- cstrtypemap[cstrtype] = execute(rql, {'ct': unicode(cstrtype)},
+ cstrtypemap[cstrtype] = execute(rql, {'ct': text_type(cstrtype)},
build_descr=False)[0][0]
pb.update()
# serialize relations
@@ -381,10 +384,10 @@
continue
execschemarql(execute, rschema, rschema2rql(rschema, addrdef=False))
if rschema.symmetric:
- rdefs = [rdef for k, rdef in rschema.rdefs.iteritems()
+ rdefs = [rdef for k, rdef in rschema.rdefs.items()
if (rdef.subject, rdef.object) == k]
else:
- rdefs = rschema.rdefs.itervalues()
+ rdefs = rschema.rdefs.values()
for rdef in rdefs:
execschemarql(execute, rdef,
rdef2rql(rdef, cstrtypemap, groupmap))
@@ -397,7 +400,7 @@
for rql, kwargs in specialize2rql(schema):
execute(rql, kwargs, build_descr=False)
pb.update()
- print
+ print()
# high level serialization functions
@@ -455,8 +458,8 @@
columnset = set()
for columns in eschema._unique_together:
if columns in columnset:
- print ('schemaserial: skipping duplicate unique together %r %r' %
- (eschema.type, columns))
+ print('schemaserial: skipping duplicate unique together %r %r' %
+ (eschema.type, columns))
continue
columnset.add(columns)
rql, args = _uniquetogether2rql(eschema, columns)
@@ -471,7 +474,7 @@
for i, name in enumerate(unique_together):
rschema = eschema.schema.rschema(name)
rtype = 'T%d' % i
- substs[rtype] = unicode(rschema.type)
+ substs[rtype] = text_type(rschema.type)
relations.append('C relations %s' % rtype)
restrictions.append('%(rtype)s name %%(%(rtype)s)s' % {'rtype': rtype})
relations = ', '.join(relations)
@@ -483,11 +486,11 @@
def _ervalues(erschema):
try:
- type_ = unicode(erschema.type)
+ type_ = text_type(erschema.type)
except UnicodeDecodeError as e:
raise Exception("can't decode %s [was %s]" % (erschema.type, e))
try:
- desc = unicode(erschema.description) or u''
+ desc = text_type(erschema.description) or u''
except UnicodeDecodeError as e:
raise Exception("can't decode %s [was %s]" % (erschema.description, e))
return {
@@ -509,7 +512,7 @@
if addrdef:
assert cstrtypemap
# sort for testing purpose
- for rdef in sorted(rschema.rdefs.itervalues(),
+ for rdef in sorted(rschema.rdefs.values(),
key=lambda x: (x.subject, x.object)):
for rql, values in rdef2rql(rdef, cstrtypemap, groupmap):
yield rql, values
@@ -519,7 +522,7 @@
values['final'] = rschema.final
values['symmetric'] = rschema.symmetric
values['inlined'] = rschema.inlined
- if isinstance(rschema.fulltext_container, str):
+ if PY2 and isinstance(rschema.fulltext_container, str):
values['fulltext_container'] = unicode(rschema.fulltext_container)
else:
values['fulltext_container'] = rschema.fulltext_container
@@ -535,7 +538,7 @@
def crschema_relations_values(crschema):
values = _ervalues(crschema)
- values['rule'] = unicode(crschema.rule)
+ values['rule'] = text_type(crschema.rule)
# XXX why oh why?
del values['final']
relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]
@@ -581,20 +584,20 @@
value = bool(value)
elif prop == 'ordernum':
value = int(value)
- elif isinstance(value, str):
+ elif PY2 and isinstance(value, str):
value = unicode(value)
if value is not None and prop == 'default':
value = Binary.zpickle(value)
values[amap.get(prop, prop)] = value
if extra:
- values['extra_props'] = Binary(json.dumps(extra))
+ values['extra_props'] = Binary(json.dumps(extra).encode('ascii'))
relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]
return relations, values
def constraints2rql(cstrtypemap, constraints, rdefeid=None):
for constraint in constraints:
values = {'ct': cstrtypemap[constraint.type()],
- 'value': unicode(constraint.serialize()),
+ 'value': text_type(constraint.serialize()),
'x': rdefeid} # when not specified, will have to be set by the caller
yield 'INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE \
CT eid %(ct)s, EDEF eid %(x)s', values
@@ -613,23 +616,23 @@
# may occurs when modifying persistent schema
continue
for group_or_rqlexpr in grantedto:
- if isinstance(group_or_rqlexpr, basestring):
+ if isinstance(group_or_rqlexpr, string_types):
# group
try:
yield ('SET X %s_permission Y WHERE Y eid %%(g)s, X eid %%(x)s' % action,
{'g': groupmap[group_or_rqlexpr]})
except KeyError:
- print ("WARNING: group %s used in permissions for %s was ignored because it doesn't exist."
- " You may want to add it into a precreate.py file" % (group_or_rqlexpr, erschema))
+ print("WARNING: group %s used in permissions for %s was ignored because it doesn't exist."
+ " You may want to add it into a precreate.py file" % (group_or_rqlexpr, erschema))
continue
else:
# rqlexpr
rqlexpr = group_or_rqlexpr
yield ('INSERT RQLExpression E: E expression %%(e)s, E exprtype %%(t)s, '
'E mainvars %%(v)s, X %s_permission E WHERE X eid %%(x)s' % action,
- {'e': unicode(rqlexpr.expression),
- 'v': unicode(','.join(sorted(rqlexpr.mainvars))),
- 't': unicode(rqlexpr.__class__.__name__)})
+ {'e': text_type(rqlexpr.expression),
+ 'v': text_type(','.join(sorted(rqlexpr.mainvars))),
+ 't': text_type(rqlexpr.__class__.__name__)})
# update functions
@@ -641,7 +644,7 @@
def updaterschema2rql(rschema, eid):
if rschema.rule:
yield ('SET X rule %(r)s WHERE X eid %(x)s',
- {'x': eid, 'r': unicode(rschema.rule)})
+ {'x': eid, 'r': text_type(rschema.rule)})
else:
relations, values = rschema_relations_values(rschema)
values['x'] = eid
--- a/server/serverconfig.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/serverconfig.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,12 +16,14 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""server.serverconfig definition"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
import sys
from os.path import join, exists
-from StringIO import StringIO
+
+from six.moves import StringIO
import logilab.common.configuration as lgconfig
from logilab.common.decorators import cached
@@ -234,18 +236,19 @@
def bootstrap_cubes(self):
from logilab.common.textutils import splitstrip
- for line in file(join(self.apphome, 'bootstrap_cubes')):
- line = line.strip()
- if not line or line.startswith('#'):
- continue
- self.init_cubes(self.expand_cubes(splitstrip(line)))
- break
- else:
- # no cubes
- self.init_cubes(())
+ with open(join(self.apphome, 'bootstrap_cubes')) as f:
+ for line in f:
+ line = line.strip()
+ if not line or line.startswith('#'):
+ continue
+ self.init_cubes(self.expand_cubes(splitstrip(line)))
+ break
+ else:
+ # no cubes
+ self.init_cubes(())
def write_bootstrap_cubes_file(self, cubes):
- stream = file(join(self.apphome, 'bootstrap_cubes'), 'w')
+ stream = open(join(self.apphome, 'bootstrap_cubes'), 'w')
stream.write('# this is a generated file only used for bootstraping\n')
stream.write('# you should not have to edit this\n')
stream.write('%s\n' % ','.join(cubes))
@@ -276,7 +279,7 @@
assert len(self.sources_mode) == 1
if source.connect_for_migration:
return True
- print 'not connecting to source', source.uri, 'during migration'
+ print('not connecting to source', source.uri, 'during migration')
return False
if 'all' in self.sources_mode:
assert len(self.sources_mode) == 1
--- a/server/serverctl.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/serverctl.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""cubicweb-ctl commands and command handlers specific to the repository"""
+from __future__ import print_function
__docformat__ = 'restructuredtext en'
@@ -28,6 +29,9 @@
import logging
import subprocess
+from six import string_types
+from six.moves import input
+
from logilab.common import nullobject
from logilab.common.configuration import Configuration, merge_options
from logilab.common.shellutils import ASK, generate_password
@@ -55,27 +59,27 @@
driver = source['db-driver']
dbhelper = get_db_helper(driver)
if interactive:
- print '-> connecting to %s database' % driver,
+ print('-> connecting to %s database' % driver, end=' ')
if dbhost:
- print '%s@%s' % (dbname, dbhost),
+ print('%s@%s' % (dbname, dbhost), end=' ')
else:
- print dbname,
+ print(dbname, end=' ')
if dbhelper.users_support:
if not interactive or (not special_privs and source.get('db-user')):
user = source.get('db-user', os.environ.get('USER', ''))
if interactive:
- print 'as', user
+ print('as', user)
password = source.get('db-password')
else:
- print
+ print()
if special_privs:
- print 'WARNING'
+ print('WARNING')
print ('the user will need the following special access rights '
'on the database:')
- print special_privs
- print
+ print(special_privs)
+ print()
default_user = source.get('db-user', os.environ.get('USER', ''))
- user = raw_input('Connect as user ? [%r]: ' % default_user)
+ user = input('Connect as user ? [%r]: ' % default_user)
user = user.strip() or default_user
if user == source.get('db-user'):
password = source.get('db-password')
@@ -146,7 +150,7 @@
cnx = repoapi.connect(repo, login, password=pwd)
return repo, cnx
except AuthenticationError:
- print '-> Error: wrong user/password.'
+ print('-> Error: wrong user/password.')
# reset cubes else we'll have an assertion error on next retry
config._cubes = None
login, pwd = manager_userpasswd()
@@ -164,9 +168,9 @@
"""
config = self.config
if not automatic:
- print underline_title('Configuring the repository')
+ print(underline_title('Configuring the repository'))
config.input_config('email', inputlevel)
- print '\n'+underline_title('Configuring the sources')
+ print('\n'+underline_title('Configuring the sources'))
sourcesfile = config.sources_file()
# hack to make Method('default_instance_id') usable in db option defs
# (in native.py)
@@ -174,12 +178,12 @@
options=SOURCE_TYPES['native'].options)
if not automatic:
sconfig.input_config(inputlevel=inputlevel)
- print
+ print()
sourcescfg = {'system': sconfig}
if automatic:
# XXX modify a copy
password = generate_password()
- print '-> set administrator account to admin / %s' % password
+ print('-> set administrator account to admin / %s' % password)
USER_OPTIONS[1][1]['default'] = password
sconfig = Configuration(options=USER_OPTIONS)
else:
@@ -197,8 +201,8 @@
CWCTL.run(['db-create', '--config-level', str(inputlevel),
self.config.appid])
else:
- print ('-> nevermind, you can do it later with '
- '"cubicweb-ctl db-create %s".' % self.config.appid)
+ print('-> nevermind, you can do it later with '
+ '"cubicweb-ctl db-create %s".' % self.config.appid)
@contextmanager
@@ -242,26 +246,26 @@
with db_transaction(source, privilege='DROP SCHEMA') as cursor:
helper = get_db_helper(source['db-driver'])
helper.drop_schema(cursor, db_namespace)
- print '-> database schema %s dropped' % db_namespace
+ print('-> database schema %s dropped' % db_namespace)
def _drop_database(self, source):
dbname = source['db-name']
if source['db-driver'] == 'sqlite':
- print 'deleting database file %(db-name)s' % source
+ print('deleting database file %(db-name)s' % source)
os.unlink(source['db-name'])
- print '-> database %(db-name)s dropped.' % source
+ print('-> database %(db-name)s dropped.' % source)
else:
helper = get_db_helper(source['db-driver'])
with db_sys_transaction(source, privilege='DROP DATABASE') as cursor:
- print 'dropping database %(db-name)s' % source
+ print('dropping database %(db-name)s' % source)
cursor.execute('DROP DATABASE "%(db-name)s"' % source)
- print '-> database %(db-name)s dropped.' % source
+ print('-> database %(db-name)s dropped.' % source)
def _drop_user(self, source):
user = source['db-user'] or None
if user is not None:
with db_sys_transaction(source, privilege='DROP USER') as cursor:
- print 'dropping user %s' % user
+ print('dropping user %s' % user)
cursor.execute('DROP USER %s' % user)
def _cleanup_steps(self, source):
@@ -288,7 +292,7 @@
try:
step(source)
except Exception as exc:
- print 'ERROR', exc
+ print('ERROR', exc)
if ASK.confirm('An error occurred. Continue anyway?',
default_is_yes=False):
continue
@@ -357,7 +361,7 @@
ASK.confirm('Database %s already exists. Drop it?' % dbname)):
os.unlink(dbname)
elif self.config.create_db:
- print '\n'+underline_title('Creating the system database')
+ print('\n'+underline_title('Creating the system database'))
# connect on the dbms system base to create our base
dbcnx = _db_sys_cnx(source, 'CREATE/DROP DATABASE and / or USER',
interactive=not automatic)
@@ -368,17 +372,17 @@
if not helper.user_exists(cursor, user) and (automatic or \
ASK.confirm('Create db user %s ?' % user, default_is_yes=False)):
helper.create_user(source['db-user'], source.get('db-password'))
- print '-> user %s created.' % user
+ print('-> user %s created.' % user)
if dbname in helper.list_databases(cursor):
if automatic or ASK.confirm('Database %s already exists -- do you want to drop it ?' % dbname):
cursor.execute('DROP DATABASE "%s"' % dbname)
else:
- print ('you may want to run "cubicweb-ctl db-init '
- '--drop %s" manually to continue.' % config.appid)
+ print('you may want to run "cubicweb-ctl db-init '
+ '--drop %s" manually to continue.' % config.appid)
return
createdb(helper, source, dbcnx, cursor)
dbcnx.commit()
- print '-> database %s created.' % dbname
+ print('-> database %s created.' % dbname)
except BaseException:
dbcnx.rollback()
raise
@@ -400,13 +404,13 @@
try:
helper.create_language(cursor, extlang)
except Exception as exc:
- print '-> ERROR:', exc
- print '-> could not create language %s, some stored procedures might be unusable' % extlang
+ print('-> ERROR:', exc)
+ print('-> could not create language %s, some stored procedures might be unusable' % extlang)
cnx.rollback()
else:
cnx.commit()
- print '-> database for instance %s created and necessary extensions installed.' % appid
- print
+ print('-> database for instance %s created and necessary extensions installed.' % appid)
+ print()
if automatic:
CWCTL.run(['db-init', '--automatic', '--config-level', '0',
config.appid])
@@ -414,8 +418,8 @@
CWCTL.run(['db-init', '--config-level',
str(self.config.config_level), config.appid])
else:
- print ('-> nevermind, you can do it later with '
- '"cubicweb-ctl db-init %s".' % config.appid)
+ print('-> nevermind, you can do it later with '
+ '"cubicweb-ctl db-init %s".' % config.appid)
class InitInstanceCommand(Command):
@@ -452,7 +456,7 @@
def run(self, args):
check_options_consistency(self.config)
- print '\n'+underline_title('Initializing the system database')
+ print('\n'+underline_title('Initializing the system database'))
from cubicweb.server import init_repository
appid = args[0]
config = ServerConfiguration.config_for(appid)
@@ -503,10 +507,10 @@
used = set(n for n, in cnx.execute('Any SN WHERE S is CWSource, S name SN'))
cubes = repo.get_cubes()
while True:
- type = raw_input('source type (%s): '
+ type = input('source type (%s): '
% ', '.join(sorted(SOURCE_TYPES)))
if type not in SOURCE_TYPES:
- print '-> unknown source type, use one of the available types.'
+ print('-> unknown source type, use one of the available types.')
continue
sourcemodule = SOURCE_TYPES[type].module
if not sourcemodule.startswith('cubicweb.'):
@@ -520,23 +524,23 @@
continue
break
while True:
- parser = raw_input('parser type (%s): '
+ parser = input('parser type (%s): '
% ', '.join(sorted(repo.vreg['parsers'])))
if parser in repo.vreg['parsers']:
break
- print '-> unknown parser identifier, use one of the available types.'
+ print('-> unknown parser identifier, use one of the available types.')
while True:
- sourceuri = raw_input('source identifier (a unique name used to '
+ sourceuri = input('source identifier (a unique name used to '
'tell sources apart): ').strip()
if not sourceuri:
- print '-> mandatory.'
+ print('-> mandatory.')
else:
sourceuri = unicode(sourceuri, sys.stdin.encoding)
if sourceuri in used:
- print '-> uri already used, choose another one.'
+ print('-> uri already used, choose another one.')
else:
break
- url = raw_input('source URL (leave empty for none): ').strip()
+ url = input('source URL (leave empty for none): ').strip()
url = unicode(url) if url else None
# XXX configurable inputlevel
sconfig = ask_source_config(config, type, inputlevel=self.config.config_level)
@@ -583,10 +587,10 @@
cnx.rollback()
import traceback
traceback.print_exc()
- print '-> an error occurred:', ex
+ print('-> an error occurred:', ex)
else:
cnx.commit()
- print '-> rights granted to %s on instance %s.' % (appid, user)
+ print('-> rights granted to %s on instance %s.' % (appid, user))
class ResetAdminPasswordCommand(Command):
@@ -617,7 +621,7 @@
try:
adminlogin = sourcescfg['admin']['login']
except KeyError:
- print '-> Error: could not get cubicweb administrator login.'
+ print('-> Error: could not get cubicweb administrator login.')
sys.exit(1)
cnx = source_cnx(sourcescfg['system'])
driver = sourcescfg['system']['db-driver']
@@ -627,9 +631,9 @@
cursor.execute("SELECT * FROM cw_CWUser WHERE cw_login=%(l)s",
{'l': adminlogin})
if not cursor.fetchall():
- print ("-> error: admin user %r specified in sources doesn't exist "
- "in the database" % adminlogin)
- print " fix your sources file before running this command"
+ print("-> error: admin user %r specified in sources doesn't exist "
+ "in the database" % adminlogin)
+ print(" fix your sources file before running this command")
cnx.close()
sys.exit(1)
if self.config.password is None:
@@ -650,10 +654,10 @@
cnx.rollback()
import traceback
traceback.print_exc()
- print '-> an error occurred:', ex
+ print('-> an error occurred:', ex)
else:
cnx.commit()
- print '-> password reset, sources file regenerated.'
+ print('-> password reset, sources file regenerated.')
cnx.close()
@@ -666,17 +670,17 @@
if sudo:
dmpcmd = 'sudo %s' % (dmpcmd)
dmpcmd = 'ssh -t %s "%s"' % (host, dmpcmd)
- print dmpcmd
+ print(dmpcmd)
if os.system(dmpcmd):
raise ExecutionError('Error while dumping the database')
if output is None:
output = filename
cmd = 'scp %s:/tmp/%s %s' % (host, filename, output)
- print cmd
+ print(cmd)
if os.system(cmd):
raise ExecutionError('Error while retrieving the dump at /tmp/%s' % filename)
rmcmd = 'ssh -t %s "rm -f /tmp/%s"' % (host, filename)
- print rmcmd
+ print(rmcmd)
if os.system(rmcmd) and not ASK.confirm(
'An error occurred while deleting remote dump at /tmp/%s. '
'Continue anyway?' % filename):
@@ -686,7 +690,7 @@
def _local_dump(appid, output, format='native'):
config = ServerConfiguration.config_for(appid)
config.quick_start = True
- mih = config.migration_handler(connect=False, verbosity=1)
+ mih = config.migration_handler(verbosity=1)
mih.backup_database(output, askconfirm=False, format=format)
mih.shutdown()
@@ -696,28 +700,28 @@
config.quick_start = True
mih = config.migration_handler(connect=False, verbosity=1)
mih.restore_database(backupfile, drop, askconfirm=False, format=format)
- repo = mih.repo_connect()
+ repo = mih.repo
# version of the database
dbversions = repo.get_versions()
mih.shutdown()
if not dbversions:
- print "bad or missing version information in the database, don't upgrade file system"
+ print("bad or missing version information in the database, don't upgrade file system")
return
# version of installed software
eversion = dbversions['cubicweb']
status = instance_status(config, eversion, dbversions)
# * database version > installed software
if status == 'needsoftupgrade':
- print "** The database of %s is more recent than the installed software!" % config.appid
- print "** Upgrade your software, then migrate the database by running the command"
- print "** 'cubicweb-ctl upgrade %s'" % config.appid
+ print("** The database of %s is more recent than the installed software!" % config.appid)
+ print("** Upgrade your software, then migrate the database by running the command")
+ print("** 'cubicweb-ctl upgrade %s'" % config.appid)
return
# * database version < installed software, an upgrade will be necessary
# anyway, just rewrite vc.conf and warn user he has to upgrade
elif status == 'needapplupgrade':
- print "** The database of %s is older than the installed software." % config.appid
- print "** Migrate the database by running the command"
- print "** 'cubicweb-ctl upgrade %s'" % config.appid
+ print("** The database of %s is older than the installed software." % config.appid)
+ print("** Migrate the database by running the command")
+ print("** 'cubicweb-ctl upgrade %s'" % config.appid)
return
# * database version = installed software, database version = instance fs version
# ok!
@@ -732,12 +736,12 @@
try:
softversion = config.cube_version(cube)
except ConfigurationError:
- print '-> Error: no cube version information for %s, please check that the cube is installed.' % cube
+ print('-> Error: no cube version information for %s, please check that the cube is installed.' % cube)
continue
try:
applversion = vcconf[cube]
except KeyError:
- print '-> Error: no cube version information for %s in version configuration.' % cube
+ print('-> Error: no cube version information for %s in version configuration.' % cube)
continue
if softversion == applversion:
continue
@@ -883,7 +887,7 @@
_local_restore(destappid, output, not self.config.no_drop,
self.config.format)
if self.config.keep_dump:
- print '-> you can get the dump file at', output
+ print('-> you can get the dump file at', output)
else:
os.remove(output)
@@ -1001,9 +1005,9 @@
stats = source.pull_data(cnx, force=True, raise_on_error=True)
finally:
repo.shutdown()
- for key, val in stats.iteritems():
+ for key, val in stats.items():
if val:
- print key, ':', val
+ print(key, ':', val)
@@ -1019,7 +1023,7 @@
for p in ('read', 'add', 'update', 'delete'):
rule = perms.get(p)
if rule:
- perms[p] = tuple(str(x) if isinstance(x, basestring) else x
+ perms[p] = tuple(str(x) if isinstance(x, string_types) else x
for x in rule)
return perms, perms in defaultrelperms or perms in defaulteperms
@@ -1079,7 +1083,7 @@
if self.config.db is not None:
appcfg = ServerConfiguration.config_for(appid)
srccfg = appcfg.read_sources_file()
- for key, value in self.config.db.iteritems():
+ for key, value in self.config.db.items():
if '.' in key:
section, key = key.split('.', 1)
else:
--- a/server/session.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/session.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,8 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""Repository users' and internal' sessions."""
+from __future__ import print_function
+
__docformat__ = "restructuredtext en"
import sys
@@ -25,6 +27,8 @@
import functools
from contextlib import contextmanager
+from six import text_type
+
from logilab.common.deprecation import deprecated
from logilab.common.textutils import unormalize
from logilab.common.registry import objectify_predicate
@@ -556,7 +560,7 @@
else:
relations_dict[rtype] = eids
self.repo.glob_add_relations(self, relations_dict)
- for edited in edited_entities.itervalues():
+ for edited in edited_entities.values():
self.repo.glob_update_entity(self, edited)
@@ -769,7 +773,7 @@
def transaction_uuid(self, set=True):
uuid = self.transaction_data.get('tx_uuid')
if set and uuid is None:
- self.transaction_data['tx_uuid'] = uuid = unicode(uuid4().hex)
+ self.transaction_data['tx_uuid'] = uuid = text_type(uuid4().hex)
self.repo.system_source.start_undoable_transaction(self, uuid)
return uuid
@@ -874,7 +878,7 @@
processed = []
self.commit_state = 'precommit'
if debug:
- print self.commit_state, '*' * 20
+ print(self.commit_state, '*' * 20)
try:
with self.running_hooks_ops():
while self.pending_operations:
@@ -882,7 +886,7 @@
operation.processed = 'precommit'
processed.append(operation)
if debug:
- print operation
+ print(operation)
operation.handle_event('precommit_event')
self.pending_operations[:] = processed
self.debug('precommit transaction %s done', self.connectionid)
@@ -899,11 +903,11 @@
# and revertcommit, that will be enough in mont case.
operation.failed = True
if debug:
- print self.commit_state, '*' * 20
+ print(self.commit_state, '*' * 20)
with self.running_hooks_ops():
for operation in reversed(processed):
if debug:
- print operation
+ print(operation)
try:
operation.handle_event('revertprecommit_event')
except BaseException:
@@ -917,12 +921,12 @@
self.cnxset.commit()
self.commit_state = 'postcommit'
if debug:
- print self.commit_state, '*' * 20
+ print(self.commit_state, '*' * 20)
with self.running_hooks_ops():
while self.pending_operations:
operation = self.pending_operations.pop(0)
if debug:
- print operation
+ print(operation)
operation.processed = 'postcommit'
try:
operation.handle_event('postcommit_event')
@@ -1004,7 +1008,7 @@
"""
def __init__(self, user, repo, cnxprops=None, _id=None):
- self.sessionid = _id or make_uid(unormalize(user.login).encode('UTF8'))
+ self.sessionid = _id or make_uid(unormalize(user.login))
self.user = user # XXX repoapi: deprecated and store only a login.
self.repo = repo
self.vreg = repo.vreg
--- a/server/sources/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/sources/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,13 +16,18 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""cubicweb server sources support"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
from time import time
from logging import getLogger
+from base64 import b64decode
+
+from six import text_type
from logilab.common import configuration
+from logilab.common.textutils import unormalize
from logilab.common.deprecation import deprecated
from yams.schema import role_name
@@ -35,25 +40,25 @@
def dbg_st_search(uri, union, varmap, args, cachekey=None, prefix='rql for'):
if server.DEBUG & server.DBG_RQL:
global t
- print ' %s %s source: %s' % (prefix, uri, repr(union.as_string()))
+ print(' %s %s source: %s' % (prefix, uri, repr(union.as_string())))
t = time()
if varmap:
- print ' using varmap', varmap
+ print(' using varmap', varmap)
if server.DEBUG & server.DBG_MORE:
- print ' args', repr(args)
- print ' cache key', cachekey
- print ' solutions', ','.join(str(s.solutions)
- for s in union.children)
+ print(' args', repr(args))
+ print(' cache key', cachekey)
+ print(' solutions', ','.join(str(s.solutions)
+ for s in union.children))
# return true so it can be used as assertion (and so be killed by python -O)
return True
def dbg_results(results):
if server.DEBUG & server.DBG_RQL:
if len(results) > 10:
- print ' -->', results[:10], '...', len(results),
+ print(' -->', results[:10], '...', len(results), end=' ')
else:
- print ' -->', results,
- print 'time: ', time() - t
+ print(' -->', results, end=' ')
+ print('time: ', time() - t)
# return true so it can be used as assertion (and so be killed by python -O)
return True
@@ -104,7 +109,9 @@
self.public_config['use-cwuri-as-url'] = self.use_cwuri_as_url
self.remove_sensitive_information(self.public_config)
self.uri = source_config.pop('uri')
- set_log_methods(self, getLogger('cubicweb.sources.'+self.uri))
+ # unormalize to avoid non-ascii characters in logger's name, this will cause decoding error
+ # on logging
+ set_log_methods(self, getLogger('cubicweb.sources.' + unormalize(text_type(self.uri))))
source_config.pop('type')
self.update_config(None, self.check_conf_dict(eid, source_config,
fail_if_unknown=False))
@@ -140,7 +147,7 @@
pass
@classmethod
- def check_conf_dict(cls, eid, confdict, _=unicode, fail_if_unknown=True):
+ def check_conf_dict(cls, eid, confdict, _=text_type, fail_if_unknown=True):
"""check configuration of source entity. Return config dict properly
typed with defaults set.
"""
@@ -157,7 +164,7 @@
try:
value = configuration._validate(value, optdict, optname)
except Exception as ex:
- msg = unicode(ex) # XXX internationalization
+ msg = text_type(ex) # XXX internationalization
raise ValidationError(eid, {role_name('config', 'subject'): msg})
processed[optname] = value
# cw < 3.10 bw compat
@@ -199,6 +206,12 @@
else:
self.urls = []
+ @staticmethod
+ def decode_extid(extid):
+ if extid is None:
+ return extid
+ return b64decode(extid)
+
# source initialization / finalization #####################################
def set_schema(self, schema):
--- a/server/sources/datafeed.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/sources/datafeed.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,15 +19,20 @@
database
"""
-import urllib2
-import StringIO
+from io import BytesIO
from os.path import exists
from datetime import datetime, timedelta
-from base64 import b64decode
-from cookielib import CookieJar
-import urlparse
+
+from six import text_type
+from six.moves.urllib.parse import urlparse
+from six.moves.urllib.request import Request, build_opener, HTTPCookieProcessor
+from six.moves.urllib.error import HTTPError
+from six.moves.http_cookiejar import CookieJar
+
from lxml import etree
+from logilab.common.deprecation import deprecated
+
from cubicweb import RegistryNotFound, ObjectNotFound, ValidationError, UnknownEid
from cubicweb.server.repository import preprocess_inlined_relations
from cubicweb.server.sources import AbstractSource
@@ -244,6 +249,7 @@
error = True
return error
+ @deprecated('[3.21] use the new store API')
def before_entity_insertion(self, cnx, lid, etype, eid, sourceparams):
"""called by the repository when an eid has been attributed for an
entity stored here but the entity has not been inserted in the system
@@ -259,6 +265,7 @@
sourceparams['parser'].before_entity_copy(entity, sourceparams)
return entity
+ @deprecated('[3.21] use the new store API')
def after_entity_insertion(self, cnx, lid, entity, sourceparams):
"""called by the repository after an entity stored here has been
inserted in the system table.
@@ -282,7 +289,7 @@
sql = ('SELECT extid, eid, type FROM entities, cw_source_relation '
'WHERE entities.eid=cw_source_relation.eid_from '
'AND cw_source_relation.eid_to=%s' % self.eid)
- return dict((b64decode(uri), (eid, type))
+ return dict((self.decode_extid(uri), (eid, type))
for uri, eid, type in cnx.system_sql(sql).fetchall())
def init_import_log(self, cnx, **kwargs):
@@ -328,7 +335,7 @@
For http URLs, it will try to find a cwclientlib config entry
(if available) and use it as requester.
"""
- purl = urlparse.urlparse(url)
+ purl = urlparse(url)
if purl.scheme == 'file':
return URLLibResponseAdapter(open(url[7:]), url)
@@ -344,7 +351,7 @@
self.source.info('Using cwclientlib for %s' % url)
resp = cnx.get(url)
resp.raise_for_status()
- return URLLibResponseAdapter(StringIO.StringIO(resp.text), url)
+ return URLLibResponseAdapter(BytesIO(resp.text), url)
except (ImportError, ValueError, EnvironmentError) as exc:
# ImportError: not available
# ValueError: no config entry found
@@ -354,11 +361,11 @@
# no chance with cwclientlib, fall back to former implementation
if purl.scheme in ('http', 'https'):
self.source.info('GET %s', url)
- req = urllib2.Request(url)
+ req = Request(url)
return _OPENER.open(req, timeout=self.source.http_timeout)
# url is probably plain content
- return URLLibResponseAdapter(StringIO.StringIO(url), url)
+ return URLLibResponseAdapter(BytesIO(url.encode('ascii')), url)
def add_schema_config(self, schemacfg, checkonly=False):
"""added CWSourceSchemaConfig, modify mapping accordingly"""
@@ -370,6 +377,7 @@
msg = schemacfg._cw._("this parser doesn't use a mapping")
raise ValidationError(schemacfg.eid, {None: msg})
+ @deprecated('[3.21] use the new store API')
def extid2entity(self, uri, etype, **sourceparams):
"""Return an entity for the given uri. May return None if it should be
skipped.
@@ -388,11 +396,11 @@
else:
source = self.source
sourceparams['parser'] = self
- if isinstance(uri, unicode):
+ if isinstance(uri, text_type):
uri = uri.encode('utf-8')
try:
- eid = cnx.repo.extid2eid(source, str(uri), etype, cnx,
- sourceparams=sourceparams)
+ eid = cnx.repo.extid2eid(source, uri, etype, cnx,
+ sourceparams=sourceparams)
except ValidationError as ex:
if raise_on_error:
raise
@@ -419,9 +427,11 @@
"""main callback: process the url"""
raise NotImplementedError
+ @deprecated('[3.21] use the new store API')
def before_entity_copy(self, entity, sourceparams):
raise NotImplementedError
+ @deprecated('[3.21] use the new store API')
def after_entity_copy(self, entity, sourceparams):
self.stats['created'].add(entity.eid)
@@ -447,10 +457,10 @@
def handle_deletion(self, config, cnx, myuris):
if config['delete-entities'] and myuris:
byetype = {}
- for extid, (eid, etype) in myuris.iteritems():
+ for extid, (eid, etype) in myuris.items():
if self.is_deleted(extid, etype, eid):
byetype.setdefault(etype, []).append(str(eid))
- for etype, eids in byetype.iteritems():
+ for etype, eids in byetype.items():
self.warning('delete %s %s entities', len(eids), etype)
cnx.execute('DELETE %s X WHERE X eid IN (%s)'
% (etype, ','.join(eids)))
@@ -463,7 +473,7 @@
self.notify_checked(entity)
mdate = attrs.get('modification_date')
if not mdate or mdate > entity.modification_date:
- attrs = dict( (k, v) for k, v in attrs.iteritems()
+ attrs = dict( (k, v) for k, v in attrs.items()
if v != getattr(entity, k))
if attrs:
entity.cw_set(**attrs)
@@ -472,6 +482,7 @@
class DataFeedXMLParser(DataFeedParser):
+ @deprecated()
def process(self, url, raise_on_error=False):
"""IDataFeedParser main entry point"""
try:
@@ -530,10 +541,10 @@
self.source.debug(str(exc))
# no chance with cwclientlib, fall back to former implementation
- if urlparse.urlparse(url).scheme in ('http', 'https'):
+ if urlparse(url).scheme in ('http', 'https'):
try:
_OPENER.open(url, timeout=self.source.http_timeout)
- except urllib2.HTTPError as ex:
+ except HTTPError as ex:
if ex.code == 404:
return True
return False
@@ -555,15 +566,12 @@
def getcode(self):
return self.code
- def info(self):
- from mimetools import Message
- return Message(StringIO.StringIO())
# use a cookie enabled opener to use session cookie if any
-_OPENER = urllib2.build_opener()
+_OPENER = build_opener()
try:
from logilab.common import urllib2ext
_OPENER.add_handler(urllib2ext.HTTPGssapiAuthHandler())
except ImportError: # python-kerberos not available
pass
-_OPENER.add_handler(urllib2.HTTPCookieProcessor(CookieJar()))
+_OPENER.add_handler(HTTPCookieProcessor(CookieJar()))
--- a/server/sources/ldapfeed.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/sources/ldapfeed.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -17,14 +17,13 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""cubicweb ldap feed source"""
-from __future__ import division # XXX why?
+from __future__ import division # XXX why?
from datetime import datetime
-import ldap
-from ldap.ldapobject import ReconnectLDAPObject
-from ldap.filter import filter_format
-from ldapurl import LDAPUrl
+from six import PY2, string_types
+
+import ldap3
from logilab.common.configuration import merge_options
@@ -32,15 +31,15 @@
from cubicweb.server import utils
from cubicweb.server.sources import datafeed
-_ = unicode
+from cubicweb import _
# search scopes
-BASE = ldap.SCOPE_BASE
-ONELEVEL = ldap.SCOPE_ONELEVEL
-SUBTREE = ldap.SCOPE_SUBTREE
-LDAP_SCOPES = {'BASE': ldap.SCOPE_BASE,
- 'ONELEVEL': ldap.SCOPE_ONELEVEL,
- 'SUBTREE': ldap.SCOPE_SUBTREE}
+BASE = ldap3.SEARCH_SCOPE_BASE_OBJECT
+ONELEVEL = ldap3.SEARCH_SCOPE_SINGLE_LEVEL
+SUBTREE = ldap3.SEARCH_SCOPE_WHOLE_SUBTREE
+LDAP_SCOPES = {'BASE': BASE,
+ 'ONELEVEL': ONELEVEL,
+ 'SUBTREE': SUBTREE}
# map ldap protocol to their standard port
PROTO_PORT = {'ldap': 389,
@@ -49,6 +48,15 @@
}
+def replace_filter(s):
+ s = s.replace('*', '\\2A')
+ s = s.replace('(', '\\28')
+ s = s.replace(')', '\\29')
+ s = s.replace('\\', '\\5c')
+ s = s.replace('\0', '\\00')
+ return s
+
+
class LDAPFeedSource(datafeed.DataFeedSource):
"""LDAP feed source: unlike ldapuser source, this source is copy based and
will import ldap content (beside passwords for authentication) into the
@@ -61,7 +69,7 @@
('auth-mode',
{'type' : 'choice',
'default': 'simple',
- 'choices': ('simple', 'cram_md5', 'digest_md5', 'gssapi'),
+ 'choices': ('simple', 'digest_md5', 'gssapi'),
'help': 'authentication mode used to authenticate user to the ldap.',
'group': 'ldap-source', 'level': 3,
}),
@@ -183,8 +191,8 @@
self.user_default_groups = typedconfig['user-default-group']
self.user_attrs = {'dn': 'eid', 'modifyTimestamp': 'modification_date'}
self.user_attrs.update(typedconfig['user-attrs-map'])
- self.user_rev_attrs = dict((v, k) for k, v in self.user_attrs.iteritems())
- self.base_filters = [filter_format('(%s=%s)', ('objectClass', o))
+ self.user_rev_attrs = dict((v, k) for k, v in self.user_attrs.items())
+ self.base_filters = ['(objectclass=%s)' % replace_filter(o)
for o in typedconfig['user-classes']]
if typedconfig['user-filter']:
self.base_filters.append(typedconfig['user-filter'])
@@ -193,8 +201,8 @@
self.group_attrs = typedconfig['group-attrs-map']
self.group_attrs = {'dn': 'eid', 'modifyTimestamp': 'modification_date'}
self.group_attrs.update(typedconfig['group-attrs-map'])
- self.group_rev_attrs = dict((v, k) for k, v in self.group_attrs.iteritems())
- self.group_base_filters = [filter_format('(%s=%s)', ('objectClass', o))
+ self.group_rev_attrs = dict((v, k) for k, v in self.group_attrs.items())
+ self.group_base_filters = ['(objectClass=%s)' % replace_filter(o)
for o in typedconfig['group-classes']]
if typedconfig['group-filter']:
self.group_base_filters.append(typedconfig['group-filter'])
@@ -215,9 +223,11 @@
def connection_info(self):
assert len(self.urls) == 1, self.urls
protocol, hostport = self.urls[0].split('://')
- if protocol != 'ldapi' and not ':' in hostport:
- hostport = '%s:%s' % (hostport, PROTO_PORT[protocol])
- return protocol, hostport
+ if protocol != 'ldapi' and ':' in hostport:
+ host, port = hostport.rsplit(':', 1)
+ else:
+ host, port = hostport, PROTO_PORT[protocol]
+ return protocol, host, port
def authenticate(self, cnx, login, password=None, **kwargs):
"""return CWUser eid for the given login/password if this account is
@@ -232,59 +242,43 @@
# You get Authenticated as: 'NT AUTHORITY\ANONYMOUS LOGON'.
# we really really don't want that
raise AuthenticationError()
- searchfilter = [filter_format('(%s=%s)', (self.user_login_attr, login))]
+ searchfilter = ['(%s=%s)' % (replace_filter(self.user_login_attr), replace_filter(login))]
searchfilter.extend(self.base_filters)
searchstr = '(&%s)' % ''.join(searchfilter)
# first search the user
try:
user = self._search(cnx, self.user_base_dn,
self.user_base_scope, searchstr)[0]
- except (IndexError, ldap.SERVER_DOWN):
+ except IndexError:
# no such user
raise AuthenticationError()
# check password by establishing a (unused) connection
try:
self._connect(user, password)
- except ldap.LDAPError as ex:
+ except ldap3.LDAPException as ex:
# Something went wrong, most likely bad credentials
self.info('while trying to authenticate %s: %s', user, ex)
raise AuthenticationError()
except Exception:
self.error('while trying to authenticate %s', user, exc_info=True)
raise AuthenticationError()
- eid = self.repo.extid2eid(self, user['dn'], 'CWUser', cnx, insert=False)
- if eid < 0:
- # user has been moved away from this source
+ eid = self.repo.system_source.extid2eid(cnx, user['dn'].encode('ascii'))
+ if eid is None or eid < 0:
+ # user is not known or has been moved away from this source
raise AuthenticationError()
return eid
def _connect(self, user=None, userpwd=None):
- protocol, hostport = self.connection_info()
- self.info('connecting %s://%s as %s', protocol, hostport,
+ protocol, host, port = self.connection_info()
+ self.info('connecting %s://%s:%s as %s', protocol, host, port,
user and user['dn'] or 'anonymous')
- # don't require server certificate when using ldaps (will
- # enable self signed certs)
- ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
- url = LDAPUrl(urlscheme=protocol, hostport=hostport)
- conn = ReconnectLDAPObject(url.initializeUrl())
- # Set the protocol version - version 3 is preferred
- try:
- conn.set_option(ldap.OPT_PROTOCOL_VERSION, ldap.VERSION3)
- except ldap.LDAPError: # Invalid protocol version, fall back safely
- conn.set_option(ldap.OPT_PROTOCOL_VERSION, ldap.VERSION2)
- # Deny auto-chasing of referrals to be safe, we handle them instead
- # Required for AD
- try:
- conn.set_option(ldap.OPT_REFERRALS, 0)
- except ldap.LDAPError: # Cannot set referrals, so do nothing
- pass
- #conn.set_option(ldap.OPT_NETWORK_TIMEOUT, conn_timeout)
- #conn.timeout = op_timeout
+ server = ldap3.Server(host, port=int(port))
+ conn = ldap3.Connection(server, user=user and user['dn'], client_strategy=ldap3.STRATEGY_SYNC_RESTARTABLE, auto_referrals=False)
# Now bind with the credentials given. Let exceptions propagate out.
if user is None:
# XXX always use simple bind for data connection
if not self.cnx_dn:
- conn.simple_bind_s(self.cnx_dn, self.cnx_pwd)
+ conn.bind()
else:
self._authenticate(conn, {'dn': self.cnx_dn}, self.cnx_pwd)
else:
@@ -294,25 +288,22 @@
return conn
def _auth_simple(self, conn, user, userpwd):
- conn.simple_bind_s(user['dn'], userpwd)
-
- def _auth_cram_md5(self, conn, user, userpwd):
- from ldap import sasl
- auth_token = sasl.cram_md5(user['dn'], userpwd)
- conn.sasl_interactive_bind_s('', auth_token)
+ conn.authentication = ldap3.AUTH_SIMPLE
+ conn.user = user['dn']
+ conn.password = userpwd
+ conn.bind()
def _auth_digest_md5(self, conn, user, userpwd):
- from ldap import sasl
- auth_token = sasl.digest_md5(user['dn'], userpwd)
- conn.sasl_interactive_bind_s('', auth_token)
+ conn.authentication = ldap3.AUTH_SASL
+ conn.sasl_mechanism = 'DIGEST-MD5'
+ # realm, user, password, authz-id
+ conn.sasl_credentials = (None, user['dn'], userpwd, None)
+ conn.bind()
def _auth_gssapi(self, conn, user, userpwd):
- # print XXX not proper sasl/gssapi
- import kerberos
- if not kerberos.checkPassword(user[self.user_login_attr], userpwd):
- raise Exception('BAD login / mdp')
- #from ldap import sasl
- #conn.sasl_interactive_bind_s('', sasl.gssapi())
+ conn.authentication = ldap3.AUTH_SASL
+ conn.sasl_mechanism = 'GSSAPI'
+ conn.bind()
def _search(self, cnx, base, scope,
searchstr='(objectClass=*)', attrs=()):
@@ -322,37 +313,15 @@
if self._conn is None:
self._conn = self._connect()
ldapcnx = self._conn
- try:
- res = ldapcnx.search_s(base, scope, searchstr, attrs)
- except ldap.PARTIAL_RESULTS:
- res = ldapcnx.result(all=0)[1]
- except ldap.NO_SUCH_OBJECT:
- self.info('ldap NO SUCH OBJECT %s %s %s', base, scope, searchstr)
- self._process_no_such_object(cnx, base)
+ if not ldapcnx.search(base, searchstr, search_scope=scope, attributes=attrs):
return []
- # except ldap.REFERRAL as e:
- # ldapcnx = self.handle_referral(e)
- # try:
- # res = ldapcnx.search_s(base, scope, searchstr, attrs)
- # except ldap.PARTIAL_RESULTS:
- # res_type, res = ldapcnx.result(all=0)
result = []
- for rec_dn, rec_dict in res:
- # When used against Active Directory, "rec_dict" may not be
- # be a dictionary in some cases (instead, it can be a list)
- #
- # An example of a useless "res" entry that can be ignored
- # from AD is
- # (None, ['ldap://ForestDnsZones.PORTAL.LOCAL/DC=ForestDnsZones,DC=PORTAL,DC=LOCAL'])
- # This appears to be some sort of internal referral, but
- # we can't handle it, so we need to skip over it.
- try:
- items = rec_dict.iteritems()
- except AttributeError:
+ for rec in ldapcnx.response:
+ if rec['type'] != 'searchResEntry':
continue
- else:
- itemdict = self._process_ldap_item(rec_dn, items)
- result.append(itemdict)
+ items = rec['attributes'].items()
+ itemdict = self._process_ldap_item(rec['dn'], items)
+ result.append(itemdict)
self.debug('ldap built results %s', len(result))
return result
@@ -363,20 +332,21 @@
if self.user_attrs.get(key) == 'upassword': # XXx better password detection
value = value[0].encode('utf-8')
# we only support ldap_salted_sha1 for ldap sources, see: server/utils.py
- if not value.startswith('{SSHA}'):
+ if not value.startswith(b'{SSHA}'):
value = utils.crypt_password(value)
itemdict[key] = Binary(value)
elif self.user_attrs.get(key) == 'modification_date':
itemdict[key] = datetime.strptime(value[0], '%Y%m%d%H%M%SZ')
else:
- value = [unicode(val, 'utf-8', 'replace') for val in value]
+ if PY2 and value and isinstance(value[0], str):
+ value = [unicode(val, 'utf-8', 'replace') for val in value]
if len(value) == 1:
itemdict[key] = value = value[0]
else:
itemdict[key] = value
# we expect memberUid to be a list of user ids, make sure of it
member = self.group_rev_attrs['member']
- if isinstance(itemdict.get(member), basestring):
+ if isinstance(itemdict.get(member), string_types):
itemdict[member] = [itemdict[member]]
return itemdict
--- a/server/sources/native.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/sources/native.py Wed Dec 16 11:23:48 2015 +0100
@@ -23,13 +23,13 @@
string. This is because it should actually be Bytes but we want an index on
it for fast querying.
"""
+from __future__ import print_function
+
__docformat__ = "restructuredtext en"
-from cPickle import loads, dumps
-import cPickle as pickle
from threading import Lock
from datetime import datetime
-from base64 import b64decode, b64encode
+from base64 import b64encode
from contextlib import contextmanager
from os.path import basename
import re
@@ -38,6 +38,9 @@
import logging
import sys
+from six import PY2, text_type, binary_type, string_types
+from six.moves import range, cPickle as pickle
+
from logilab.common.decorators import cached, clear_cache
from logilab.common.configuration import Method
from logilab.common.shellutils import getlogin
@@ -76,12 +79,12 @@
it's a function just so that it shows up in profiling
"""
if server.DEBUG & server.DBG_SQL:
- print 'exec', query, args
+ print('exec', query, args)
try:
self.cu.execute(str(query), args)
except Exception as ex:
- print "sql: %r\n args: %s\ndbms message: %r" % (
- query, args, ex.args[0])
+ print("sql: %r\n args: %s\ndbms message: %r" % (
+ query, args, ex.args[0]))
raise
def fetchall(self):
@@ -134,7 +137,7 @@
Type of _UndoException message must be `unicode` by design in CubicWeb.
"""
- assert isinstance(self.args[0], unicode)
+ assert isinstance(self.args[0], text_type)
return self.args[0]
@@ -556,7 +559,7 @@
sql, qargs, cbs = self._rql_sqlgen.generate(union, args, varmap)
self._cache[cachekey] = sql, qargs, cbs
args = self.merge_args(args, qargs)
- assert isinstance(sql, basestring), repr(sql)
+ assert isinstance(sql, string_types), repr(sql)
cursor = self.doexec(cnx, sql, args)
results = self.process_result(cursor, cnx, cbs)
assert dbg_results(results)
@@ -611,7 +614,7 @@
self.doexec(cnx, sql, attrs)
if cnx.ertype_supports_undo(entity.cw_etype):
self._record_tx_action(cnx, 'tx_entity_actions', u'C',
- etype=unicode(entity.cw_etype), eid=entity.eid)
+ etype=text_type(entity.cw_etype), eid=entity.eid)
def update_entity(self, cnx, entity):
"""replace an entity in the source"""
@@ -620,8 +623,8 @@
if cnx.ertype_supports_undo(entity.cw_etype):
changes = self._save_attrs(cnx, entity, attrs)
self._record_tx_action(cnx, 'tx_entity_actions', u'U',
- etype=unicode(entity.cw_etype), eid=entity.eid,
- changes=self._binary(dumps(changes)))
+ etype=text_type(entity.cw_etype), eid=entity.eid,
+ changes=self._binary(pickle.dumps(changes)))
sql = self.sqlgen.update(SQL_PREFIX + entity.cw_etype, attrs,
['cw_eid'])
self.doexec(cnx, sql, attrs)
@@ -635,8 +638,8 @@
if (r.final or r.inlined) and not r in VIRTUAL_RTYPES]
changes = self._save_attrs(cnx, entity, attrs)
self._record_tx_action(cnx, 'tx_entity_actions', u'D',
- etype=unicode(entity.cw_etype), eid=entity.eid,
- changes=self._binary(dumps(changes)))
+ etype=text_type(entity.cw_etype), eid=entity.eid,
+ changes=self._binary(pickle.dumps(changes)))
attrs = {'cw_eid': entity.eid}
sql = self.sqlgen.delete(SQL_PREFIX + entity.cw_etype, attrs)
self.doexec(cnx, sql, attrs)
@@ -646,7 +649,7 @@
self._add_relations(cnx, rtype, [(subject, object)], inlined)
if cnx.ertype_supports_undo(rtype):
self._record_tx_action(cnx, 'tx_relation_actions', u'A',
- eid_from=subject, rtype=unicode(rtype), eid_to=object)
+ eid_from=subject, rtype=text_type(rtype), eid_to=object)
def add_relations(self, cnx, rtype, subj_obj_list, inlined=False):
"""add a relations to the source"""
@@ -654,7 +657,7 @@
if cnx.ertype_supports_undo(rtype):
for subject, object in subj_obj_list:
self._record_tx_action(cnx, 'tx_relation_actions', u'A',
- eid_from=subject, rtype=unicode(rtype), eid_to=object)
+ eid_from=subject, rtype=text_type(rtype), eid_to=object)
def _add_relations(self, cnx, rtype, subj_obj_list, inlined=False):
"""add a relation to the source"""
@@ -671,7 +674,7 @@
etypes[etype].append((subject, object))
else:
etypes[etype] = [(subject, object)]
- for subj_etype, subj_obj_list in etypes.iteritems():
+ for subj_etype, subj_obj_list in etypes.items():
attrs = [{'cw_eid': subject, SQL_PREFIX + rtype: object}
for subject, object in subj_obj_list]
sql.append((self.sqlgen.update(SQL_PREFIX + etype, attrs[0],
@@ -686,7 +689,7 @@
self._delete_relation(cnx, subject, rtype, object, rschema.inlined)
if cnx.ertype_supports_undo(rtype):
self._record_tx_action(cnx, 'tx_relation_actions', u'R',
- eid_from=subject, rtype=unicode(rtype), eid_to=object)
+ eid_from=subject, rtype=text_type(rtype), eid_to=object)
def _delete_relation(self, cnx, subject, rtype, object, inlined=False):
"""delete a relation from the source"""
@@ -708,7 +711,7 @@
"""
cursor = cnx.cnxset.cu
if server.DEBUG & server.DBG_SQL:
- print 'exec', query, args, cnx.cnxset.cnx
+ print('exec', query, args, cnx.cnxset.cnx)
try:
# str(query) to avoid error if it's a unicode string
cursor.execute(str(query), args)
@@ -767,7 +770,7 @@
it's a function just so that it shows up in profiling
"""
if server.DEBUG & server.DBG_SQL:
- print 'execmany', query, 'with', len(args), 'arguments', cnx.cnxset.cnx
+ print('execmany', query, 'with', len(args), 'arguments', cnx.cnxset.cnx)
cursor = cnx.cnxset.cu
try:
# str(query) to avoid error if it's a unicode string
@@ -852,10 +855,9 @@
"""return a tuple (type, extid, source) for the entity with id <eid>"""
sql = 'SELECT type, extid, asource FROM entities WHERE eid=%s' % eid
res = self._eid_type_source(cnx, eid, sql)
- if res[-2] is not None:
- if not isinstance(res, list):
- res = list(res)
- res[-2] = b64decode(res[-2])
+ if not isinstance(res, list):
+ res = list(res)
+ res[-2] = self.decode_extid(res[-2])
return res
def eid_type_source_pre_131(self, cnx, eid):
@@ -864,15 +866,14 @@
res = self._eid_type_source(cnx, eid, sql)
if not isinstance(res, list):
res = list(res)
- if res[-1] is not None:
- res[-1] = b64decode(res[-1])
+ res[-1] = self.decode_extid(res[-1])
res.append("system")
return res
def extid2eid(self, cnx, extid):
"""get eid from an external id. Return None if no record found."""
- assert isinstance(extid, str)
- args = {'x': b64encode(extid)}
+ assert isinstance(extid, binary_type)
+ args = {'x': b64encode(extid).decode('ascii')}
cursor = self.doexec(cnx,
'SELECT eid FROM entities WHERE extid=%(x)s',
args)
@@ -911,10 +912,10 @@
assert cnx.cnxset is not None
# begin by inserting eid/type/source/extid into the entities table
if extid is not None:
- assert isinstance(extid, str)
- extid = b64encode(extid)
- attrs = {'type': unicode(entity.cw_etype), 'eid': entity.eid, 'extid': extid and unicode(extid),
- 'asource': unicode(source.uri)}
+ assert isinstance(extid, binary_type)
+ extid = b64encode(extid).decode('ascii')
+ attrs = {'type': text_type(entity.cw_etype), 'eid': entity.eid, 'extid': extid,
+ 'asource': text_type(source.uri)}
self._handle_insert_entity_sql(cnx, self.sqlgen.insert('entities', attrs), attrs)
# insert core relations: is, is_instance_of and cw_source
try:
@@ -975,13 +976,13 @@
if actionfilters.pop('public', True):
genrestr['txa_public'] = True
# put additional filters in trarestr and/or tearestr
- for key, val in actionfilters.iteritems():
+ for key, val in actionfilters.items():
if key == 'etype':
# filtering on etype implies filtering on entity actions
# only, and with no eid specified
assert actionfilters.get('action', 'C') in 'CUD'
assert not 'eid' in actionfilters
- tearestr['etype'] = unicode(val)
+ tearestr['etype'] = text_type(val)
elif key == 'eid':
# eid filter may apply to 'eid' of tx_entity_actions or to
# 'eid_from' OR 'eid_to' of tx_relation_actions
@@ -992,10 +993,10 @@
trarestr['eid_to'] = val
elif key == 'action':
if val in 'CUD':
- tearestr['txa_action'] = unicode(val)
+ tearestr['txa_action'] = text_type(val)
else:
assert val in 'AR'
- trarestr['txa_action'] = unicode(val)
+ trarestr['txa_action'] = text_type(val)
else:
raise AssertionError('unknow filter %s' % key)
assert trarestr or tearestr, "can't only filter on 'public'"
@@ -1029,11 +1030,11 @@
def tx_info(self, cnx, txuuid):
"""See :class:`cubicweb.repoapi.Connection.transaction_info`"""
- return tx.Transaction(cnx, txuuid, *self._tx_info(cnx, unicode(txuuid)))
+ return tx.Transaction(cnx, txuuid, *self._tx_info(cnx, text_type(txuuid)))
def tx_actions(self, cnx, txuuid, public):
"""See :class:`cubicweb.repoapi.Connection.transaction_actions`"""
- txuuid = unicode(txuuid)
+ txuuid = text_type(txuuid)
self._tx_info(cnx, txuuid)
restr = {'tx_uuid': txuuid}
if public:
@@ -1044,7 +1045,7 @@
'etype', 'eid', 'changes'))
with cnx.ensure_cnx_set:
cu = self.doexec(cnx, sql, restr)
- actions = [tx.EntityAction(a,p,o,et,e,c and loads(self.binary_to_str(c)))
+ actions = [tx.EntityAction(a,p,o,et,e,c and pickle.loads(self.binary_to_str(c)))
for a,p,o,et,e,c in cu.fetchall()]
sql = self.sqlgen.select('tx_relation_actions', restr,
('txa_action', 'txa_public', 'txa_order',
@@ -1168,8 +1169,8 @@
elif eschema.destination(rtype) in ('Bytes', 'Password'):
changes[column] = self._binary(value)
edited[rtype] = Binary(value)
- elif isinstance(value, str):
- edited[rtype] = unicode(value, cnx.encoding, 'replace')
+ elif PY2 and isinstance(value, str):
+ edited[rtype] = text_type(value, cnx.encoding, 'replace')
else:
edited[rtype] = value
# This must only be done after init_entitiy_caches : defered in calling functions
@@ -1210,14 +1211,14 @@
try:
sentity, oentity, rdef = _undo_rel_info(cnx, subj, rtype, obj)
except _UndoException as ex:
- errors.append(unicode(ex))
+ errors.append(text_type(ex))
else:
for role, entity in (('subject', sentity),
('object', oentity)):
try:
_undo_check_relation_target(entity, rdef, role)
except _UndoException as ex:
- errors.append(unicode(ex))
+ errors.append(text_type(ex))
continue
if not errors:
self.repo.hm.call_hooks('before_add_relation', cnx,
@@ -1293,7 +1294,7 @@
try:
sentity, oentity, rdef = _undo_rel_info(cnx, subj, rtype, obj)
except _UndoException as ex:
- errors.append(unicode(ex))
+ errors.append(text_type(ex))
else:
rschema = rdef.rtype
if rschema.inlined:
@@ -1544,7 +1545,7 @@
SQL_PREFIX + 'CWUser',
SQL_PREFIX + 'upassword',
SQL_PREFIX + 'login'),
- {'newhash': self.source._binary(newhash),
+ {'newhash': self.source._binary(newhash.encode('ascii')),
'login': login})
cnx.commit()
return user
@@ -1692,7 +1693,7 @@
self.logger.info('number of rows: %d', rowcount)
blocksize = self.blocksize
if rowcount > 0:
- for i, start in enumerate(xrange(0, rowcount, blocksize)):
+ for i, start in enumerate(range(0, rowcount, blocksize)):
rows = list(itertools.islice(rows_iterator, blocksize))
serialized = self._serialize(table, columns, rows)
archive.writestr('tables/%s.%04d' % (table, i), serialized)
@@ -1713,7 +1714,7 @@
return tuple(columns), rows
def _serialize(self, name, columns, rows):
- return dumps((name, columns, rows), pickle.HIGHEST_PROTOCOL)
+ return pickle.dumps((name, columns, rows), pickle.HIGHEST_PROTOCOL)
def restore(self, backupfile):
archive = zipfile.ZipFile(backupfile, 'r', allowZip64=True)
@@ -1761,7 +1762,7 @@
return sequences, numranges, tables, table_chunks
def read_sequence(self, archive, seq):
- seqname, columns, rows = loads(archive.read('sequences/%s' % seq))
+ seqname, columns, rows = pickle.loads(archive.read('sequences/%s' % seq))
assert seqname == seq
assert len(rows) == 1
assert len(rows[0]) == 1
@@ -1771,7 +1772,7 @@
self.cnx.commit()
def read_numrange(self, archive, numrange):
- rangename, columns, rows = loads(archive.read('numrange/%s' % numrange))
+ rangename, columns, rows = pickle.loads(archive.read('numrange/%s' % numrange))
assert rangename == numrange
assert len(rows) == 1
assert len(rows[0]) == 1
@@ -1786,7 +1787,7 @@
self.cnx.commit()
row_count = 0
for filename in filenames:
- tablename, columns, rows = loads(archive.read(filename))
+ tablename, columns, rows = pickle.loads(archive.read(filename))
assert tablename == table
if not rows:
continue
--- a/server/sources/rql2sql.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/sources/rql2sql.py Wed Dec 16 11:23:48 2015 +0100
@@ -51,6 +51,9 @@
import threading
+from six import PY2
+from six.moves import range
+
from logilab.database import FunctionDescr, SQL_FUNCTIONS_REGISTRY
from rql import BadRQLQuery, CoercionError
@@ -172,7 +175,7 @@
existssols = {}
unstable = set()
invariants = {}
- for vname, var in rqlst.defined_vars.iteritems():
+ for vname, var in rqlst.defined_vars.items():
vtype = newsols[0][vname]
if var._q_invariant or vname in varmap:
# remove invariant variable from solutions to remove duplicates
@@ -187,13 +190,13 @@
thisexistssols = [newsols[0]]
thisexistsvars = set()
existssols[var.scope] = thisexistssols, thisexistsvars
- for i in xrange(len(newsols)-1, 0, -1):
+ for i in range(len(newsols)-1, 0, -1):
if vtype != newsols[i][vname]:
thisexistssols.append(newsols.pop(i))
thisexistsvars.add(vname)
else:
# remember unstable variables
- for i in xrange(1, len(newsols)):
+ for i in range(1, len(newsols)):
if vtype != newsols[i][vname]:
unstable.add(vname)
if invariants:
@@ -205,11 +208,11 @@
newsols = newsols_
# reinsert solutions for invariants
for sol in newsols:
- for invvar, vartype in invariants[id(sol)].iteritems():
+ for invvar, vartype in invariants[id(sol)].items():
sol[invvar] = vartype
for sol in existssols:
try:
- for invvar, vartype in invariants[id(sol)].iteritems():
+ for invvar, vartype in invariants[id(sol)].items():
sol[invvar] = vartype
except KeyError:
continue
@@ -257,7 +260,7 @@
append(term)
if groups:
for vref in term.iget_nodes(VariableRef):
- if not vref in groups:
+ if not any(vref.is_equivalent(g) for g in groups):
groups.append(vref)
def fix_selection_and_group(rqlst, needwrap, selectsortterms,
@@ -273,7 +276,7 @@
(isinstance(term, Function) and
get_func_descr(term.name).aggregat)):
for vref in term.iget_nodes(VariableRef):
- if not vref in groupvrefs:
+ if not any(vref.is_equivalent(group) for group in groupvrefs):
groups.append(vref)
groupvrefs.append(vref)
if needwrap and (groups or having):
@@ -364,7 +367,7 @@
self.done = set()
self.tables = self.subtables.copy()
self.actual_tables = [[]]
- for _, tsql in self.tables.itervalues():
+ for _, tsql in self.tables.values():
self.actual_tables[-1].append(tsql)
self.outer_chains = []
self.outer_tables = {}
@@ -398,7 +401,7 @@
notdone_outside_vars = set()
# when iterating other solutions inner to an EXISTS subquery, we should
# reset variables which have this exists node as scope at each iteration
- for var in exists.stmt.defined_vars.itervalues():
+ for var in exists.stmt.defined_vars.values():
if var.scope is exists:
thisexistsvars.add(var.name)
elif var.name not in self.done:
@@ -600,7 +603,7 @@
self.outer_chains.remove(lchain)
rchain += lchain
self.mark_as_used_in_outer_join(leftalias)
- for alias, (aouter, aconditions, achain) in outer_tables.iteritems():
+ for alias, (aouter, aconditions, achain) in outer_tables.items():
if achain is lchain:
outer_tables[alias] = (aouter, aconditions, rchain)
else:
@@ -1475,7 +1478,7 @@
"""generate SQL name for a function"""
if func.name == 'FTIRANK':
try:
- rel = iter(func.children[0].variable.stinfo['ftirels']).next()
+ rel = next(iter(func.children[0].variable.stinfo['ftirels']))
except KeyError:
raise BadRQLQuery("can't use FTIRANK on variable not used in an"
" 'has_text' relation (eg full-text search)")
@@ -1512,7 +1515,7 @@
return self._mapped_term(constant, '%%(%s)s' % value)[0]
except KeyError:
_id = value
- if isinstance(_id, unicode):
+ if PY2 and isinstance(_id, unicode):
_id = _id.encode()
else:
_id = str(id(constant)).replace('-', '', 1)
@@ -1561,7 +1564,7 @@
# add additional restriction on entities.type column
pts = variable.stinfo['possibletypes']
if len(pts) == 1:
- etype = iter(variable.stinfo['possibletypes']).next()
+ etype = next(iter(variable.stinfo['possibletypes']))
restr = "%s.type='%s'" % (vtablename, etype)
else:
etypes = ','.join("'%s'" % et for et in pts)
@@ -1609,7 +1612,7 @@
def _temp_table_scope(self, select, table):
scope = 9999
- for var, sql in self._varmap.iteritems():
+ for var, sql in self._varmap.items():
# skip "attribute variable" in varmap (such 'T.login')
if not '.' in var and table == sql.split('.', 1)[0]:
try:
@@ -1668,7 +1671,7 @@
except KeyError:
pass
rel = (variable.stinfo.get('principal') or
- iter(variable.stinfo['rhsrelations']).next())
+ next(iter(variable.stinfo['rhsrelations'])))
linkedvar = rel.children[0].variable
if rel.r_type == 'eid':
return linkedvar.accept(self)
--- a/server/sources/storages.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/sources/storages.py Wed Dec 16 11:23:48 2015 +0100
@@ -23,6 +23,10 @@
from contextlib import contextmanager
import tempfile
+from six import PY2, PY3, text_type, binary_type
+
+from logilab.common import nullobject
+
from yams.schema import role_name
from cubicweb import Binary, ValidationError
@@ -44,7 +48,7 @@
query result process of fetched attribute's value and should have the
following prototype::
- callback(self, source, session, value)
+ callback(self, source, cnx, value)
where `value` is the value actually stored in the backend. None values
will be skipped (eg callback won't be called).
@@ -92,24 +96,33 @@
return tempfile.mkstemp(prefix=base, suffix=ext, dir=dirpath)
@contextmanager
-def fsimport(session):
- present = 'fs_importing' in session.transaction_data
- old_value = session.transaction_data.get('fs_importing')
- session.transaction_data['fs_importing'] = True
+def fsimport(cnx):
+ present = 'fs_importing' in cnx.transaction_data
+ old_value = cnx.transaction_data.get('fs_importing')
+ cnx.transaction_data['fs_importing'] = True
yield
if present:
- session.transaction_data['fs_importing'] = old_value
+ cnx.transaction_data['fs_importing'] = old_value
else:
- del session.transaction_data['fs_importing']
+ del cnx.transaction_data['fs_importing']
+
+
+_marker = nullobject()
class BytesFileSystemStorage(Storage):
"""store Bytes attribute value on the file system"""
- def __init__(self, defaultdir, fsencoding='utf-8', wmode=0444):
- if type(defaultdir) is unicode:
- defaultdir = defaultdir.encode(fsencoding)
+ def __init__(self, defaultdir, fsencoding=_marker, wmode=0o444):
+ if PY3:
+ if not isinstance(defaultdir, text_type):
+ raise TypeError('defaultdir must be a unicode object in python 3')
+ if fsencoding is not _marker:
+ raise ValueError('fsencoding is no longer supported in python 3')
+ else:
+ self.fsencoding = fsencoding or 'utf-8'
+ if isinstance(defaultdir, text_type):
+ defaultdir = defaultdir.encode(fsencoding)
self.default_directory = defaultdir
- self.fsencoding = fsencoding
# extra umask to use when creating file
# 0444 as in "only allow read bit in permission"
self._wmode = wmode
@@ -126,7 +139,7 @@
fileobj.close()
- def callback(self, source, session, value):
+ def callback(self, source, cnx, value):
"""sql generator callback when some attribute with a custom storage is
accessed
"""
@@ -145,7 +158,8 @@
binary = entity.cw_edited.pop(attr)
fd, fpath = self.new_fs_path(entity, attr)
# bytes storage used to store file's path
- entity.cw_edited.edited_attribute(attr, Binary(fpath))
+ binary_obj = Binary(fpath if PY2 else fpath.encode('utf-8'))
+ entity.cw_edited.edited_attribute(attr, binary_obj)
self._writecontent(fd, binary)
AddFileOp.get_instance(entity._cw).add_data(fpath)
return binary
@@ -187,7 +201,8 @@
entity.cw_edited.edited_attribute(attr, None)
else:
# register the new location for the file.
- entity.cw_edited.edited_attribute(attr, Binary(fpath))
+ binary_obj = Binary(fpath if PY2 else fpath.encode('utf-8'))
+ entity.cw_edited.edited_attribute(attr, binary_obj)
if oldpath is not None and oldpath != fpath:
# Mark the old file as useless so the file will be removed at
# commit.
@@ -206,16 +221,19 @@
# available. Keeping the extension is useful for example in the case of
# PIL processing that use filename extension to detect content-type, as
# well as providing more understandable file names on the fs.
+ if PY2:
+ attr = attr.encode('ascii')
basename = [str(entity.eid), attr]
name = entity.cw_attr_metadata(attr, 'name')
if name is not None:
- basename.append(name.encode(self.fsencoding))
+ basename.append(name.encode(self.fsencoding) if PY2 else name)
fd, fspath = uniquify_path(self.default_directory,
'_'.join(basename))
if fspath is None:
msg = entity._cw._('failed to uniquify path (%s, %s)') % (
self.default_directory, '_'.join(basename))
raise ValidationError(entity.eid, {role_name(attr, 'subject'): msg})
+ assert isinstance(fspath, str) # bytes on py2, unicode on py3
return fd, fspath
def current_fs_path(self, entity, attr):
@@ -229,34 +247,40 @@
rawvalue = cu.fetchone()[0]
if rawvalue is None: # no previous value
return None
- return sysource._process_value(rawvalue, cu.description[0],
- binarywrap=str)
+ fspath = sysource._process_value(rawvalue, cu.description[0],
+ binarywrap=binary_type)
+ if PY3:
+ fspath = fspath.decode('utf-8')
+ assert isinstance(fspath, str) # bytes on py2, unicode on py3
+ return fspath
def migrate_entity(self, entity, attribute):
"""migrate an entity attribute to the storage"""
entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache)
self.entity_added(entity, attribute)
- session = entity._cw
- source = session.repo.system_source
+ cnx = entity._cw
+ source = cnx.repo.system_source
attrs = source.preprocess_entity(entity)
sql = source.sqlgen.update('cw_' + entity.cw_etype, attrs,
['cw_eid'])
- source.doexec(session, sql, attrs)
+ source.doexec(cnx, sql, attrs)
entity.cw_edited = None
class AddFileOp(hook.DataOperationMixIn, hook.Operation):
def rollback_event(self):
for filepath in self.get_data():
+ assert isinstance(filepath, str) # bytes on py2, unicode on py3
try:
unlink(filepath)
except Exception as ex:
- self.error('cant remove %s: %s' % (filepath, ex))
+ self.error("can't remove %s: %s" % (filepath, ex))
class DeleteFileOp(hook.DataOperationMixIn, hook.Operation):
def postcommit_event(self):
for filepath in self.get_data():
+ assert isinstance(filepath, str) # bytes on py2, unicode on py3
try:
unlink(filepath)
except Exception as ex:
- self.error('cant remove %s: %s' % (filepath, ex))
+ self.error("can't remove %s: %s" % (filepath, ex))
--- a/server/sqlutils.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/sqlutils.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""SQL utilities functions and classes."""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -23,10 +24,12 @@
import re
import subprocess
from os.path import abspath
-from itertools import ifilter
from logging import getLogger
from datetime import time, datetime
+from six import string_types, text_type
+from six.moves import filter
+
from logilab import database as db, common as lgc
from logilab.common.shellutils import ProgressBar, DummyProgressBar
from logilab.common.deprecation import deprecated
@@ -44,8 +47,12 @@
SQL_PREFIX = 'cw_'
def _run_command(cmd):
- print ' '.join(cmd)
- return subprocess.call(cmd)
+ if isinstance(cmd, string_types):
+ print(cmd)
+ return subprocess.call(cmd, shell=True)
+ else:
+ print(' '.join(cmd))
+ return subprocess.call(cmd)
def sqlexec(sqlstmts, cursor_or_execute, withpb=True,
@@ -69,7 +76,7 @@
else:
execute = cursor_or_execute
sqlstmts_as_string = False
- if isinstance(sqlstmts, basestring):
+ if isinstance(sqlstmts, string_types):
sqlstmts_as_string = True
sqlstmts = sqlstmts.split(delimiter)
if withpb:
@@ -87,7 +94,7 @@
try:
# some dbapi modules doesn't accept unicode for sql string
execute(str(sql))
- except Exception, err:
+ except Exception as err:
if cnx:
cnx.rollback()
failed.append(sql)
@@ -95,7 +102,7 @@
if cnx:
cnx.commit()
if withpb:
- print
+ print()
if sqlstmts_as_string:
failed = delimiter.join(failed)
return failed
@@ -178,9 +185,9 @@
# for mssql, we need to drop views before tables
if hasattr(dbhelper, 'list_views'):
cmds += ['DROP VIEW %s;' % name
- for name in ifilter(_SQL_DROP_ALL_USER_TABLES_FILTER_FUNCTION, dbhelper.list_views(sqlcursor))]
+ for name in filter(_SQL_DROP_ALL_USER_TABLES_FILTER_FUNCTION, dbhelper.list_views(sqlcursor))]
cmds += ['DROP TABLE %s;' % name
- for name in ifilter(_SQL_DROP_ALL_USER_TABLES_FILTER_FUNCTION, dbhelper.list_tables(sqlcursor))]
+ for name in filter(_SQL_DROP_ALL_USER_TABLES_FILTER_FUNCTION, dbhelper.list_tables(sqlcursor))]
return '\n'.join(cmds)
@@ -370,7 +377,7 @@
def merge_args(self, args, query_args):
if args is not None:
newargs = {}
- for key, val in args.iteritems():
+ for key, val in args.items():
# convert cubicweb binary into db binary
if isinstance(val, Binary):
val = self._binary(val.getvalue())
@@ -441,7 +448,7 @@
attrs = {}
eschema = entity.e_schema
converters = getattr(self.dbhelper, 'TYPE_CONVERTERS', {})
- for attr, value in entity.cw_edited.iteritems():
+ for attr, value in entity.cw_edited.items():
if value is not None and eschema.subjrels[attr].final:
atype = str(entity.e_schema.destination(attr))
if atype in converters:
@@ -481,7 +488,7 @@
if value is not None:
self.values.add(value)
def finalize(self):
- return ', '.join(unicode(v) for v in self.values)
+ return ', '.join(text_type(v) for v in self.values)
cnx.create_aggregate("GROUP_CONCAT", 1, group_concat)
--- a/server/ssplanner.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/ssplanner.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,6 +19,8 @@
__docformat__ = "restructuredtext en"
+from six import text_type
+
from rql.stmts import Union, Select
from rql.nodes import Constant, Relation
@@ -54,7 +56,7 @@
value = rhs.eval(plan.args)
eschema = edef.entity.e_schema
attrtype = eschema.subjrels[rtype].objects(eschema)[0]
- if attrtype == 'Password' and isinstance(value, unicode):
+ if attrtype == 'Password' and isinstance(value, text_type):
value = value.encode('UTF8')
edef.edited_attribute(rtype, value)
elif str(rhs) in to_build:
@@ -306,7 +308,7 @@
if varmap is None:
return varmap
maprepr = {}
- for var, sql in varmap.iteritems():
+ for var, sql in varmap.items():
table, col = sql.split('.')
maprepr[var] = '%s.%s' % (tablesinorder[table], col)
return maprepr
@@ -527,7 +529,7 @@
result[i] = newrow
# update entities
repo.glob_add_relations(cnx, relations)
- for eid, edited in edefs.iteritems():
+ for eid, edited in edefs.items():
repo.glob_update_entity(cnx, edited)
return result
--- a/server/test/data-cwep002/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data-cwep002/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -33,4 +33,3 @@
class has_employee(ComputedRelation):
rule = 'O works_for S'
__permissions__ = {'read': ('managers',)}
-
--- a/server/test/data-migractions/cubes/fakeemail/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data-migractions/cubes/fakeemail/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -5,7 +5,7 @@
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
# pylint: disable-msg=E0611,F0401
from yams.buildobjs import (SubjectRelation, RelationType, EntityType,
@@ -84,5 +84,3 @@
subject = 'Comment'
name = 'generated_by'
object = 'Email'
-
-
--- a/server/test/data-migractions/migratedapp/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data-migractions/migratedapp/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,6 +21,7 @@
SubjectRelation, Bytes,
RichString, String, Int, Boolean, Datetime, Date, Float)
from yams.constraints import SizeConstraint, UniqueConstraint
+from cubicweb import _
from cubicweb.schema import (WorkflowableEntityType, RQLConstraint,
RQLVocabularyConstraint,
ERQLExpression, RRQLExpression)
--- a/server/test/data-migractions/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data-migractions/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -24,6 +24,8 @@
RQLConstraint, RQLUniqueConstraint,
RQLVocabularyConstraint,
ERQLExpression, RRQLExpression)
+from cubicweb import _
+
class Affaire(WorkflowableEntityType):
__permissions__ = {
@@ -85,7 +87,7 @@
object = 'SubDivision'
from cubicweb.schemas.base import CWUser
-CWUser.get_relations('login').next().fulltextindexed = True
+next(CWUser.get_relations('login')).fulltextindexed = True
class Note(WorkflowableEntityType):
date = String(maxsize=10)
@@ -223,13 +225,13 @@
class ecrit_par_1(RelationDefinition):
name = 'ecrit_par'
subject = 'Note'
- object ='Personne'
+ object = 'Personne'
cardinality = '?*'
class ecrit_par_2(RelationDefinition):
name = 'ecrit_par'
subject = 'Note'
- object ='CWUser'
+ object = 'CWUser'
cardinality='?*'
--- a/server/test/data-schema2sql/schema/Dates.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data-schema2sql/schema/Dates.py Wed Dec 16 11:23:48 2015 +0100
@@ -26,4 +26,3 @@
d2 = Date(default=date(2007, 12, 11))
t1 = Time(default=time(8, 40))
t2 = Time(default=time(9, 45))
-
--- a/server/test/data-schema2sql/schema/State.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data-schema2sql/schema/State.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,7 +19,7 @@
SubjectRelation, Int, String, Boolean)
from yams.constraints import SizeConstraint, UniqueConstraint
-from __init__ import RESTRICTED_RTYPE_PERMS
+from . import RESTRICTED_RTYPE_PERMS
class State(EntityType):
"""used to associate simple states to an entity
--- a/server/test/data-schema2sql/schema/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data-schema2sql/schema/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -110,4 +110,3 @@
'add': ('managers',),
'delete': ('managers',),
}
-
--- a/server/test/data-schemaserial/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data-schemaserial/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -29,4 +29,3 @@
inline2 = SubjectRelation('Affaire', inlined=True, cardinality='?*')
custom_field_of_jungle = BabarTestType(jungle_speed=42)
-
--- a/server/test/data-schemaserial/site_cubicweb.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data-schemaserial/site_cubicweb.py Wed Dec 16 11:23:48 2015 +0100
@@ -27,4 +27,3 @@
def dumb_sort(something):
return something
register_sqlite_pyfunc(dumb_sort)
-
--- a/server/test/data/migration/postcreate.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data/migration/postcreate.py Wed Dec 16 11:23:48 2015 +0100
@@ -35,4 +35,3 @@
wf.add_transition(u'start', pitetre, encours)
wf.add_transition(u'end', encours, finie)
commit()
-
--- a/server/test/data/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/data/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -24,6 +24,7 @@
RQLConstraint, RQLUniqueConstraint,
RQLVocabularyConstraint,
ERQLExpression, RRQLExpression)
+from cubicweb import _
class Affaire(WorkflowableEntityType):
__permissions__ = {
@@ -85,7 +86,7 @@
object = 'SubDivision'
from cubicweb.schemas.base import CWUser
-CWUser.get_relations('login').next().fulltextindexed = True
+next(CWUser.get_relations('login')).fulltextindexed = True
class Note(WorkflowableEntityType):
date = String(maxsize=10)
--- a/server/test/datacomputed/migratedapp/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/datacomputed/migratedapp/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -59,3 +59,8 @@
class renamed(ComputedRelation):
rule = 'S employees E, O concerns E'
+
+
+class perm_changes(ComputedRelation):
+ __permissions__ = {'read': ('managers',)}
+ rule = 'S employees E, O concerns E'
--- a/server/test/datacomputed/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/datacomputed/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -58,3 +58,8 @@
class to_be_renamed(ComputedRelation):
rule = 'S employees E, O concerns E'
+
+
+class perm_changes(ComputedRelation):
+ __permissions__ = {'read': ('managers', 'users')}
+ rule = 'S employees E, O concerns E'
--- a/server/test/requirements.txt Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/requirements.txt Wed Dec 16 11:23:48 2015 +0100
@@ -1,4 +1,5 @@
psycopg2
+ldap3
cubicweb-basket
cubicweb-card
cubicweb-comment
--- a/server/test/unittest_checkintegrity.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_checkintegrity.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,7 +17,13 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
import sys
-from StringIO import StringIO
+
+from six import PY2
+if PY2:
+ from StringIO import StringIO
+else:
+ from io import StringIO
+
from logilab.common.testlib import TestCase, unittest_main
from cubicweb.devtools import get_test_db_handler, TestServerConfiguration
--- a/server/test/unittest_datafeed.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_datafeed.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,3 +1,4 @@
+# coding: utf-8
# copyright 2011-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
@@ -16,7 +17,6 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-import mimetools
from datetime import timedelta
from contextlib import contextmanager
@@ -28,7 +28,7 @@
def setup_database(self):
with self.admin_access.repo_cnx() as cnx:
with self.base_parser(cnx):
- cnx.create_entity('CWSource', name=u'myfeed', type=u'datafeed',
+ cnx.create_entity('CWSource', name=u'ô myfeed', type=u'datafeed',
parser=u'testparser', url=u'ignored',
config=u'synchronization-interval=1min')
cnx.commit()
@@ -48,21 +48,23 @@
entity.cw_edited.update(sourceparams['item'])
with self.temporary_appobjects(AParser):
- if 'myfeed' in self.repo.sources_by_uri:
- yield self.repo.sources_by_uri['myfeed']._get_parser(session)
+ if u'ô myfeed' in self.repo.sources_by_uri:
+ yield self.repo.sources_by_uri[u'ô myfeed']._get_parser(session)
else:
yield
def test(self):
- self.assertIn('myfeed', self.repo.sources_by_uri)
- dfsource = self.repo.sources_by_uri['myfeed']
+ self.assertIn(u'ô myfeed', self.repo.sources_by_uri)
+ dfsource = self.repo.sources_by_uri[u'ô myfeed']
self.assertNotIn('use_cwuri_as_url', dfsource.__dict__)
- self.assertEqual({'type': u'datafeed', 'uri': u'myfeed', 'use-cwuri-as-url': True},
+ self.assertEqual({'type': u'datafeed', 'uri': u'ô myfeed', 'use-cwuri-as-url': True},
dfsource.public_config)
self.assertEqual(dfsource.use_cwuri_as_url, True)
self.assertEqual(dfsource.latest_retrieval, None)
self.assertEqual(dfsource.synchro_interval, timedelta(seconds=60))
self.assertFalse(dfsource.fresh())
+ # ensure source's logger name has been unormalized
+ self.assertEqual(dfsource.info.__self__.name, 'cubicweb.sources.o myfeed')
with self.repo.internal_cnx() as cnx:
with self.base_parser(cnx):
@@ -78,17 +80,17 @@
self.assertEqual(entity.title, 'cubicweb.org')
self.assertEqual(entity.content, 'the cw web site')
self.assertEqual(entity.cwuri, 'http://www.cubicweb.org/')
- self.assertEqual(entity.cw_source[0].name, 'myfeed')
+ self.assertEqual(entity.cw_source[0].name, u'ô myfeed')
self.assertEqual(entity.cw_metainformation(),
{'type': 'Card',
- 'source': {'uri': 'myfeed', 'type': 'datafeed', 'use-cwuri-as-url': True},
- 'extid': 'http://www.cubicweb.org/'}
+ 'source': {'uri': u'ô myfeed', 'type': 'datafeed', 'use-cwuri-as-url': True},
+ 'extid': b'http://www.cubicweb.org/'}
)
self.assertEqual(entity.absolute_url(), 'http://www.cubicweb.org/')
# test repo cache keys
self.assertEqual(self.repo._type_source_cache[entity.eid],
- ('Card', 'http://www.cubicweb.org/', 'myfeed'))
- self.assertEqual(self.repo._extid_cache['http://www.cubicweb.org/'],
+ ('Card', b'http://www.cubicweb.org/', u'ô myfeed'))
+ self.assertEqual(self.repo._extid_cache[b'http://www.cubicweb.org/'],
entity.eid)
# test repull
stats = dfsource.pull_data(cnx, force=True)
@@ -101,19 +103,18 @@
self.assertEqual(stats['created'], set())
self.assertEqual(stats['updated'], set((entity.eid,)))
self.assertEqual(self.repo._type_source_cache[entity.eid],
- ('Card', 'http://www.cubicweb.org/', 'myfeed'))
- self.assertEqual(self.repo._extid_cache['http://www.cubicweb.org/'],
+ ('Card', b'http://www.cubicweb.org/', u'ô myfeed'))
+ self.assertEqual(self.repo._extid_cache[b'http://www.cubicweb.org/'],
entity.eid)
self.assertEqual(dfsource.source_cwuris(cnx),
- {'http://www.cubicweb.org/': (entity.eid, 'Card')}
- )
+ {b'http://www.cubicweb.org/': (entity.eid, 'Card')})
self.assertTrue(dfsource.latest_retrieval)
self.assertTrue(dfsource.fresh())
# test_rename_source
with self.admin_access.repo_cnx() as cnx:
- cnx.execute('SET S name "myrenamedfeed" WHERE S is CWSource, S name "myfeed"')
+ cnx.entity_from_eid(dfsource.eid).cw_set(name=u"myrenamedfeed")
cnx.commit()
entity = cnx.execute('Card X').get_entity(0, 0)
self.assertEqual(entity.cwuri, 'http://www.cubicweb.org/')
@@ -121,11 +122,11 @@
self.assertEqual(entity.cw_metainformation(),
{'type': 'Card',
'source': {'uri': 'myrenamedfeed', 'type': 'datafeed', 'use-cwuri-as-url': True},
- 'extid': 'http://www.cubicweb.org/'}
+ 'extid': b'http://www.cubicweb.org/'}
)
self.assertEqual(self.repo._type_source_cache[entity.eid],
- ('Card', 'http://www.cubicweb.org/', 'myrenamedfeed'))
- self.assertEqual(self.repo._extid_cache['http://www.cubicweb.org/'],
+ ('Card', b'http://www.cubicweb.org/', 'myrenamedfeed'))
+ self.assertEqual(self.repo._extid_cache[b'http://www.cubicweb.org/'],
entity.eid)
# test_delete_source
@@ -140,7 +141,14 @@
value = parser.retrieve_url('a string')
self.assertEqual(200, value.getcode())
self.assertEqual('a string', value.geturl())
- self.assertIsInstance(value.info(), mimetools.Message)
+
+ def test_update_url(self):
+ dfsource = self.repo.sources_by_uri[u'ô myfeed']
+ with self.admin_access.repo_cnx() as cnx:
+ cnx.entity_from_eid(dfsource.eid).cw_set(url=u"http://pouet.com\nhttp://pouet.org")
+ self.assertEqual(dfsource.urls, [u'ignored'])
+ cnx.commit()
+ self.assertEqual(dfsource.urls, [u"http://pouet.com", u"http://pouet.org"])
class DataFeedConfigTC(CubicWebTC):
--- a/server/test/unittest_ldapsource.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_ldapsource.py Wed Dec 16 11:23:48 2015 +0100
@@ -15,19 +15,26 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-"""cubicweb.server.sources.ldapusers unit and functional tests"""
+"""cubicweb.server.sources.ldapfeed unit and functional tests
+
+Those tests expect to have slapd, python-ldap3 and ldapscripts packages installed.
+"""
+from __future__ import print_function
import os
import sys
import shutil
import time
-from os.path import join, exists
import subprocess
import tempfile
+import unittest
+from os.path import join
+
+from six import string_types
+from six.moves import range
from cubicweb import AuthenticationError
from cubicweb.devtools.testlib import CubicWebTC
-from cubicweb.devtools.repotest import RQLGeneratorTC
from cubicweb.devtools.httptest import get_available_port
@@ -44,13 +51,14 @@
URL = None
+
def create_slapd_configuration(cls):
global URL
slapddir = tempfile.mkdtemp('cw-unittest-ldap')
config = cls.config
slapdconf = join(config.apphome, "slapd.conf")
- confin = file(join(config.apphome, "slapd.conf.in")).read()
- confstream = file(slapdconf, 'w')
+ confin = open(join(config.apphome, "slapd.conf.in")).read()
+ confstream = open(slapdconf, 'w')
confstream.write(confin % {'apphome': config.apphome, 'testdir': slapddir})
confstream.close()
# fill ldap server with some data
@@ -61,16 +69,16 @@
slapproc = subprocess.Popen(cmdline, stdout=PIPE, stderr=PIPE)
stdout, stderr = slapproc.communicate()
if slapproc.returncode:
- print >> sys.stderr, ('slapadd returned with status: %s'
- % slapproc.returncode)
+ print('slapadd returned with status: %s'
+ % slapproc.returncode, file=sys.stderr)
sys.stdout.write(stdout)
sys.stderr.write(stderr)
- #ldapuri = 'ldapi://' + join(basedir, "ldapi").replace('/', '%2f')
- port = get_available_port(xrange(9000, 9100))
+ # ldapuri = 'ldapi://' + join(basedir, "ldapi").replace('/', '%2f')
+ port = get_available_port(range(9000, 9100))
host = 'localhost:%s' % port
ldapuri = 'ldap://%s' % host
- cmdline = ["/usr/sbin/slapd", "-f", slapdconf, "-h", ldapuri, "-d", "0"]
+ cmdline = ["/usr/sbin/slapd", "-f", slapdconf, "-h", ldapuri, "-d", "0"]
config.info('Starting slapd:', ' '.join(cmdline))
PIPE = subprocess.PIPE
cls.slapd_process = subprocess.Popen(cmdline, stdout=PIPE, stderr=PIPE)
@@ -83,6 +91,7 @@
URL = u'ldap://%s' % host
return slapddir
+
def terminate_slapd(cls):
config = cls.config
if cls.slapd_process and cls.slapd_process.returncode is None:
@@ -90,12 +99,12 @@
if hasattr(cls.slapd_process, 'terminate'):
cls.slapd_process.terminate()
else:
- import os, signal
+ import signal
os.kill(cls.slapd_process.pid, signal.SIGTERM)
stdout, stderr = cls.slapd_process.communicate()
if cls.slapd_process.returncode:
- print >> sys.stderr, ('slapd returned with status: %s'
- % cls.slapd_process.returncode)
+ print('slapd returned with status: %s'
+ % cls.slapd_process.returncode, file=sys.stderr)
sys.stdout.write(stdout)
sys.stderr.write(stderr)
config.info('DONE')
@@ -107,6 +116,8 @@
@classmethod
def setUpClass(cls):
+ if not os.path.exists('/usr/sbin/slapd'):
+ raise unittest.SkipTest('slapd not found')
from cubicweb.cwctl import init_cmdline_log_threshold
init_cmdline_log_threshold(cls.config, cls.loglevel)
cls._tmpdir = create_slapd_configuration(cls)
@@ -139,7 +150,7 @@
cnx.execute('DELETE Any E WHERE E cw_source S, S name "ldap"')
cnx.execute('SET S config %(conf)s, S url %(url)s '
'WHERE S is CWSource, S name "ldap"',
- {"conf": CONFIG_LDAPFEED, 'url': URL} )
+ {"conf": CONFIG_LDAPFEED, 'url': URL})
cnx.commit()
with self.repo.internal_cnx() as cnx:
self.pull(cnx)
@@ -148,32 +159,32 @@
"""
add an LDAP entity
"""
- modcmd = ['dn: %s'%dn, 'changetype: add']
- for key, values in mods.iteritems():
- if isinstance(values, basestring):
+ modcmd = ['dn: %s' % dn, 'changetype: add']
+ for key, values in mods.items():
+ if isinstance(values, string_types):
values = [values]
for value in values:
- modcmd.append('%s: %s'%(key, value))
+ modcmd.append('%s: %s' % (key, value))
self._ldapmodify(modcmd)
def delete_ldap_entry(self, dn):
"""
delete an LDAP entity
"""
- modcmd = ['dn: %s'%dn, 'changetype: delete']
+ modcmd = ['dn: %s' % dn, 'changetype: delete']
self._ldapmodify(modcmd)
def update_ldap_entry(self, dn, mods):
"""
modify one or more attributes of an LDAP entity
"""
- modcmd = ['dn: %s'%dn, 'changetype: modify']
- for (kind, key), values in mods.iteritems():
+ modcmd = ['dn: %s' % dn, 'changetype: modify']
+ for (kind, key), values in mods.items():
modcmd.append('%s: %s' % (kind, key))
- if isinstance(values, basestring):
+ if isinstance(values, string_types):
values = [values]
for value in values:
- modcmd.append('%s: %s'%(key, value))
+ modcmd.append('%s: %s' % (key, value))
modcmd.append('-')
self._ldapmodify(modcmd)
@@ -183,10 +194,11 @@
'cn=admin,dc=cubicweb,dc=test', '-w', 'cw']
PIPE = subprocess.PIPE
p = subprocess.Popen(updatecmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
- p.stdin.write('\n'.join(modcmd))
+ p.stdin.write('\n'.join(modcmd).encode('ascii'))
p.stdin.close()
if p.wait():
- raise RuntimeError("ldap update failed: %s"%('\n'.join(p.stderr.readlines())))
+ raise RuntimeError("ldap update failed: %s" % ('\n'.join(p.stderr.readlines())))
+
class CheckWrongGroup(LDAPFeedTestBase):
"""
@@ -196,18 +208,17 @@
def test_wrong_group(self):
with self.admin_access.repo_cnx() as cnx:
- source = cnx.execute('CWSource S WHERE S type="ldapfeed"').get_entity(0,0)
+ source = cnx.execute('CWSource S WHERE S type="ldapfeed"').get_entity(0, 0)
config = source.repo_source.check_config(source)
# inject a bogus group here, along with at least a valid one
- config['user-default-group'] = ('thisgroupdoesnotexists','users')
+ config['user-default-group'] = ('thisgroupdoesnotexists', 'users')
source.repo_source.update_config(source, config)
cnx.commit()
# here we emitted an error log entry
- stats = source.repo_source.pull_data(cnx, force=True, raise_on_error=True)
+ source.repo_source.pull_data(cnx, force=True, raise_on_error=True)
cnx.commit()
-
class LDAPFeedUserTC(LDAPFeedTestBase):
"""
A testcase for CWUser support in ldapfeed (basic tests and authentication).
@@ -284,12 +295,12 @@
# and that the password stored in the system source is not empty or so
user = cnx.execute('CWUser U WHERE U login "syt"').get_entity(0, 0)
user.cw_clear_all_caches()
- pwd = cnx.system_sql("SELECT cw_upassword FROM cw_cwuser WHERE cw_login='syt';").fetchall()[0][0]
+ cu = cnx.system_sql("SELECT cw_upassword FROM cw_cwuser WHERE cw_login='syt';")
+ pwd = cu.fetchall()[0][0]
self.assertIsNotNone(pwd)
self.assertTrue(str(pwd))
-
class LDAPFeedUserDeletionTC(LDAPFeedTestBase):
"""
A testcase for situations where users are deleted from or
@@ -299,7 +310,7 @@
def test_a_filter_inactivate(self):
""" filtered out people should be deactivated, unable to authenticate """
with self.admin_access.repo_cnx() as cnx:
- source = cnx.execute('CWSource S WHERE S type="ldapfeed"').get_entity(0,0)
+ source = cnx.execute('CWSource S WHERE S type="ldapfeed"').get_entity(0, 0)
config = source.repo_source.check_config(source)
# filter with adim's phone number
config['user-filter'] = u'(%s=%s)' % ('telephoneNumber', '109')
@@ -346,21 +357,22 @@
self.pull(cnx)
# reinsert syt
self.add_ldap_entry('uid=syt,ou=People,dc=cubicweb,dc=test',
- { 'objectClass': ['OpenLDAPperson','posixAccount','top','shadowAccount'],
- 'cn': 'Sylvain Thenault',
- 'sn': 'Thenault',
- 'gidNumber': '1004',
- 'uid': 'syt',
- 'homeDirectory': '/home/syt',
- 'shadowFlag': '134538764',
- 'uidNumber': '1004',
- 'givenName': 'Sylvain',
- 'telephoneNumber': '106',
- 'displayName': 'sthenault',
- 'gecos': 'Sylvain Thenault',
- 'mail': ['sylvain.thenault@logilab.fr','syt@logilab.fr'],
- 'userPassword': 'syt',
- })
+ {'objectClass': ['OpenLDAPperson', 'posixAccount', 'top',
+ 'shadowAccount'],
+ 'cn': 'Sylvain Thenault',
+ 'sn': 'Thenault',
+ 'gidNumber': '1004',
+ 'uid': 'syt',
+ 'homeDirectory': '/home/syt',
+ 'shadowFlag': '134538764',
+ 'uidNumber': '1004',
+ 'givenName': 'Sylvain',
+ 'telephoneNumber': '106',
+ 'displayName': 'sthenault',
+ 'gecos': 'Sylvain Thenault',
+ 'mail': ['sylvain.thenault@logilab.fr', 'syt@logilab.fr'],
+ 'userPassword': 'syt',
+ })
with self.repo.internal_cnx() as cnx:
self.pull(cnx)
with self.admin_access.repo_cnx() as cnx:
@@ -433,8 +445,7 @@
try:
self.update_ldap_entry('cn=logilab,ou=Group,dc=cubicweb,dc=test',
- {('add', 'memberUid'): ['syt']})
- time.sleep(1.1) # timestamps precision is 1s
+ {('add', 'memberUid'): ['syt']})
with self.repo.internal_cnx() as cnx:
self.pull(cnx)
@@ -452,7 +463,7 @@
def test_group_member_deleted(self):
with self.repo.internal_cnx() as cnx:
- self.pull(cnx) # ensure we are sync'ed
+ self.pull(cnx) # ensure we are sync'ed
with self.admin_access.repo_cnx() as cnx:
rset = cnx.execute('Any L WHERE U in_group G, G name %(name)s, U login L',
{'name': 'logilab'})
@@ -462,21 +473,19 @@
try:
self.update_ldap_entry('cn=logilab,ou=Group,dc=cubicweb,dc=test',
{('delete', 'memberUid'): ['adim']})
- time.sleep(1.1) # timestamps precision is 1s
with self.repo.internal_cnx() as cnx:
self.pull(cnx)
with self.admin_access.repo_cnx() as cnx:
rset = cnx.execute('Any L WHERE U in_group G, G name %(name)s, U login L',
{'name': 'logilab'})
- self.assertEqual(len(rset), 0)
+ self.assertEqual(len(rset), 0, rset.rows)
finally:
# back to normal ldap setup
self.tearDownClass()
self.setUpClass()
-
if __name__ == '__main__':
from logilab.common.testlib import unittest_main
unittest_main()
--- a/server/test/unittest_migractions.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_migractions.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,6 +22,7 @@
from contextlib import contextmanager
from logilab.common.testlib import unittest_main, Tags, tag
+from logilab.common import tempattr
from yams.constraints import UniqueConstraint
@@ -54,7 +55,8 @@
class MigrationConfig(cubicweb.devtools.TestServerConfiguration):
default_sources = cubicweb.devtools.DEFAULT_PSQL_SOURCES
- CUBES_PATH = [osp.join(HERE, 'data-migractions', 'cubes')]
+ CUBES_PATH = cubicweb.devtools.TestServerConfiguration.CUBES_PATH + [
+ osp.join(HERE, 'data-migractions', 'cubes')]
class MigrationTC(CubicWebTC):
@@ -151,7 +153,7 @@
orderdict2 = dict(mh.rqlexec('Any RTN, O WHERE X name "Note", RDEF from_entity X, '
'RDEF relation_type RT, RDEF ordernum O, RT name RTN'))
whateverorder = migrschema['whatever'].rdef('Note', 'Int').order
- for k, v in orderdict.iteritems():
+ for k, v in orderdict.items():
if v >= whateverorder:
orderdict[k] = v+1
orderdict['whatever'] = whateverorder
@@ -524,12 +526,12 @@
# remaining orphan rql expr which should be deleted at commit (composite relation)
# unattached expressions -> pending deletion on commit
self.assertEqual(cnx.execute('Any COUNT(X) WHERE X is RQLExpression, X exprtype "ERQLExpression",'
- 'NOT ET1 read_permission X, NOT ET2 add_permission X, '
- 'NOT ET3 delete_permission X, NOT ET4 update_permission X')[0][0],
+ 'NOT ET1 read_permission X, NOT ET2 add_permission X, '
+ 'NOT ET3 delete_permission X, NOT ET4 update_permission X')[0][0],
7)
self.assertEqual(cnx.execute('Any COUNT(X) WHERE X is RQLExpression, X exprtype "RRQLExpression",'
- 'NOT ET1 read_permission X, NOT ET2 add_permission X, '
- 'NOT ET3 delete_permission X, NOT ET4 update_permission X')[0][0],
+ 'NOT ET1 read_permission X, NOT ET2 add_permission X, '
+ 'NOT ET3 delete_permission X, NOT ET4 update_permission X')[0][0],
2)
# finally
self.assertEqual(cnx.execute('Any COUNT(X) WHERE X is RQLExpression')[0][0],
@@ -579,7 +581,7 @@
def test_add_drop_cube_and_deps(self):
with self.mh() as (cnx, mh):
schema = self.repo.schema
- self.assertEqual(sorted((str(s), str(o)) for s, o in schema['see_also'].rdefs.iterkeys()),
+ self.assertEqual(sorted((str(s), str(o)) for s, o in schema['see_also'].rdefs),
sorted([('EmailThread', 'EmailThread'), ('Folder', 'Folder'),
('Bookmark', 'Bookmark'), ('Bookmark', 'Note'),
('Note', 'Note'), ('Note', 'Bookmark')]))
@@ -593,7 +595,7 @@
for ertype in ('Email', 'EmailThread', 'EmailPart', 'File',
'sender', 'in_thread', 'reply_to', 'data_format'):
self.assertNotIn(ertype, schema)
- self.assertEqual(sorted(schema['see_also'].rdefs.iterkeys()),
+ self.assertEqual(sorted(schema['see_also'].rdefs),
sorted([('Folder', 'Folder'),
('Bookmark', 'Bookmark'),
('Bookmark', 'Note'),
@@ -612,12 +614,12 @@
for ertype in ('Email', 'EmailThread', 'EmailPart', 'File',
'sender', 'in_thread', 'reply_to', 'data_format'):
self.assertIn(ertype, schema)
- self.assertEqual(sorted(schema['see_also'].rdefs.iterkeys()),
- sorted([('EmailThread', 'EmailThread'), ('Folder', 'Folder'),
- ('Bookmark', 'Bookmark'),
- ('Bookmark', 'Note'),
- ('Note', 'Note'),
- ('Note', 'Bookmark')]))
+ self.assertEqual(sorted(schema['see_also'].rdefs),
+ sorted([('EmailThread', 'EmailThread'), ('Folder', 'Folder'),
+ ('Bookmark', 'Bookmark'),
+ ('Bookmark', 'Note'),
+ ('Note', 'Note'),
+ ('Note', 'Bookmark')]))
self.assertEqual(sorted(schema['see_also'].subjects()), ['Bookmark', 'EmailThread', 'Folder', 'Note'])
self.assertEqual(sorted(schema['see_also'].objects()), ['Bookmark', 'EmailThread', 'Folder', 'Note'])
from cubes.fakeemail.__pkginfo__ import version as email_version
@@ -719,6 +721,24 @@
self.assertEqual(tel, 1.0)
self.assertIsInstance(tel, float)
+ def test_drop_required_inlined_relation(self):
+ with self.mh() as (cnx, mh):
+ bob = mh.cmd_create_entity('Personne', nom=u'bob')
+ note = mh.cmd_create_entity('Note', ecrit_par=bob)
+ mh.commit()
+ rdef = mh.fs_schema.rschema('ecrit_par').rdefs[('Note', 'Personne')]
+ with tempattr(rdef, 'cardinality', '1*'):
+ mh.sync_schema_props_perms('ecrit_par', syncperms=False)
+ mh.cmd_drop_relation_type('ecrit_par')
+ self.assertNotIn('%secrit_par' % SQL_PREFIX,
+ self.table_schema(mh, '%sPersonne' % SQL_PREFIX))
+
+ def test_drop_inlined_rdef_delete_data(self):
+ with self.mh() as (cnx, mh):
+ note = mh.cmd_create_entity('Note', ecrit_par=cnx.user.eid)
+ mh.commit()
+ mh.drop_relation_definition('Note', 'ecrit_par', 'CWUser')
+ self.assertFalse(mh.sqlexec('SELECT * FROM cw_Note WHERE cw_ecrit_par IS NOT NULL'))
class MigrationCommandsComputedTC(MigrationTC):
""" Unit tests for computed relations and attributes
@@ -784,6 +804,20 @@
self.assertEqual(self.schema['whatever'].subjects(), ('Company',))
self.assertFalse(self.table_sql(mh, 'whatever_relation'))
+ def test_computed_relation_sync_schema_props_perms_security(self):
+ with self.mh() as (cnx, mh):
+ rdef = next(iter(self.schema['perm_changes'].rdefs.values()))
+ self.assertEqual(rdef.permissions,
+ {'add': (), 'delete': (),
+ 'read': ('managers', 'users')})
+ mh.cmd_sync_schema_props_perms('perm_changes')
+ self.assertEqual(self.schema['perm_changes'].permissions,
+ {'read': ('managers',)})
+ rdef = next(iter(self.schema['perm_changes'].rdefs.values()))
+ self.assertEqual(rdef.permissions,
+ {'add': (), 'delete': (),
+ 'read': ('managers',)})
+
def test_computed_relation_sync_schema_props_perms_on_rdef(self):
self.assertIn('whatever', self.schema)
with self.mh() as (cnx, mh):
--- a/server/test/unittest_postgres.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_postgres.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,8 +19,11 @@
from datetime import datetime
from threading import Thread
+from six.moves import range
+
from logilab.common.testlib import SkipTest
+import logilab.database as lgdb
from cubicweb import ValidationError
from cubicweb.devtools import PostgresApptestConfiguration, startpgcluster, stoppgcluster
from cubicweb.devtools.testlib import CubicWebTC
@@ -49,13 +52,21 @@
class PostgresFTITC(CubicWebTC):
configcls = PostgresTimeoutConfiguration
+ @classmethod
+ def setUpClass(cls):
+ cls.orig_connect_hooks = lgdb.SQL_CONNECT_HOOKS['postgres'][:]
+
+ @classmethod
+ def tearDownClass(cls):
+ lgdb.SQL_CONNECT_HOOKS['postgres'] = cls.orig_connect_hooks
+
def test_eid_range(self):
# concurrent allocation of eid ranges
source = self.session.repo.sources_by_uri['system']
range1 = []
range2 = []
def allocate_eid_ranges(session, target):
- for x in xrange(1, 10):
+ for x in range(1, 10):
eid = source.create_eid(session, count=x)
target.extend(range(eid-x, eid))
--- a/server/test/unittest_querier.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_querier.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,6 +21,7 @@
from datetime import date, datetime, timedelta, tzinfo
+from six import PY2, integer_types, binary_type, text_type
from logilab.common.testlib import TestCase, unittest_main
from rql import BadRQLQuery, RQLSyntaxError
@@ -129,8 +130,8 @@
def assertRQLEqual(self, expected, got):
from rql import parse
- self.assertMultiLineEqual(unicode(parse(expected)),
- unicode(parse(got)))
+ self.assertMultiLineEqual(text_type(parse(expected)),
+ text_type(parse(got)))
def test_preprocess_security(self):
s = self.user_groups_session('users')
@@ -178,46 +179,46 @@
' Comment, Division, Email, EmailPart, EmailThread, ExternalUri, File, Folder, '
' Frozable, Note, Old, Personne, RQLExpression, Societe, State, SubDivision, '
' SubWorkflowExitPoint, Tag, TrInfo, Transition, Workflow, WorkflowTransition)')
- self.assertListEqual(sorted(solutions),
- sorted([{'X': 'BaseTransition', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Bookmark', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Card', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Comment', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Division', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWCache', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWComputedRType', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWConstraint', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWConstraintType', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWEType', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWAttribute', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWGroup', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWRelation', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWPermission', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWProperty', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWRType', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWSource', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWUniqueTogetherConstraint', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'CWUser', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Email', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'EmailPart', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'EmailThread', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'ExternalUri', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'File', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Folder', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Frozable', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Note', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Old', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Personne', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'RQLExpression', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Societe', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'State', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'SubDivision', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'SubWorkflowExitPoint', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Tag', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Transition', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'TrInfo', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'Workflow', 'ETN': 'String', 'ET': 'CWEType'},
- {'X': 'WorkflowTransition', 'ETN': 'String', 'ET': 'CWEType'}]))
+ self.assertCountEqual(solutions,
+ [{'X': 'BaseTransition', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Bookmark', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Card', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Comment', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Division', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWCache', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWComputedRType', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWConstraint', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWConstraintType', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWEType', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWAttribute', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWGroup', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWRelation', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWPermission', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWProperty', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWRType', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWSource', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWUniqueTogetherConstraint', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'CWUser', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Email', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'EmailPart', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'EmailThread', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'ExternalUri', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'File', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Folder', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Frozable', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Note', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Old', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Personne', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'RQLExpression', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Societe', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'State', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'SubDivision', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'SubWorkflowExitPoint', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Tag', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Transition', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'TrInfo', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'Workflow', 'ETN': 'String', 'ET': 'CWEType'},
+ {'X': 'WorkflowTransition', 'ETN': 'String', 'ET': 'CWEType'}])
rql, solutions = partrqls[2]
self.assertEqual(rql,
'Any ETN,X WHERE X is ET, ET name ETN, EXISTS(%(D)s use_email X), '
@@ -263,8 +264,9 @@
self.assertEqual(rset.description[0][0], 'Datetime')
rset = self.qexecute('Any %(x)s', {'x': 1})
self.assertEqual(rset.description[0][0], 'Int')
- rset = self.qexecute('Any %(x)s', {'x': 1L})
- self.assertEqual(rset.description[0][0], 'Int')
+ if PY2:
+ rset = self.qexecute('Any %(x)s', {'x': long(1)})
+ self.assertEqual(rset.description[0][0], 'Int')
rset = self.qexecute('Any %(x)s', {'x': True})
self.assertEqual(rset.description[0][0], 'Boolean')
rset = self.qexecute('Any %(x)s', {'x': 1.0})
@@ -307,10 +309,6 @@
setUpClass = classmethod(setUpClass)
tearDownClass = classmethod(tearDownClass)
- def test_encoding_pb(self):
- self.assertRaises(RQLSyntaxError, self.qexecute,
- 'Any X WHERE X is CWRType, X name "öwned_by"')
-
def test_unknown_eid(self):
# should return an empty result set
self.assertFalse(self.qexecute('Any X WHERE X eid 99999999'))
@@ -318,15 +316,15 @@
def test_typed_eid(self):
# should return an empty result set
rset = self.qexecute('Any X WHERE X eid %(x)s', {'x': '1'})
- self.assertIsInstance(rset[0][0], (int, long))
+ self.assertIsInstance(rset[0][0], integer_types)
def test_bytes_storage(self):
feid = self.qexecute('INSERT File X: X data_name "foo.pdf", '
'X data_format "text/plain", X data %(data)s',
- {'data': Binary("xxx")})[0][0]
+ {'data': Binary(b"xxx")})[0][0]
fdata = self.qexecute('Any D WHERE X data D, X eid %(x)s', {'x': feid})[0][0]
self.assertIsInstance(fdata, Binary)
- self.assertEqual(fdata.getvalue(), 'xxx')
+ self.assertEqual(fdata.getvalue(), b'xxx')
# selection queries tests #################################################
@@ -886,18 +884,18 @@
def test_select_constant(self):
rset = self.qexecute('Any X, "toto" ORDERBY X WHERE X is CWGroup')
self.assertEqual(rset.rows,
- map(list, zip((2,3,4,5), ('toto','toto','toto','toto',))))
- self.assertIsInstance(rset[0][1], unicode)
+ [list(x) for x in zip((2,3,4,5), ('toto','toto','toto','toto',))])
+ self.assertIsInstance(rset[0][1], text_type)
self.assertEqual(rset.description,
- zip(('CWGroup', 'CWGroup', 'CWGroup', 'CWGroup'),
- ('String', 'String', 'String', 'String',)))
+ list(zip(('CWGroup', 'CWGroup', 'CWGroup', 'CWGroup'),
+ ('String', 'String', 'String', 'String',))))
rset = self.qexecute('Any X, %(value)s ORDERBY X WHERE X is CWGroup', {'value': 'toto'})
self.assertEqual(rset.rows,
- map(list, zip((2,3,4,5), ('toto','toto','toto','toto',))))
- self.assertIsInstance(rset[0][1], unicode)
+ list(map(list, zip((2,3,4,5), ('toto','toto','toto','toto',)))))
+ self.assertIsInstance(rset[0][1], text_type)
self.assertEqual(rset.description,
- zip(('CWGroup', 'CWGroup', 'CWGroup', 'CWGroup'),
- ('String', 'String', 'String', 'String',)))
+ list(zip(('CWGroup', 'CWGroup', 'CWGroup', 'CWGroup'),
+ ('String', 'String', 'String', 'String',))))
rset = self.qexecute('Any X,GN WHERE X is CWUser, G is CWGroup, X login "syt", '
'X in_group G, G name GN')
@@ -1015,7 +1013,7 @@
self.assertEqual(len(rset.rows), 1)
self.assertEqual(rset.description, [('Personne',)])
rset = self.qexecute('Personne X WHERE X nom "bidule"')
- self.assert_(rset.rows)
+ self.assertTrue(rset.rows)
self.assertEqual(rset.description, [('Personne',)])
def test_insert_1_multiple(self):
@@ -1029,20 +1027,20 @@
rset = self.qexecute("INSERT Personne X, Personne Y: X nom 'bidule', Y nom 'tutu'")
self.assertEqual(rset.description, [('Personne', 'Personne')])
rset = self.qexecute('Personne X WHERE X nom "bidule" or X nom "tutu"')
- self.assert_(rset.rows)
+ self.assertTrue(rset.rows)
self.assertEqual(rset.description, [('Personne',), ('Personne',)])
def test_insert_3(self):
self.qexecute("INSERT Personne X: X nom Y WHERE U login 'admin', U login Y")
rset = self.qexecute('Personne X WHERE X nom "admin"')
- self.assert_(rset.rows)
+ self.assertTrue(rset.rows)
self.assertEqual(rset.description, [('Personne',)])
def test_insert_4(self):
self.qexecute("INSERT Societe Y: Y nom 'toto'")
self.qexecute("INSERT Personne X: X nom 'bidule', X travaille Y WHERE Y nom 'toto'")
rset = self.qexecute('Any X, Y WHERE X nom "bidule", Y nom "toto", X travaille Y')
- self.assert_(rset.rows)
+ self.assertTrue(rset.rows)
self.assertEqual(rset.description, [('Personne', 'Societe',)])
def test_insert_4bis(self):
@@ -1057,17 +1055,17 @@
def test_insert_4ter(self):
peid = self.qexecute("INSERT Personne X: X nom 'bidule'")[0][0]
seid = self.qexecute("INSERT Societe Y: Y nom 'toto', X travaille Y WHERE X eid %(x)s",
- {'x': unicode(peid)})[0][0]
+ {'x': text_type(peid)})[0][0]
self.assertEqual(len(self.qexecute('Any X, Y WHERE X travaille Y')), 1)
self.qexecute("INSERT Personne X: X nom 'chouette', X travaille Y WHERE Y eid %(x)s",
- {'x': unicode(seid)})
+ {'x': text_type(seid)})
self.assertEqual(len(self.qexecute('Any X, Y WHERE X travaille Y')), 2)
def test_insert_5(self):
self.qexecute("INSERT Personne X: X nom 'bidule'")
self.qexecute("INSERT Societe Y: Y nom 'toto', X travaille Y WHERE X nom 'bidule'")
rset = self.qexecute('Any X, Y WHERE X nom "bidule", Y nom "toto", X travaille Y')
- self.assert_(rset.rows)
+ self.assertTrue(rset.rows)
self.assertEqual(rset.description, [('Personne', 'Societe',)])
def test_insert_5bis(self):
@@ -1075,20 +1073,20 @@
self.qexecute("INSERT Societe Y: Y nom 'toto', X travaille Y WHERE X eid %(x)s",
{'x': peid})
rset = self.qexecute('Any X, Y WHERE X nom "bidule", Y nom "toto", X travaille Y')
- self.assert_(rset.rows)
+ self.assertTrue(rset.rows)
self.assertEqual(rset.description, [('Personne', 'Societe',)])
def test_insert_6(self):
self.qexecute("INSERT Personne X, Societe Y: X nom 'bidule', Y nom 'toto', X travaille Y")
rset = self.qexecute('Any X, Y WHERE X nom "bidule", Y nom "toto", X travaille Y')
- self.assert_(rset.rows)
+ self.assertTrue(rset.rows)
self.assertEqual(rset.description, [('Personne', 'Societe',)])
def test_insert_7(self):
self.qexecute("INSERT Personne X, Societe Y: X nom N, Y nom 'toto', "
"X travaille Y WHERE U login 'admin', U login N")
rset = self.qexecute('Any X, Y WHERE X nom "admin", Y nom "toto", X travaille Y')
- self.assert_(rset.rows)
+ self.assertTrue(rset.rows)
self.assertEqual(rset.description, [('Personne', 'Societe',)])
def test_insert_7_2(self):
@@ -1103,7 +1101,7 @@
self.qexecute("INSERT Societe Y, Personne X: Y nom N, X nom 'toto', X travaille Y "
"WHERE U login 'admin', U login N")
rset = self.qexecute('Any X, Y WHERE X nom "toto", Y nom "admin", X travaille Y')
- self.assert_(rset.rows)
+ self.assertTrue(rset.rows)
self.assertEqual(rset.description, [('Personne', 'Societe',)])
def test_insert_9(self):
@@ -1267,7 +1265,7 @@
rset = self.qexecute("INSERT Personne X, Societe Y: X nom 'bidule', Y nom 'toto'")
eid1, eid2 = rset[0][0], rset[0][1]
self.qexecute("SET X travaille Y WHERE X eid %(x)s, Y eid %(y)s",
- {'x': unicode(eid1), 'y': unicode(eid2)})
+ {'x': text_type(eid1), 'y': text_type(eid2)})
rset = self.qexecute('Any X, Y WHERE X travaille Y')
self.assertEqual(len(rset.rows), 1)
@@ -1317,7 +1315,7 @@
eid1, eid2 = rset[0][0], rset[0][1]
rset = self.qexecute("SET X travaille Y WHERE X eid %(x)s, Y eid %(y)s, "
"NOT EXISTS(Z ecrit_par X)",
- {'x': unicode(eid1), 'y': unicode(eid2)})
+ {'x': text_type(eid1), 'y': text_type(eid2)})
self.assertEqual(tuplify(rset.rows), [(eid1, eid2)])
def test_update_query_error(self):
@@ -1364,7 +1362,7 @@
cursor = cnx.cnxset.cu
cursor.execute("SELECT %supassword from %sCWUser WHERE %slogin='bob'"
% (SQL_PREFIX, SQL_PREFIX, SQL_PREFIX))
- passwd = str(cursor.fetchone()[0])
+ passwd = binary_type(cursor.fetchone()[0])
self.assertEqual(passwd, crypt_password('toto', passwd))
rset = self.qexecute("Any X WHERE X is CWUser, X login 'bob', X upassword %(pwd)s",
{'pwd': Binary(passwd)})
@@ -1377,11 +1375,11 @@
{'pwd': 'toto'})
self.assertEqual(rset.description[0][0], 'CWUser')
rset = cnx.execute("SET X upassword %(pwd)s WHERE X is CWUser, X login 'bob'",
- {'pwd': 'tutu'})
+ {'pwd': b'tutu'})
cursor = cnx.cnxset.cu
cursor.execute("SELECT %supassword from %sCWUser WHERE %slogin='bob'"
% (SQL_PREFIX, SQL_PREFIX, SQL_PREFIX))
- passwd = str(cursor.fetchone()[0])
+ passwd = binary_type(cursor.fetchone()[0])
self.assertEqual(passwd, crypt_password('tutu', passwd))
rset = cnx.execute("Any X WHERE X is CWUser, X login 'bob', X upassword %(pwd)s",
{'pwd': Binary(passwd)})
--- a/server/test/unittest_repository.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_repository.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,6 +22,8 @@
import time
import logging
+from six.moves import range
+
from yams.constraints import UniqueConstraint
from yams import register_base_type, unregister_base_type
@@ -77,7 +79,7 @@
def test_connect(self):
cnxid = self.repo.connect(self.admlogin, password=self.admpassword)
- self.assert_(cnxid)
+ self.assertTrue(cnxid)
self.repo.close(cnxid)
self.assertRaises(AuthenticationError,
self.repo.connect, self.admlogin, password='nimportnawak')
@@ -100,7 +102,7 @@
cnx.commit()
repo = self.repo
cnxid = repo.connect(u"barnabé", password=u"héhéhé".encode('UTF8'))
- self.assert_(cnxid)
+ self.assertTrue(cnxid)
repo.close(cnxid)
def test_rollback_on_execute_validation_error(self):
@@ -145,7 +147,7 @@
def test_close(self):
repo = self.repo
cnxid = repo.connect(self.admlogin, password=self.admpassword)
- self.assert_(cnxid)
+ self.assertTrue(cnxid)
repo.close(cnxid)
def test_check_session(self):
@@ -192,7 +194,7 @@
constraints = schema.rschema('relation_type').rdef('CWAttribute', 'CWRType').constraints
self.assertEqual(len(constraints), 1)
cstr = constraints[0]
- self.assert_(isinstance(cstr, RQLConstraint))
+ self.assertIsInstance(cstr, RQLConstraint)
self.assertEqual(cstr.expression, 'O final TRUE')
ownedby = schema.rschema('owned_by')
@@ -589,11 +591,11 @@
with self.admin_access.repo_cnx() as cnx:
personnes = []
t0 = time.time()
- for i in xrange(2000):
+ for i in range(2000):
p = cnx.create_entity('Personne', nom=u'Doe%03d'%i, prenom=u'John', sexe=u'M')
personnes.append(p)
abraham = cnx.create_entity('Personne', nom=u'Abraham', prenom=u'John', sexe=u'M')
- for j in xrange(0, 2000, 100):
+ for j in range(0, 2000, 100):
abraham.cw_set(personne_composite=personnes[j:j+100])
t1 = time.time()
self.info('creation: %.2gs', (t1 - t0))
@@ -610,7 +612,7 @@
def test_add_relation_non_inlined(self):
with self.admin_access.repo_cnx() as cnx:
personnes = []
- for i in xrange(2000):
+ for i in range(2000):
p = cnx.create_entity('Personne', nom=u'Doe%03d'%i, prenom=u'John', sexe=u'M')
personnes.append(p)
cnx.commit()
@@ -619,7 +621,7 @@
personne_composite=personnes[:100])
t1 = time.time()
self.info('creation: %.2gs', (t1 - t0))
- for j in xrange(100, 2000, 100):
+ for j in range(100, 2000, 100):
abraham.cw_set(personne_composite=personnes[j:j+100])
t2 = time.time()
self.info('more relations: %.2gs', (t2-t1))
@@ -630,7 +632,7 @@
def test_add_relation_inlined(self):
with self.admin_access.repo_cnx() as cnx:
personnes = []
- for i in xrange(2000):
+ for i in range(2000):
p = cnx.create_entity('Personne', nom=u'Doe%03d'%i, prenom=u'John', sexe=u'M')
personnes.append(p)
cnx.commit()
@@ -639,7 +641,7 @@
personne_inlined=personnes[:100])
t1 = time.time()
self.info('creation: %.2gs', (t1 - t0))
- for j in xrange(100, 2000, 100):
+ for j in range(100, 2000, 100):
abraham.cw_set(personne_inlined=personnes[j:j+100])
t2 = time.time()
self.info('more relations: %.2gs', (t2-t1))
@@ -652,7 +654,7 @@
""" to be compared with test_session_add_relations"""
with self.admin_access.repo_cnx() as cnx:
personnes = []
- for i in xrange(2000):
+ for i in range(2000):
p = cnx.create_entity('Personne', nom=u'Doe%03d'%i, prenom=u'John', sexe=u'M')
personnes.append(p)
abraham = cnx.create_entity('Personne', nom=u'Abraham', prenom=u'John', sexe=u'M')
@@ -669,7 +671,7 @@
""" to be compared with test_session_add_relation"""
with self.admin_access.repo_cnx() as cnx:
personnes = []
- for i in xrange(2000):
+ for i in range(2000):
p = cnx.create_entity('Personne', nom=u'Doe%03d'%i, prenom=u'John', sexe=u'M')
personnes.append(p)
abraham = cnx.create_entity('Personne', nom=u'Abraham', prenom=u'John', sexe=u'M')
@@ -686,7 +688,7 @@
""" to be compared with test_session_add_relations"""
with self.admin_access.repo_cnx() as cnx:
personnes = []
- for i in xrange(2000):
+ for i in range(2000):
p = cnx.create_entity('Personne', nom=u'Doe%03d'%i, prenom=u'John', sexe=u'M')
personnes.append(p)
abraham = cnx.create_entity('Personne', nom=u'Abraham', prenom=u'John', sexe=u'M')
@@ -703,7 +705,7 @@
""" to be compared with test_session_add_relation"""
with self.admin_access.repo_cnx() as cnx:
personnes = []
- for i in xrange(2000):
+ for i in range(2000):
p = cnx.create_entity('Personne', nom=u'Doe%03d'%i, prenom=u'John', sexe=u'M')
personnes.append(p)
abraham = cnx.create_entity('Personne', nom=u'Abraham', prenom=u'John', sexe=u'M')
--- a/server/test/unittest_rql2sql.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_rql2sql.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""unit tests for module cubicweb.server.sources.rql2sql"""
+from __future__ import print_function
import sys
import os
@@ -1246,13 +1247,13 @@
except Exception as ex:
if 'r' in locals():
try:
- print (r%args).strip()
+ print((r%args).strip())
except KeyError:
- print 'strange, missing substitution'
- print r, nargs
- print '!='
- print sql.strip()
- print 'RQL:', rql
+ print('strange, missing substitution')
+ print(r, nargs)
+ print('!=')
+ print(sql.strip())
+ print('RQL:', rql)
raise
def _parse(self, rqls):
@@ -1269,11 +1270,11 @@
r, args, cbs = self.o.generate(rqlst, args)
self.assertEqual((r.strip(), args), sql)
except Exception as ex:
- print rql
+ print(rql)
if 'r' in locals():
- print r.strip()
- print '!='
- print sql[0].strip()
+ print(r.strip())
+ print('!=')
+ print(sql[0].strip())
raise
return
--- a/server/test/unittest_rqlannotation.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_rqlannotation.py Wed Dec 16 11:23:48 2015 +0100
@@ -64,7 +64,7 @@
rqlst = self._prepare(cnx, 'Any A,B,C WHERE A eid 12,A comment B, '
'A ?wf_info_for C')
self.assertEqual(rqlst.defined_vars['A']._q_invariant, False)
- self.assert_(rqlst.defined_vars['B'].stinfo['attrvar'])
+ self.assertTrue(rqlst.defined_vars['B'].stinfo['attrvar'])
self.assertEqual(rqlst.defined_vars['C']._q_invariant, False)
self.assertEqual(rqlst.solutions, [{'A': 'TrInfo', 'B': 'String', 'C': 'Affaire'},
{'A': 'TrInfo', 'B': 'String', 'C': 'CWUser'},
@@ -87,7 +87,7 @@
'Y nom NX, X eid XE, not Y eid XE')
self.assertEqual(rqlst.defined_vars['X']._q_invariant, False)
self.assertEqual(rqlst.defined_vars['Y']._q_invariant, False)
- self.assert_(rqlst.defined_vars['XE'].stinfo['attrvar'])
+ self.assertTrue(rqlst.defined_vars['XE'].stinfo['attrvar'])
def test_0_8(self):
with self.session.new_cnx() as cnx:
--- a/server/test/unittest_schemaserial.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_schemaserial.py Wed Dec 16 11:23:48 2015 +0100
@@ -292,7 +292,7 @@
{'cardinality': u'?1',
'defaultval': None,
'description': u'',
- 'extra_props': '{"jungle_speed": 42}',
+ 'extra_props': b'{"jungle_speed": 42}',
'formula': None,
'indexed': False,
'oe': None,
--- a/server/test/unittest_security.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_security.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,6 +17,8 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""functional tests for server'security"""
+from six.moves import range
+
from logilab.common.testlib import unittest_main
from cubicweb.devtools.testlib import CubicWebTC
@@ -33,7 +35,7 @@
with self.admin_access.client_cnx() as cnx:
self.create_user(cnx, u'iaminusersgrouponly')
hash = _CRYPTO_CTX.encrypt('oldpassword', scheme='des_crypt')
- self.create_user(cnx, u'oldpassword', password=Binary(hash))
+ self.create_user(cnx, u'oldpassword', password=Binary(hash.encode('ascii')))
class LowLevelSecurityFunctionTC(BaseSecurityTC):
@@ -79,17 +81,20 @@
it will be updated on next login
"""
with self.repo.internal_cnx() as cnx:
- oldhash = str(cnx.system_sql("SELECT cw_upassword FROM cw_CWUser "
- "WHERE cw_login = 'oldpassword'").fetchone()[0])
+ oldhash = cnx.system_sql("SELECT cw_upassword FROM cw_CWUser "
+ "WHERE cw_login = 'oldpassword'").fetchone()[0]
+ oldhash = self.repo.system_source.binary_to_str(oldhash)
self.repo.close(self.repo.connect('oldpassword', password='oldpassword'))
- newhash = str(cnx.system_sql("SELECT cw_upassword FROM cw_CWUser "
- "WHERE cw_login = 'oldpassword'").fetchone()[0])
+ newhash = cnx.system_sql("SELECT cw_upassword FROM cw_CWUser "
+ "WHERE cw_login = 'oldpassword'").fetchone()[0]
+ newhash = self.repo.system_source.binary_to_str(newhash)
self.assertNotEqual(oldhash, newhash)
- self.assertTrue(newhash.startswith('$6$'))
+ self.assertTrue(newhash.startswith(b'$6$'))
self.repo.close(self.repo.connect('oldpassword', password='oldpassword'))
- self.assertEqual(newhash,
- str(cnx.system_sql("SELECT cw_upassword FROM cw_CWUser WHERE "
- "cw_login = 'oldpassword'").fetchone()[0]))
+ newnewhash = cnx.system_sql("SELECT cw_upassword FROM cw_CWUser WHERE "
+ "cw_login = 'oldpassword'").fetchone()[0]
+ newnewhash = self.repo.system_source.binary_to_str(newnewhash)
+ self.assertEqual(newhash, newnewhash)
class SecurityRewritingTC(BaseSecurityTC):
@@ -293,7 +298,7 @@
ueid = self.create_user(cnx, u'user').eid
with self.new_access(u'user').repo_cnx() as cnx:
cnx.execute('SET X upassword %(passwd)s WHERE X eid %(x)s',
- {'x': ueid, 'passwd': 'newpwd'})
+ {'x': ueid, 'passwd': b'newpwd'})
cnx.commit()
self.repo.close(self.repo.connect('user', password='newpwd'))
@@ -302,7 +307,7 @@
ueid = self.create_user(cnx, u'otheruser').eid
with self.new_access(u'iaminusersgrouponly').repo_cnx() as cnx:
cnx.execute('SET X upassword %(passwd)s WHERE X eid %(x)s',
- {'x': ueid, 'passwd': 'newpwd'})
+ {'x': ueid, 'passwd': b'newpwd'})
self.assertRaises(Unauthorized, cnx.commit)
# read security test
@@ -559,7 +564,7 @@
rset = cnx.execute('CWUser X')
self.assertEqual([[anon.eid]], rset.rows)
# anonymous user can read groups (necessary to check allowed transitions for instance)
- self.assert_(cnx.execute('CWGroup X'))
+ self.assertTrue(cnx.execute('CWGroup X'))
# should only be able to read the anonymous user, not another one
self.assertRaises(Unauthorized,
cnx.execute, 'CWUser X WHERE X eid %(x)s', {'x': admineid})
@@ -666,7 +671,7 @@
rset = cnx.execute('Any X, U WHERE X is EmailAddress, U use_email X')
msg = ['Preexisting email readable by anon found!']
tmpl = ' - "%s" used by user "%s"'
- for i in xrange(len(rset)):
+ for i in range(len(rset)):
email, user = rset.get_entity(i, 0), rset.get_entity(i, 1)
msg.append(tmpl % (email.dc_title(), user.dc_title()))
raise RuntimeError('\n'.join(msg))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/server/test/unittest_serverctl.py Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,26 @@
+import os.path as osp
+import shutil
+
+from cubicweb.devtools import testlib, ApptestConfiguration
+from cubicweb.server.serverctl import _local_dump, DBDumpCommand
+from cubicweb.server.serverconfig import ServerConfiguration
+
+class ServerCTLTC(testlib.CubicWebTC):
+ def setUp(self):
+ super(ServerCTLTC, self).setUp()
+ self.orig_config_for = ServerConfiguration.config_for
+ config_for = lambda appid: ApptestConfiguration(appid, apphome=self.datadir)
+ ServerConfiguration.config_for = staticmethod(config_for)
+
+ def tearDown(self):
+ ServerConfiguration.config_for = self.orig_config_for
+ super(ServerCTLTC, self).tearDown()
+
+ def test_dump(self):
+ DBDumpCommand(None).run([self.appid])
+ shutil.rmtree(osp.join(self.config.apphome, 'backup'))
+
+
+if __name__ == '__main__':
+ from unittest import main
+ main()
--- a/server/test/unittest_storage.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_storage.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,12 +17,15 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""unit tests for module cubicweb.server.sources.storages"""
+from six import PY2
+
from logilab.common.testlib import unittest_main, tag, Tags
from cubicweb.devtools.testlib import CubicWebTC
from glob import glob
import os
import os.path as osp
+import sys
import shutil
import tempfile
@@ -57,24 +60,26 @@
def setup_database(self):
self.tempdir = tempfile.mkdtemp()
bfs_storage = storages.BytesFileSystemStorage(self.tempdir)
+ self.bfs_storage = bfs_storage
storages.set_attribute_storage(self.repo, 'File', 'data', bfs_storage)
storages.set_attribute_storage(self.repo, 'BFSSTestable', 'opt_attr', bfs_storage)
def tearDown(self):
super(StorageTC, self).tearDown()
storages.unset_attribute_storage(self.repo, 'File', 'data')
+ del self.bfs_storage
shutil.rmtree(self.tempdir)
- def create_file(self, cnx, content='the-data'):
+ def create_file(self, cnx, content=b'the-data'):
return cnx.create_entity('File', data=Binary(content),
data_format=u'text/plain',
data_name=u'foo.pdf')
def fspath(self, cnx, entity):
fspath = cnx.execute('Any fspath(D) WHERE F eid %(f)s, F data D',
- {'f': entity.eid})[0][0]
- return fspath.getvalue()
+ {'f': entity.eid})[0][0].getvalue()
+ return fspath if PY2 else fspath.decode('utf-8')
def test_bfss_wrong_fspath_usage(self):
with self.admin_access.repo_cnx() as cnx:
@@ -109,7 +114,7 @@
# add f1 back to the entity cache with req as _cw
f1 = req.entity_from_eid(f1.eid)
- f1.cw_set(data=Binary('the new data'))
+ f1.cw_set(data=Binary(b'the new data'))
cnx.rollback()
self.assertEqual(open(expected_filepath).read(), 'the-data')
f1.cw_delete()
@@ -132,7 +137,7 @@
with self.admin_access.repo_cnx() as cnx:
cnx.transaction_data['fs_importing'] = True
filepath = osp.abspath(__file__)
- f1 = cnx.create_entity('File', data=Binary(filepath),
+ f1 = cnx.create_entity('File', data=Binary(filepath.encode(sys.getfilesystemencoding())),
data_format=u'text/plain', data_name=u'foo')
self.assertEqual(self.fspath(cnx, f1), filepath)
@@ -185,8 +190,8 @@
self.assertEqual(len(rset), 2)
self.assertEqual(rset[0][0], f1.eid)
self.assertEqual(rset[1][0], f1.eid)
- self.assertEqual(rset[0][1].getvalue(), 'the-data')
- self.assertEqual(rset[1][1].getvalue(), 'the-data')
+ self.assertEqual(rset[0][1].getvalue(), b'the-data')
+ self.assertEqual(rset[1][1].getvalue(), b'the-data')
rset = cnx.execute('Any X,LENGTH(D) WHERE X eid %(x)s, X data D',
{'x': f1.eid})
self.assertEqual(len(rset), 1)
@@ -212,31 +217,31 @@
with self.admin_access.repo_cnx() as cnx:
cnx.transaction_data['fs_importing'] = True
filepath = osp.abspath(__file__)
- f1 = cnx.create_entity('File', data=Binary(filepath),
+ f1 = cnx.create_entity('File', data=Binary(filepath.encode(sys.getfilesystemencoding())),
data_format=u'text/plain', data_name=u'foo')
cw_value = f1.data.getvalue()
- fs_value = file(filepath).read()
+ fs_value = open(filepath, 'rb').read()
if cw_value != fs_value:
self.fail('cw value %r is different from file content' % cw_value)
@tag('update')
def test_bfss_update_with_existing_data(self):
with self.admin_access.repo_cnx() as cnx:
- f1 = cnx.create_entity('File', data=Binary('some data'),
+ f1 = cnx.create_entity('File', data=Binary(b'some data'),
data_format=u'text/plain', data_name=u'foo')
# NOTE: do not use cw_set() which would automatically
# update f1's local dict. We want the pure rql version to work
cnx.execute('SET F data %(d)s WHERE F eid %(f)s',
- {'d': Binary('some other data'), 'f': f1.eid})
- self.assertEqual(f1.data.getvalue(), 'some other data')
+ {'d': Binary(b'some other data'), 'f': f1.eid})
+ self.assertEqual(f1.data.getvalue(), b'some other data')
cnx.commit()
f2 = cnx.execute('Any F WHERE F eid %(f)s, F is File', {'f': f1.eid}).get_entity(0, 0)
- self.assertEqual(f2.data.getvalue(), 'some other data')
+ self.assertEqual(f2.data.getvalue(), b'some other data')
@tag('update', 'extension', 'commit')
def test_bfss_update_with_different_extension_commited(self):
with self.admin_access.repo_cnx() as cnx:
- f1 = cnx.create_entity('File', data=Binary('some data'),
+ f1 = cnx.create_entity('File', data=Binary(b'some data'),
data_format=u'text/plain', data_name=u'foo.txt')
# NOTE: do not use cw_set() which would automatically
# update f1's local dict. We want the pure rql version to work
@@ -246,7 +251,7 @@
self.assertEqual(osp.splitext(old_path)[1], '.txt')
cnx.execute('SET F data %(d)s, F data_name %(dn)s, '
'F data_format %(df)s WHERE F eid %(f)s',
- {'d': Binary('some other data'), 'f': f1.eid,
+ {'d': Binary(b'some other data'), 'f': f1.eid,
'dn': u'bar.jpg', 'df': u'image/jpeg'})
cnx.commit()
# the new file exists with correct extension
@@ -260,7 +265,7 @@
@tag('update', 'extension', 'rollback')
def test_bfss_update_with_different_extension_rolled_back(self):
with self.admin_access.repo_cnx() as cnx:
- f1 = cnx.create_entity('File', data=Binary('some data'),
+ f1 = cnx.create_entity('File', data=Binary(b'some data'),
data_format=u'text/plain', data_name=u'foo.txt')
# NOTE: do not use cw_set() which would automatically
# update f1's local dict. We want the pure rql version to work
@@ -271,7 +276,7 @@
self.assertEqual(osp.splitext(old_path)[1], '.txt')
cnx.execute('SET F data %(d)s, F data_name %(dn)s, '
'F data_format %(df)s WHERE F eid %(f)s',
- {'d': Binary('some other data'),
+ {'d': Binary(b'some other data'),
'f': f1.eid,
'dn': u'bar.jpg',
'df': u'image/jpeg'})
@@ -290,7 +295,7 @@
@tag('update', 'NULL')
def test_bfss_update_to_None(self):
with self.admin_access.repo_cnx() as cnx:
- f = cnx.create_entity('Affaire', opt_attr=Binary('toto'))
+ f = cnx.create_entity('Affaire', opt_attr=Binary(b'toto'))
cnx.commit()
f.cw_set(opt_attr=None)
cnx.commit()
@@ -298,17 +303,17 @@
@tag('fs_importing', 'update')
def test_bfss_update_with_fs_importing(self):
with self.admin_access.repo_cnx() as cnx:
- f1 = cnx.create_entity('File', data=Binary('some data'),
+ f1 = cnx.create_entity('File', data=Binary(b'some data'),
data_format=u'text/plain',
data_name=u'foo')
old_fspath = self.fspath(cnx, f1)
cnx.transaction_data['fs_importing'] = True
new_fspath = osp.join(self.tempdir, 'newfile.txt')
- file(new_fspath, 'w').write('the new data')
+ open(new_fspath, 'w').write('the new data')
cnx.execute('SET F data %(d)s WHERE F eid %(f)s',
- {'d': Binary(new_fspath), 'f': f1.eid})
+ {'d': Binary(new_fspath.encode(sys.getfilesystemencoding())), 'f': f1.eid})
cnx.commit()
- self.assertEqual(f1.data.getvalue(), 'the new data')
+ self.assertEqual(f1.data.getvalue(), b'the new data')
self.assertEqual(self.fspath(cnx, f1), new_fspath)
self.assertFalse(osp.isfile(old_fspath))
--- a/server/test/unittest_undo.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_undo.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,6 +17,8 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+from six import text_type
+
from cubicweb import ValidationError
from cubicweb.devtools.testlib import CubicWebTC
import cubicweb.server.session
@@ -255,7 +257,7 @@
"%s doesn't exist anymore." % g.eid])
with self.assertRaises(ValidationError) as cm:
cnx.commit()
- cm.exception.translate(unicode)
+ cm.exception.translate(text_type)
self.assertEqual(cm.exception.entity, self.totoeid)
self.assertEqual(cm.exception.errors,
{'in_group-subject': u'at least one relation in_group is '
@@ -461,7 +463,7 @@
# problem occurs in string manipulation for python < 2.6
def test___unicode__method(self):
u = _UndoException(u"voilà ")
- self.assertIsInstance(unicode(u), unicode)
+ self.assertIsInstance(text_type(u), text_type)
if __name__ == '__main__':
--- a/server/test/unittest_utils.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/test/unittest_utils.py Wed Dec 16 11:23:48 2015 +0100
@@ -26,13 +26,13 @@
def test_crypt(self):
for hash in (
utils.crypt_password('xxx'), # default sha512
- 'ab$5UsKFxRKKN.d8iBIFBnQ80', # custom md5
- 'ab4Vlm81ZUHlg', # DES
+ b'ab$5UsKFxRKKN.d8iBIFBnQ80', # custom md5
+ b'ab4Vlm81ZUHlg', # DES
):
self.assertEqual(utils.crypt_password('xxx', hash), hash)
self.assertEqual(utils.crypt_password(u'xxx', hash), hash)
- self.assertEqual(utils.crypt_password(u'xxx', unicode(hash)), hash)
- self.assertEqual(utils.crypt_password('yyy', hash), '')
+ self.assertEqual(utils.crypt_password(u'xxx', hash.decode('ascii')), hash.decode('ascii'))
+ self.assertEqual(utils.crypt_password('yyy', hash), b'')
# accept any password for empty hashes (is it a good idea?)
self.assertEqual(utils.crypt_password('xxx', ''), '')
--- a/server/utils.py Tue Dec 15 19:05:41 2015 +0100
+++ b/server/utils.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""Some utilities for the CubicWeb server."""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -24,6 +25,9 @@
from threading import Timer, Thread
from getpass import getpass
+from six import PY2, text_type
+from six.moves import input
+
from passlib.utils import handlers as uh, to_hash_str
from passlib.context import CryptContext
@@ -60,7 +64,7 @@
"""return the encrypted password using the given salt or a generated one
"""
if salt is None:
- return _CRYPTO_CTX.encrypt(passwd)
+ return _CRYPTO_CTX.encrypt(passwd).encode('ascii')
# empty hash, accept any password for backwards compat
if salt == '':
return salt
@@ -70,7 +74,7 @@
except ValueError: # e.g. couldn't identify hash
pass
# wrong password
- return ''
+ return b''
def eschema_eid(cnx, eschema):
@@ -81,7 +85,7 @@
if eschema.eid is None:
eschema.eid = cnx.execute(
'Any X WHERE X is CWEType, X name %(name)s',
- {'name': unicode(eschema)})[0][0]
+ {'name': text_type(eschema)})[0][0]
return eschema.eid
@@ -92,17 +96,18 @@
passwdmsg='password'):
if not user:
if msg:
- print msg
+ print(msg)
while not user:
- user = raw_input('login: ')
- user = unicode(user, sys.stdin.encoding)
+ user = input('login: ')
+ if PY2:
+ user = unicode(user, sys.stdin.encoding)
passwd = getpass('%s: ' % passwdmsg)
if confirm:
while True:
passwd2 = getpass('confirm password: ')
if passwd == passwd2:
break
- print 'password doesn\'t match'
+ print('password doesn\'t match')
passwd = getpass('password: ')
# XXX decode password using stdin encoding then encode it using appl'encoding
return user, passwd
@@ -236,4 +241,3 @@
from logging import getLogger
from cubicweb import set_log_methods
set_log_methods(TasksManager, getLogger('cubicweb.repository'))
-
--- a/setup.py Tue Dec 15 19:05:41 2015 +0100
+++ b/setup.py Wed Dec 16 11:23:48 2015 +0100
@@ -42,7 +42,7 @@
from __pkginfo__ import modname, version, license, description, web, \
author, author_email
-long_description = file('README').read()
+long_description = open('README').read()
# import optional features
import __pkginfo__
@@ -51,7 +51,7 @@
for entry in ("__depends__",): # "__recommends__"):
requires.update(getattr(__pkginfo__, entry, {}))
install_requires = [("%s %s" % (d, v and v or "")).strip()
- for d, v in requires.iteritems()]
+ for d, v in requires.items()]
else:
install_requires = []
--- a/skeleton/DISTNAME.spec.tmpl Tue Dec 15 19:05:41 2015 +0100
+++ b/skeleton/DISTNAME.spec.tmpl Wed Dec 16 11:23:48 2015 +0100
@@ -21,6 +21,7 @@
BuildRequires: %%{python} %%{python}-setuptools
Requires: cubicweb >= %(version)s
+Requires: %%{python}-six >= 1.4.0
%%description
%(longdesc)s
--- a/skeleton/debian/control.tmpl Tue Dec 15 19:05:41 2015 +0100
+++ b/skeleton/debian/control.tmpl Wed Dec 16 11:23:48 2015 +0100
@@ -12,6 +12,7 @@
Architecture: all
Depends:
cubicweb-common (>= %(version)s),
+ python-six (>= 1.4.0),
${python:Depends},
${misc:Depends},
Description: %(shortdesc)s
--- a/skeleton/debian/rules Tue Dec 15 19:05:41 2015 +0100
+++ b/skeleton/debian/rules Wed Dec 16 11:23:48 2015 +0100
@@ -5,10 +5,5 @@
%:
dh $@ --with python2
-override_dh_auto_install:
- dh_auto_install
- # remove generated .egg-info file
- rm -rf debian/*/usr/lib/python*
-
override_dh_python2:
dh_python2 -i /usr/share/cubicweb
--- a/skeleton/setup.py Tue Dec 15 19:05:41 2015 +0100
+++ b/skeleton/setup.py Wed Dec 16 11:23:48 2015 +0100
@@ -4,7 +4,7 @@
# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
-# This file is part of CubicWeb tag cube.
+# This file is part of CubicWeb.
#
# CubicWeb is free software: you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
@@ -44,7 +44,7 @@
author, author_email, classifiers
if exists('README'):
- long_description = file('README').read()
+ long_description = open('README').read()
else:
long_description = ''
@@ -55,7 +55,7 @@
for entry in ("__depends__",): # "__recommends__"):
requires.update(getattr(__pkginfo__, entry, {}))
install_requires = [("%s %s" % (d, v and v or "")).strip()
- for d, v in requires.iteritems()]
+ for d, v in requires.items()]
else:
install_requires = []
--- a/sobjects/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/sobjects/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,11 +20,11 @@
import os.path as osp
def registration_callback(vreg):
- vreg.register_all(globals().itervalues(), __name__)
+ vreg.register_all(globals().values(), __name__)
global URL_MAPPING
URL_MAPPING = {}
if vreg.config.apphome:
url_mapping_file = osp.join(vreg.config.apphome, 'urlmapping.py')
if osp.exists(url_mapping_file):
- URL_MAPPING = eval(file(url_mapping_file).read())
+ URL_MAPPING = eval(open(url_mapping_file).read())
vreg.info('using url mapping %s from %s', URL_MAPPING, url_mapping_file)
--- a/sobjects/cwxmlparser.py Tue Dec 15 19:05:41 2015 +0100
+++ b/sobjects/cwxmlparser.py Wed Dec 16 11:23:48 2015 +0100
@@ -32,9 +32,11 @@
"""
from datetime import datetime, time
-import urlparse
import urllib
+from six import text_type
+from six.moves.urllib.parse import urlparse, urlunparse, parse_qs, urlencode
+
from logilab.common.date import todate, totime
from logilab.common.textutils import splitstrip, text_to_dict
from logilab.common.decorators import classproperty
@@ -50,7 +52,7 @@
# XXX see cubicweb.cwvreg.YAMS_TO_PY
# XXX see cubicweb.web.views.xmlrss.SERIALIZERS
DEFAULT_CONVERTERS = BASE_CONVERTERS.copy()
-DEFAULT_CONVERTERS['String'] = unicode
+DEFAULT_CONVERTERS['String'] = text_type
DEFAULT_CONVERTERS['Password'] = lambda x: x.encode('utf8')
def convert_date(ustr):
return todate(datetime.strptime(ustr, '%Y-%m-%d'))
@@ -124,7 +126,7 @@
def list_actions(self):
reg = self._cw.vreg['components']
- return sorted(clss[0].action for rid, clss in reg.iteritems()
+ return sorted(clss[0].action for rid, clss in reg.items()
if rid.startswith('cw.entityxml.action.'))
# mapping handling #########################################################
@@ -204,7 +206,7 @@
* `rels` is for relations and structured as
{role: {relation: [(related item, related rels)...]}
"""
- entity = self.extid2entity(str(item['cwuri']), item['cwtype'],
+ entity = self.extid2entity(item['cwuri'].encode('ascii'), item['cwtype'],
cwsource=item['cwsource'], item=item,
raise_on_error=raise_on_error)
if entity is None:
@@ -220,7 +222,7 @@
def process_relations(self, entity, rels):
etype = entity.cw_etype
- for (rtype, role, action), rules in self.source.mapping.get(etype, {}).iteritems():
+ for (rtype, role, action), rules in self.source.mapping.get(etype, {}).items():
try:
related_items = rels[role][rtype]
except KeyError:
@@ -242,14 +244,14 @@
def normalize_url(self, url):
"""overridden to add vid=xml if vid is not set in the qs"""
url = super(CWEntityXMLParser, self).normalize_url(url)
- purl = urlparse.urlparse(url)
+ purl = urlparse(url)
if purl.scheme in ('http', 'https'):
- params = urlparse.parse_qs(purl.query)
+ params = parse_qs(purl.query)
if 'vid' not in params:
params['vid'] = ['xml']
purl = list(purl)
- purl[4] = urllib.urlencode(params, doseq=True)
- return urlparse.urlunparse(purl)
+ purl[4] = urlencode(params, doseq=True)
+ return urlunparse(purl)
return url
def complete_url(self, url, etype=None, known_relations=None):
@@ -263,8 +265,8 @@
If `known_relations` is given, it should be a dictionary of already
known relations, so they don't get queried again.
"""
- purl = urlparse.urlparse(url)
- params = urlparse.parse_qs(purl.query)
+ purl = urlparse(url)
+ params = parse_qs(purl.query)
if etype is None:
etype = purl.path.split('/')[-1]
try:
@@ -277,8 +279,8 @@
continue
relations.add('%s-%s' % (rtype, role))
purl = list(purl)
- purl[4] = urllib.urlencode(params, doseq=True)
- return urlparse.urlunparse(purl)
+ purl[4] = urlencode(params, doseq=True)
+ return urlunparse(purl)
def complete_item(self, item, rels):
try:
@@ -314,7 +316,7 @@
"""
node = self.node
item = dict(node.attrib.items())
- item['cwtype'] = unicode(node.tag)
+ item['cwtype'] = text_type(node.tag)
item.setdefault('cwsource', None)
try:
item['eid'] = int(item['eid'])
@@ -331,7 +333,7 @@
related += self.parser.parse_etree(child)
elif child.text:
# attribute
- item[child.tag] = unicode(child.text)
+ item[child.tag] = text_type(child.text)
else:
# None attribute (empty tag)
item[child.tag] = None
--- a/sobjects/ldapparser.py Tue Dec 15 19:05:41 2015 +0100
+++ b/sobjects/ldapparser.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,4 +1,4 @@
-# copyright 2011-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2011-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -20,12 +20,28 @@
unlike ldapuser source, this source is copy based and will import ldap content
(beside passwords for authentication) into the system source.
"""
+from six.moves import map, filter
+
from logilab.common.decorators import cached, cachedproperty
from logilab.common.shellutils import generate_password
from cubicweb import Binary, ConfigurationError
from cubicweb.server.utils import crypt_password
from cubicweb.server.sources import datafeed
+from cubicweb.dataimport import stores, importer
+
+
+class UserMetaGenerator(stores.MetaGenerator):
+ """Specific metadata generator, used to see newly created user into their initial state.
+ """
+ @cached
+ def base_etype_dicts(self, entity):
+ entity, rels = super(UserMetaGenerator, self).base_etype_dicts(entity)
+ if entity.cw_etype == 'CWUser':
+ wf_state = self._cnx.execute('Any S WHERE ET default_workflow WF, ET name %(etype)s, '
+ 'WF initial_state S', {'etype': entity.cw_etype}).one()
+ rels['in_state'] = wf_state.eid
+ return entity, rels
class DataFeedLDAPAdapter(datafeed.DataFeedParser):
@@ -48,8 +64,8 @@
def user_source_entities_by_extid(self):
source = self.source
if source.user_base_dn.strip():
- attrs = map(str, source.user_attrs.keys())
- return dict((userdict['dn'], userdict)
+ attrs = list(map(str, source.user_attrs.keys()))
+ return dict((userdict['dn'].encode('ascii'), userdict)
for userdict in source._search(self._cw,
source.user_base_dn,
source.user_base_scope,
@@ -61,7 +77,7 @@
def group_source_entities_by_extid(self):
source = self.source
if source.group_base_dn.strip():
- attrs = map(str, ['modifyTimestamp'] + source.group_attrs.keys())
+ attrs = list(map(str, ['modifyTimestamp'] + list(source.group_attrs.keys())))
return dict((groupdict['dn'], groupdict)
for groupdict in source._search(self._cw,
source.group_base_dn,
@@ -70,171 +86,170 @@
attrs))
return {}
- def _process(self, etype, sdict, raise_on_error=False):
- self.debug('fetched %s %s', etype, sdict)
- extid = sdict['dn']
- entity = self.extid2entity(extid, etype,
- raise_on_error=raise_on_error, **sdict)
- if entity is not None and not self.created_during_pull(entity):
- self.notify_updated(entity)
- attrs = self.ldap2cwattrs(sdict, etype)
- self.update_if_necessary(entity, attrs)
- if etype == 'CWUser':
- self._process_email(entity, sdict)
- if etype == 'CWGroup':
- self._process_membership(entity, sdict)
-
def process(self, url, raise_on_error=False):
"""IDataFeedParser main entry point"""
self.debug('processing ldapfeed source %s %s', self.source, self.searchfilterstr)
- for userdict in self.user_source_entities_by_extid.itervalues():
- self._process('CWUser', userdict)
+ self._group_members = {}
+ eeimporter = self.build_importer(raise_on_error)
+ for name in self.source.user_default_groups:
+ geid = self._get_group(name)
+ eeimporter.extid2eid[geid] = geid
+ entities = self.extentities_generator()
+ set_cwuri = importer.use_extid_as_cwuri(eeimporter.extid2eid)
+ eeimporter.import_entities(set_cwuri(entities))
+ self.stats['created'] = eeimporter.created
+ self.stats['updated'] = eeimporter.updated
+ # handle in_group relation
+ for group, members in self._group_members.items():
+ self._cw.execute('DELETE U in_group G WHERE G name %(g)s', {'g': group})
+ if members:
+ members = ["'%s'" % e for e in members]
+ rql = 'SET U in_group G WHERE G name %%(g)s, U login IN (%s)' % ','.join(members)
+ self._cw.execute(rql, {'g': group})
+ # ensure updated users are activated
+ for eid in eeimporter.updated:
+ entity = self._cw.entity_from_eid(eid)
+ if entity.cw_etype == 'CWUser':
+ self.ensure_activated(entity)
+ # manually set primary email if necessary, it's not handled automatically since hooks are
+ # deactivated
+ self._cw.execute('SET X primary_email E WHERE NOT X primary_email E, X use_email E, '
+ 'X cw_source S, S eid %(s)s, X in_state ST, TS name "activated"',
+ {'s': self.source.eid})
+
+ def build_importer(self, raise_on_error):
+ """Instantiate and configure an importer"""
+ etypes = ('CWUser', 'EmailAddress', 'CWGroup')
+ extid2eid = importer.cwuri2eid(self._cw, etypes, source_eid=self.source.eid)
+ existing_relations = {}
+ for rtype in ('in_group', 'use_email', 'owned_by'):
+ rql = 'Any S,O WHERE S {} O, S cw_source SO, SO eid %(s)s'.format(rtype)
+ rset = self._cw.execute(rql, {'s': self.source.eid})
+ existing_relations[rtype] = set(tuple(x) for x in rset)
+ return importer.ExtEntitiesImporter(self._cw.vreg.schema, self.build_store(),
+ extid2eid=extid2eid,
+ existing_relations=existing_relations,
+ etypes_order_hint=etypes,
+ import_log=self.import_log,
+ raise_on_error=raise_on_error)
+
+ def build_store(self):
+ """Instantiate and configure a store"""
+ metagenerator = UserMetaGenerator(self._cw, source=self.source)
+ return stores.NoHookRQLObjectStore(self._cw, metagenerator)
+
+ def extentities_generator(self):
self.debug('processing ldapfeed source %s %s', self.source, self.searchgroupfilterstr)
- for groupdict in self.group_source_entities_by_extid.itervalues():
- self._process('CWGroup', groupdict, raise_on_error=raise_on_error)
+ # generate users and email addresses
+ for userdict in self.user_source_entities_by_extid.values():
+ attrs = self.ldap2cwattrs(userdict, 'CWUser')
+ pwd = attrs.get('upassword')
+ if not pwd:
+ # generate a dumb password if not fetched from ldap (see
+ # userPassword)
+ pwd = crypt_password(generate_password())
+ attrs['upassword'] = set([Binary(pwd)])
+ extuser = importer.ExtEntity('CWUser', userdict['dn'], attrs)
+ extuser.values['owned_by'] = set([extuser.extid])
+ for extemail in self._process_email(extuser, userdict):
+ yield extemail
+ groups = list(filter(None, [self._get_group(name)
+ for name in self.source.user_default_groups]))
+ if groups:
+ extuser.values['in_group'] = groups
+ yield extuser
+ # generate groups
+ for groupdict in self.group_source_entities_by_extid.values():
+ attrs = self.ldap2cwattrs(groupdict, 'CWGroup')
+ extgroup = importer.ExtEntity('CWGroup', groupdict['dn'], attrs)
+ yield extgroup
+ # record group membership for later insertion
+ members = groupdict.get(self.source.group_rev_attrs['member'], ())
+ self._group_members[attrs['name']] = members
+
+ def _process_email(self, extuser, userdict):
+ try:
+ emailaddrs = userdict.pop(self.source.user_rev_attrs['email'])
+ except KeyError:
+ return # no email for that user, nothing to do
+ if not isinstance(emailaddrs, list):
+ emailaddrs = [emailaddrs]
+ for emailaddr in emailaddrs:
+ # search for existing email first, may be coming from another source
+ rset = self._cw.execute('EmailAddress X WHERE X address %(addr)s',
+ {'addr': emailaddr})
+ if not rset:
+ # not found, create it. first forge an external id
+ emailextid = userdict['dn'] + '@@' + emailaddr
+ extuser.values.setdefault('use_email', []).append(emailextid)
+ yield importer.ExtEntity('EmailAddress', emailextid, dict(address=[emailaddr]))
+ elif self.sourceuris:
+ # pop from sourceuris anyway, else email may be removed by the
+ # source once import is finished
+ emailextid = userdict['dn'] + '@@' + emailaddr
+ self.sourceuris.pop(emailextid, None)
+ # XXX else check use_email relation?
def handle_deletion(self, config, cnx, myuris):
if config['delete-entities']:
super(DataFeedLDAPAdapter, self).handle_deletion(config, cnx, myuris)
return
if myuris:
- byetype = {}
- for extid, (eid, etype) in myuris.iteritems():
- if self.is_deleted(extid, etype, eid):
- byetype.setdefault(etype, []).append(str(eid))
-
- for etype, eids in byetype.iteritems():
- if etype != 'CWUser':
+ for extid, (eid, etype) in myuris.items():
+ if etype != 'CWUser' or not self.is_deleted(extid, etype, eid):
continue
- self.info('deactivate %s %s entities', len(eids), etype)
- for eid in eids:
- wf = cnx.entity_from_eid(eid).cw_adapt_to('IWorkflowable')
- wf.fire_transition_if_possible('deactivate')
+ self.info('deactivate user %s', eid)
+ wf = cnx.entity_from_eid(eid).cw_adapt_to('IWorkflowable')
+ wf.fire_transition_if_possible('deactivate')
cnx.commit()
- def update_if_necessary(self, entity, attrs):
- # disable read security to allow password selection
- with entity._cw.security_enabled(read=False):
- entity.complete(tuple(attrs))
+ def ensure_activated(self, entity):
if entity.cw_etype == 'CWUser':
wf = entity.cw_adapt_to('IWorkflowable')
if wf.state == 'deactivated':
wf.fire_transition('activate')
self.info('user %s reactivated', entity.login)
- mdate = attrs.get('modification_date')
- if not mdate or mdate > entity.modification_date:
- attrs = dict( (k, v) for k, v in attrs.iteritems()
- if v != getattr(entity, k))
- if attrs:
- entity.cw_set(**attrs)
- self.notify_updated(entity)
+
+ def ldap2cwattrs(self, sdict, etype):
+ """Transform dictionary of LDAP attributes to CW.
- def ldap2cwattrs(self, sdict, etype, tdict=None):
- """ Transform dictionary of LDAP attributes to CW
- etype must be CWUser or CWGroup """
- if tdict is None:
- tdict = {}
+ etype must be CWUser or CWGroup
+ """
+ assert etype in ('CWUser', 'CWGroup'), etype
+ tdict = {}
if etype == 'CWUser':
- items = self.source.user_attrs.iteritems()
+ items = self.source.user_attrs.items()
elif etype == 'CWGroup':
- items = self.source.group_attrs.iteritems()
+ items = self.source.group_attrs.items()
for sattr, tattr in items:
if tattr not in self.non_attribute_keys:
try:
- tdict[tattr] = sdict[sattr]
+ value = sdict[sattr]
except KeyError:
- raise ConfigurationError('source attribute %s has not '
- 'been found in the source, '
- 'please check the %s-attrs-map '
- 'field and the permissions of '
- 'the LDAP binding user' %
- (sattr, etype[2:].lower()))
+ raise ConfigurationError(
+ 'source attribute %s has not been found in the source, '
+ 'please check the %s-attrs-map field and the permissions of '
+ 'the LDAP binding user' % (sattr, etype[2:].lower()))
+ if not isinstance(value, list):
+ value = [value]
+ tdict[tattr] = value
return tdict
- def before_entity_copy(self, entity, sourceparams):
- etype = entity.cw_etype
- if etype == 'EmailAddress':
- entity.cw_edited['address'] = sourceparams['address']
- else:
- self.ldap2cwattrs(sourceparams, etype, tdict=entity.cw_edited)
- if etype == 'CWUser':
- pwd = entity.cw_edited.get('upassword')
- if not pwd:
- # generate a dumb password if not fetched from ldap (see
- # userPassword)
- pwd = crypt_password(generate_password())
- entity.cw_edited['upassword'] = Binary(pwd)
- return entity
-
- def after_entity_copy(self, entity, sourceparams):
- super(DataFeedLDAPAdapter, self).after_entity_copy(entity, sourceparams)
- etype = entity.cw_etype
- if etype == 'EmailAddress':
- return
- # all CWUsers must be treated before CWGroups to have the in_group relation
- # set correctly in _associate_ldapusers
- elif etype == 'CWUser':
- groups = filter(None, [self._get_group(name)
- for name in self.source.user_default_groups])
- if groups:
- entity.cw_set(in_group=groups)
- self._process_email(entity, sourceparams)
- elif etype == 'CWGroup':
- self._process_membership(entity, sourceparams)
-
def is_deleted(self, extidplus, etype, eid):
try:
- extid, _ = extidplus.rsplit('@@', 1)
+ extid = extidplus.rsplit(b'@@', 1)[0]
except ValueError:
# for some reason extids here tend to come in both forms, e.g:
# dn, dn@@Babar
extid = extidplus
return extid not in self.user_source_entities_by_extid
- def _process_email(self, entity, userdict):
- try:
- emailaddrs = userdict[self.source.user_rev_attrs['email']]
- except KeyError:
- return # no email for that user, nothing to do
- if not isinstance(emailaddrs, list):
- emailaddrs = [emailaddrs]
- for emailaddr in emailaddrs:
- # search for existing email first, may be coming from another source
- rset = self._cw.execute('EmailAddress X WHERE X address %(addr)s',
- {'addr': emailaddr})
- if not rset:
- # not found, create it. first forge an external id
- emailextid = userdict['dn'] + '@@' + emailaddr.encode('utf-8')
- email = self.extid2entity(emailextid, 'EmailAddress',
- address=emailaddr)
- entity.cw_set(use_email=email)
- elif self.sourceuris:
- # pop from sourceuris anyway, else email may be removed by the
- # source once import is finished
- uri = userdict['dn'] + '@@' + emailaddr.encode('utf-8')
- self.sourceuris.pop(uri, None)
- # XXX else check use_email relation?
-
- def _process_membership(self, entity, sourceparams):
- """ Find existing CWUsers with the same login as the memberUids in the
- CWGroup entity and create the in_group relationship """
- mdate = sourceparams.get('modification_date')
- if (not mdate or mdate > entity.modification_date):
- self._cw.execute('DELETE U in_group G WHERE G eid %(g)s',
- {'g':entity.eid})
- members = sourceparams.get(self.source.group_rev_attrs['member'])
- if members:
- members = ["'%s'" % e for e in members]
- rql = 'SET U in_group G WHERE G eid %%(g)s, U login IN (%s)' % ','.join(members)
- self._cw.execute(rql, {'g':entity.eid, })
-
@cached
def _get_group(self, name):
try:
return self._cw.execute('Any X WHERE X is CWGroup, X name %(name)s',
- {'name': name}).get_entity(0, 0)
+ {'name': name})[0][0]
except IndexError:
self.error('group %r referenced by source configuration %r does not exist',
name, self.source.uri)
return None
-
--- a/sobjects/notification.py Tue Dec 15 19:05:41 2015 +0100
+++ b/sobjects/notification.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,10 +18,12 @@
"""some views to handle notification on data changes"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from itertools import repeat
+from six import text_type
+
from logilab.common.textutils import normalize_text
from logilab.common.deprecation import class_renamed, class_moved, deprecated
from logilab.common.registry import yes
@@ -181,8 +183,8 @@
def context(self, **kwargs):
entity = self.cw_rset.get_entity(self.cw_row or 0, self.cw_col or 0)
- for key, val in kwargs.iteritems():
- if val and isinstance(val, unicode) and val.strip():
+ for key, val in kwargs.items():
+ if val and isinstance(val, text_type) and val.strip():
kwargs[key] = self._cw._(val)
kwargs.update({'user': self.user_data['login'],
'eid': entity.eid,
@@ -255,7 +257,7 @@
def format_value(value):
- if isinstance(value, unicode):
+ if isinstance(value, text_type):
return u'"%s"' % value
return value
--- a/sobjects/services.py Tue Dec 15 19:05:41 2015 +0100
+++ b/sobjects/services.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,7 +19,10 @@
import threading
+from six import text_type
+
from yams.schema import role_name
+
from cubicweb import ValidationError
from cubicweb.server import Service
from cubicweb.predicates import match_user_groups, match_kwargs
@@ -58,6 +61,7 @@
results['threads'] = [t.name for t in threading.enumerate()]
return results
+
class GcStatsService(Service):
"""Return a dictionary containing some statistics about the repository
resources usage.
@@ -94,9 +98,9 @@
results = {}
counters, ocounters, garbage = gc_info(lookupclasses,
viewreferrersclasses=())
- values = sorted(counters.iteritems(), key=lambda x: x[1], reverse=True)
+ values = sorted(counters.items(), key=lambda x: x[1], reverse=True)
results['lookupclasses'] = values
- values = sorted(ocounters.iteritems(), key=lambda x: x[1], reverse=True)[:nmax]
+ values = sorted(ocounters.items(), key=lambda x: x[1], reverse=True)[:nmax]
results['referenced'] = values
results['unreachable'] = garbage
return results
@@ -129,7 +133,7 @@
qname = role_name('login', 'subject')
raise ValidationError(None, {qname: errmsg % login})
- if isinstance(password, unicode):
+ if isinstance(password, text_type):
# password should *always* be utf8 encoded
password = password.encode('UTF8')
cwuserkwargs['login'] = login
@@ -154,3 +158,17 @@
'WHERE U login %(login)s', d, build_descr=False)
return user
+
+
+class SourceSynchronizationService(Service):
+ """Force synchronization of a datafeed source"""
+ __regid__ = 'source-sync'
+ __select__ = Service.__select__ & match_user_groups('managers')
+
+ def call(self, source_eid):
+ source_entity = self._cw.entity_from_eid(source_eid)
+ repo = self._cw.repo # Service are repo side only.
+ with repo.internal_cnx() as cnx:
+ source = repo.sources_by_uri[source_entity.name]
+ source.pull_data(cnx)
+
--- a/sobjects/supervising.py Tue Dec 15 19:05:41 2015 +0100
+++ b/sobjects/supervising.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""some hooks and views to handle supervising of any data changes"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from cubicweb import UnknownEid
from cubicweb.predicates import none_rset
@@ -128,13 +128,15 @@
# XXX print changes
self.w(u' %s' % changedescr.entity.absolute_url())
- def delete_entity(self, (eid, etype, title)):
+ def delete_entity(self, args):
+ eid, etype, title = args
msg = self._cw._('deleted %(etype)s #%(eid)s (%(title)s)')
etype = display_name(self._cw, etype).lower()
self.w(msg % locals())
- def change_state(self, (entity, fromstate, tostate)):
+ def change_state(self, args):
_ = self._cw._
+ entity, fromstate, tostate = args
msg = _('changed state of %(etype)s #%(eid)s (%(title)s)')
self.w(u'%s\n' % (msg % self._entity_context(entity)))
self.w(_(' from state %(fromstate)s to state %(tostate)s\n' %
--- a/sobjects/test/unittest_cwxmlparser.py Tue Dec 15 19:05:41 2015 +0100
+++ b/sobjects/test/unittest_cwxmlparser.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,7 +17,8 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime
-from urlparse import urlsplit, parse_qsl
+
+from six.moves.urllib.parse import urlsplit, parse_qsl
from cubicweb.devtools.testlib import CubicWebTC
from cubicweb.sobjects.cwxmlparser import CWEntityXMLParser
@@ -214,8 +215,8 @@
with self.admin_access.web_request() as req:
user = req.execute('CWUser X WHERE X login "sthenault"').get_entity(0, 0)
- self.assertEqual(user.creation_date, datetime(2010, 01, 22, 10, 27, 59))
- self.assertEqual(user.modification_date, datetime(2011, 01, 25, 14, 14, 06))
+ self.assertEqual(user.creation_date, datetime(2010, 1, 22, 10, 27, 59))
+ self.assertEqual(user.modification_date, datetime(2011, 1, 25, 14, 14, 6))
self.assertEqual(user.cwuri, 'http://pouet.org/5')
self.assertEqual(user.cw_source[0].name, 'myfeed')
self.assertEqual(user.absolute_url(), 'http://pouet.org/5')
@@ -299,8 +300,8 @@
with self.repo.internal_cnx() as cnx:
stats = dfsource.pull_data(cnx, force=True, raise_on_error=True)
user = cnx.execute('CWUser X WHERE X login "sthenault"').get_entity(0, 0)
- self.assertEqual(user.creation_date, datetime(2010, 01, 22, 10, 27, 59))
- self.assertEqual(user.modification_date, datetime(2011, 01, 25, 14, 14, 06))
+ self.assertEqual(user.creation_date, datetime(2010, 1, 22, 10, 27, 59))
+ self.assertEqual(user.modification_date, datetime(2011, 1, 25, 14, 14, 6))
self.assertEqual(user.cwuri, 'http://pouet.org/5')
self.assertEqual(user.cw_source[0].name, 'myfeed')
--- a/sobjects/test/unittest_supervising.py Tue Dec 15 19:05:41 2015 +0100
+++ b/sobjects/test/unittest_supervising.py Wed Dec 16 11:23:48 2015 +0100
@@ -77,7 +77,7 @@
# check prepared email
op._prepare_email()
self.assertEqual(len(op.to_send), 1)
- self.assert_(op.to_send[0][0])
+ self.assertTrue(op.to_send[0][0])
self.assertEqual(op.to_send[0][1], ['test@logilab.fr'])
cnx.commit()
# some other changes #######
--- a/spa2rql.py Tue Dec 15 19:05:41 2015 +0100
+++ b/spa2rql.py Wed Dec 16 11:23:48 2015 +0100
@@ -146,9 +146,9 @@
def finalize(self):
"""return corresponding rql query (string) / args (dict)"""
- for varname, ptypes in self.possible_types.iteritems():
+ for varname, ptypes in self.possible_types.items():
if len(ptypes) == 1:
- self.restrictions.append('%s is %s' % (varname, iter(ptypes).next()))
+ self.restrictions.append('%s is %s' % (varname, next(iter(ptypes))))
unions = []
for releq, subjvar, obj in self.union_params:
thisunions = []
--- a/tags.py Tue Dec 15 19:05:41 2015 +0100
+++ b/tags.py Wed Dec 16 11:23:48 2015 +0100
@@ -59,4 +59,3 @@
html += options
html.append(u'</select>')
return u'\n'.join(html)
-
--- a/test/data/cubes/file/__pkginfo__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/data/cubes/file/__pkginfo__.py Wed Dec 16 11:23:48 2015 +0100
@@ -23,4 +23,3 @@
numversion = (1, 4, 3)
version = '.'.join(str(num) for num in numversion)
-
--- a/test/data/rqlexpr_on_computedrel.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/data/rqlexpr_on_computedrel.py Wed Dec 16 11:23:48 2015 +0100
@@ -14,5 +14,3 @@
class computed(ComputedRelation):
rule = 'S relation O'
__permissions__ = {'read': (RRQLExpression('S is ET'),)}
-
-
--- a/test/data/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/data/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -23,7 +23,7 @@
RQLConstraint, RQLVocabularyConstraint)
-_ = unicode
+from cubicweb import _
class Personne(EntityType):
--- a/test/data_schemareader/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/data_schemareader/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -7,5 +7,3 @@
cw_for_source.__permissions__ = {'read': ('managers', 'users'),
'add': ('managers',),
'delete': ('managers',)}
-
-
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test/unittest_binary.py Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,54 @@
+from six import PY2
+
+from unittest import TestCase
+from tempfile import NamedTemporaryFile
+import os.path as osp
+
+from logilab.common.shellutils import tempdir
+from cubicweb import Binary
+
+
+class BinaryTC(TestCase):
+ def test_init(self):
+ Binary()
+ Binary(b'toto')
+ Binary(bytearray(b'toto'))
+ if PY2:
+ Binary(buffer('toto'))
+ else:
+ Binary(memoryview(b'toto'))
+ with self.assertRaises((AssertionError, TypeError)):
+ # TypeError is raised by BytesIO if python runs with -O
+ Binary(u'toto')
+
+ def test_write(self):
+ b = Binary()
+ b.write(b'toto')
+ b.write(bytearray(b'toto'))
+ if PY2:
+ b.write(buffer('toto'))
+ else:
+ b.write(memoryview(b'toto'))
+ with self.assertRaises((AssertionError, TypeError)):
+ # TypeError is raised by BytesIO if python runs with -O
+ b.write(u'toto')
+
+ def test_gzpickle_roundtrip(self):
+ old = (u'foo', b'bar', 42, {})
+ new = Binary.zpickle(old).unzpickle()
+ self.assertEqual(old, new)
+ self.assertIsNot(old, new)
+
+ def test_from_file_to_file(self):
+ with tempdir() as dpath:
+ fpath = osp.join(dpath, 'binary.bin')
+ with open(fpath, 'wb') as fobj:
+ Binary(b'binaryblob').to_file(fobj)
+
+ bobj = Binary.from_file(fpath)
+ self.assertEqual(bobj.getvalue(), b'binaryblob')
+
+
+if __name__ == '__main__':
+ from unittest import main
+ main()
--- a/test/unittest_cwconfig.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_cwconfig.py Wed Dec 16 11:23:48 2015 +0100
@@ -86,15 +86,6 @@
finally:
comment_pkginfo.__recommends_cubes__ = {}
-
-# def test_vc_config(self):
-# vcconf = self.config.vc_config()
-# self.assertIsInstance(vcconf['EEMAIL'], Version)
-# self.assertEqual(vcconf['EEMAIL'], (0, 3, 1))
-# self.assertEqual(vcconf['CW'], (2, 31, 2))
-# self.assertRaises(KeyError, vcconf.__getitem__, 'CW_VERSION')
-# self.assertRaises(KeyError, vcconf.__getitem__, 'CRM')
-
def test_expand_cubes(self):
self.config.__class__.CUBES_PATH = [CUSTOM_CUBES_DIR]
self.config.adjust_sys_path()
--- a/test/unittest_cwctl.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_cwctl.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,10 @@
import sys
import os
from os.path import join
-from cStringIO import StringIO
+from io import StringIO, BytesIO
+
+from six import PY2
+
from logilab.common.testlib import TestCase, unittest_main
from cubicweb.cwconfig import CubicWebConfiguration
@@ -30,7 +33,7 @@
class CubicWebCtlTC(TestCase):
def setUp(self):
- self.stream = StringIO()
+ self.stream = BytesIO() if PY2 else StringIO()
sys.stdout = self.stream
def tearDown(self):
sys.stdout = sys.__stdout__
@@ -57,7 +60,7 @@
funcname=None)
for script, args in scripts.items():
scriptname = os.path.join(self.datadir, 'scripts', script)
- self.assert_(os.path.exists(scriptname))
+ self.assertTrue(os.path.exists(scriptname))
mih.cmd_process_script(scriptname, None, scriptargs=args)
--- a/test/unittest_entity.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_entity.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,6 +20,8 @@
from datetime import datetime
+from six import text_type
+
from logilab.common import tempattr
from logilab.common.decorators import clear_cache
@@ -644,7 +646,7 @@
def test_printable_value_bytes(self):
with self.admin_access.web_request() as req:
- e = req.create_entity('FakeFile', data=Binary('lambda x: 1'), data_format=u'text/x-python',
+ e = req.create_entity('FakeFile', data=Binary(b'lambda x: 1'), data_format=u'text/x-python',
data_encoding=u'ascii', data_name=u'toto.py')
from cubicweb import mttransforms
if mttransforms.HAS_PYGMENTS_TRANSFORMS:
@@ -663,8 +665,10 @@
<span style="color: #C00000;">lambda</span> <span style="color: #000000;">x</span><span style="color: #0000C0;">:</span> <span style="color: #0080C0;">1</span>
</pre>''')
- e = req.create_entity('FakeFile', data=Binary('*héhéhé*'), data_format=u'text/rest',
- data_encoding=u'utf-8', data_name=u'toto.txt')
+ e = req.create_entity('FakeFile',
+ data=Binary(u'*héhéhé*'.encode('utf-8')),
+ data_format=u'text/rest',
+ data_encoding=u'utf-8', data_name=u'toto.txt')
self.assertEqual(e.printable_value('data'),
u'<p><em>héhéhé</em></p>')
@@ -717,7 +721,7 @@
e = self.vreg['etypes'].etype_class('FakeFile')(req)
e.cw_attr_cache['description'] = 'du <em>html</em>'
e.cw_attr_cache['description_format'] = 'text/html'
- e.cw_attr_cache['data'] = Binary('some <em>data</em>')
+ e.cw_attr_cache['data'] = Binary(b'some <em>data</em>')
e.cw_attr_cache['data_name'] = 'an html file'
e.cw_attr_cache['data_format'] = 'text/html'
e.cw_attr_cache['data_encoding'] = 'ascii'
@@ -769,11 +773,11 @@
# ambiguity test
person2 = req.create_entity('Personne', prenom=u'remi', nom=u'doe')
person.cw_clear_all_caches()
- self.assertEqual(person.rest_path(), unicode(person.eid))
- self.assertEqual(person2.rest_path(), unicode(person2.eid))
+ self.assertEqual(person.rest_path(), text_type(person.eid))
+ self.assertEqual(person2.rest_path(), text_type(person2.eid))
# unique attr with None value (nom in this case)
friend = req.create_entity('Ami', prenom=u'bob')
- self.assertEqual(friend.rest_path(), unicode(friend.eid))
+ self.assertEqual(friend.rest_path(), text_type(friend.eid))
# 'ref' below is created without the unique but not required
# attribute, make sur that the unique _and_ required 'ean' is used
# as the rest attribute
@@ -853,4 +857,3 @@
if __name__ == '__main__':
from logilab.common.testlib import unittest_main
unittest_main()
-
--- a/test/unittest_mail.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_mail.py Wed Dec 16 11:23:48 2015 +0100
@@ -149,4 +149,3 @@
if __name__ == '__main__':
unittest_main()
-
--- a/test/unittest_migration.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_migration.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -22,7 +22,7 @@
from cubicweb.devtools import TestServerConfiguration
from cubicweb.cwconfig import CubicWebConfiguration
-from cubicweb.migration import MigrationHelper, filter_scripts
+from cubicweb.migration import MigrationHelper, filter_scripts, version_strictly_lower
from cubicweb.server.migractions import ServerMigrationHelper
@@ -76,8 +76,6 @@
def test_filter_scripts_for_mode(self):
config = CubicWebConfiguration('data')
config.verbosity = 0
- self.assertNotIsInstance(config.migration_handler(), ServerMigrationHelper)
- self.assertIsInstance(config.migration_handler(), MigrationHelper)
config = self.config
config.__class__.name = 'repository'
self.assertListEqual(filter_scripts(config, TMIGRDIR, (0,0,4), (0,1,0)),
@@ -91,6 +89,10 @@
((0, 1 ,0), TMIGRDIR+'0.1.0_repository.py')])
config.__class__.name = 'repository'
+ def test_version_strictly_lower(self):
+ self.assertTrue(version_strictly_lower(None, '1.0.0'))
+ self.assertFalse(version_strictly_lower('1.0.0', None))
+
from cubicweb.devtools import ApptestConfiguration, get_test_db_handler
--- a/test/unittest_predicates.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_predicates.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,14 +20,17 @@
from operator import eq, lt, le, gt
from contextlib import contextmanager
+from six.moves import range
+
from logilab.common.testlib import TestCase, unittest_main
from logilab.common.decorators import clear_cache
from cubicweb import Binary
from cubicweb.devtools.testlib import CubicWebTC
from cubicweb.predicates import (is_instance, adaptable, match_kwargs, match_user_groups,
- multi_lines_rset, score_entity, is_in_state,
- rql_condition, relation_possible, match_form_params)
+ multi_lines_rset, score_entity, is_in_state,
+ rql_condition, relation_possible, match_form_params,
+ paginated_rset)
from cubicweb.selectors import on_transition # XXX on_transition is deprecated
from cubicweb.view import EntityAdapter
from cubicweb.web import action
@@ -37,7 +40,7 @@
class ImplementsTC(CubicWebTC):
def test_etype_priority(self):
with self.admin_access.web_request() as req:
- f = req.create_entity('FakeFile', data_name=u'hop.txt', data=Binary('hop'),
+ f = req.create_entity('FakeFile', data_name=u'hop.txt', data=Binary(b'hop'),
data_format=u'text/plain')
rset = f.as_rset()
anyscore = is_instance('Any')(f.__class__, req, rset=rset)
@@ -488,6 +491,34 @@
"match_form_params() positional arguments must be strings")
+class PaginatedTC(CubicWebTC):
+ """tests for paginated_rset predicate"""
+
+ def setup_database(self):
+ with self.admin_access.repo_cnx() as cnx:
+ for i in range(30):
+ cnx.create_entity('CWGroup', name=u"group%d" % i)
+ cnx.commit()
+
+ def test_paginated_rset(self):
+ default_nb_pages = 1
+ web_request = self.admin_access.web_request
+ with web_request() as req:
+ rset = req.execute('Any G WHERE G is CWGroup')
+ self.assertEqual(len(rset), 34)
+ with web_request(vid='list', page_size='10') as req:
+ self.assertEqual(paginated_rset()(None, req, rset), default_nb_pages)
+ with web_request(vid='list', page_size='20') as req:
+ self.assertEqual(paginated_rset()(None, req, rset), default_nb_pages)
+ with web_request(vid='list', page_size='50') as req:
+ self.assertEqual(paginated_rset()(None, req, rset), 0)
+ with web_request(vid='list', page_size='10/') as req:
+ self.assertEqual(paginated_rset()(None, req, rset), 0)
+ with web_request(vid='list', page_size='.1') as req:
+ self.assertEqual(paginated_rset()(None, req, rset), 0)
+ with web_request(vid='list', page_size='not_an_int') as req:
+ self.assertEqual(paginated_rset()(None, req, rset), 0)
+
+
if __name__ == '__main__':
unittest_main()
-
--- a/test/unittest_rqlrewrite.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_rqlrewrite.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,8 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+from six import string_types
+
from logilab.common.testlib import unittest_main, TestCase
from logilab.common.testlib import mock_object
from yams import BadSchemaDefinition
@@ -67,9 +69,9 @@
rewriter = _prepare_rewriter(rqlrewrite.RQLRewriter, kwargs)
snippets = []
for v, exprs in sorted(snippets_map.items()):
- rqlexprs = [isinstance(snippet, basestring)
- and mock_object(snippet_rqlst=parse('Any X WHERE '+snippet).children[0],
- expression='Any X WHERE '+snippet)
+ rqlexprs = [isinstance(snippet, string_types)
+ and mock_object(snippet_rqlst=parse(u'Any X WHERE '+snippet).children[0],
+ expression=u'Any X WHERE '+snippet)
or snippet
for snippet in exprs]
snippets.append((dict([v]), rqlexprs))
@@ -90,7 +92,7 @@
selects.append(stmt)
assert node in selects, (node, selects)
for stmt in selects:
- for var in stmt.defined_vars.itervalues():
+ for var in stmt.defined_vars.values():
assert var.stinfo['references']
vrefmap = vrefmaps[stmt]
assert not (var.stinfo['references'] ^ vrefmap[var.name]), (node.as_string(), var, var.stinfo['references'], vrefmap[var.name])
@@ -108,90 +110,90 @@
def test_base_var(self):
constraint = ('X in_state S, U in_group G, P require_state S,'
'P name "read", P require_group G')
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'X'): (constraint,)}, {})
self.assertEqual(rqlst.as_string(),
- u"Any C WHERE C is Card, B eid %(D)s, "
- "EXISTS(C in_state A, B in_group E, F require_state A, "
- "F name 'read', F require_group E, A is State, E is CWGroup, F is CWPermission)")
+ u'Any C WHERE C is Card, B eid %(D)s, '
+ 'EXISTS(C in_state A, B in_group E, F require_state A, '
+ 'F name "read", F require_group E, A is State, E is CWGroup, F is CWPermission)')
def test_multiple_var(self):
card_constraint = ('X in_state S, U in_group G, P require_state S,'
'P name "read", P require_group G')
affaire_constraints = ('X ref LIKE "PUBLIC%"', 'U in_group G, G name "public"')
kwargs = {'u':2}
- rqlst = parse('Any S WHERE S documented_by C, C eid %(u)s')
+ rqlst = parse(u'Any S WHERE S documented_by C, C eid %(u)s')
rewrite(rqlst, {('C', 'X'): (card_constraint,), ('S', 'X'): affaire_constraints},
kwargs)
self.assertMultiLineEqual(
rqlst.as_string(),
- "Any S WHERE S documented_by C, C eid %(u)s, B eid %(D)s, "
- "EXISTS(C in_state A, B in_group E, F require_state A, "
- "F name 'read', F require_group E, A is State, E is CWGroup, F is CWPermission), "
- "(EXISTS(S ref LIKE 'PUBLIC%')) OR (EXISTS(B in_group G, G name 'public', G is CWGroup)), "
- "S is Affaire")
+ u'Any S WHERE S documented_by C, C eid %(u)s, B eid %(D)s, '
+ 'EXISTS(C in_state A, B in_group E, F require_state A, '
+ 'F name "read", F require_group E, A is State, E is CWGroup, F is CWPermission), '
+ '(EXISTS(S ref LIKE "PUBLIC%")) OR (EXISTS(B in_group G, G name "public", G is CWGroup)), '
+ 'S is Affaire')
self.assertIn('D', kwargs)
def test_or(self):
constraint = '(X identity U) OR (X in_state ST, CL identity U, CL in_state ST, ST name "subscribed")'
- rqlst = parse('Any S WHERE S owned_by C, C eid %(u)s, S is in (CWUser, CWGroup)')
+ rqlst = parse(u'Any S WHERE S owned_by C, C eid %(u)s, S is in (CWUser, CWGroup)')
rewrite(rqlst, {('C', 'X'): (constraint,)}, {'u':1})
self.assertEqual(rqlst.as_string(),
- "Any S WHERE S owned_by C, C eid %(u)s, S is IN(CWUser, CWGroup), A eid %(B)s, "
- "EXISTS((C identity A) OR (C in_state D, E identity A, "
- "E in_state D, D name 'subscribed'), D is State, E is CWUser)")
+ 'Any S WHERE S owned_by C, C eid %(u)s, S is IN(CWUser, CWGroup), A eid %(B)s, '
+ 'EXISTS((C identity A) OR (C in_state D, E identity A, '
+ 'E in_state D, D name "subscribed"), D is State, E is CWUser)')
def test_simplified_rqlst(self):
constraint = ('X in_state S, U in_group G, P require_state S,'
'P name "read", P require_group G')
- rqlst = parse('Any 2') # this is the simplified rql st for Any X WHERE X eid 12
+ rqlst = parse(u'Any 2') # this is the simplified rql st for Any X WHERE X eid 12
rewrite(rqlst, {('2', 'X'): (constraint,)}, {})
self.assertEqual(rqlst.as_string(),
- u"Any 2 WHERE B eid %(C)s, "
- "EXISTS(2 in_state A, B in_group D, E require_state A, "
- "E name 'read', E require_group D, A is State, D is CWGroup, E is CWPermission)")
+ u'Any 2 WHERE B eid %(C)s, '
+ 'EXISTS(2 in_state A, B in_group D, E require_state A, '
+ 'E name "read", E require_group D, A is State, D is CWGroup, E is CWPermission)')
def test_optional_var_1(self):
constraint = ('X in_state S, U in_group G, P require_state S,'
'P name "read", P require_group G')
- rqlst = parse('Any A,C WHERE A documented_by C?')
+ rqlst = parse(u'Any A,C WHERE A documented_by C?')
rewrite(rqlst, {('C', 'X'): (constraint,)}, {})
self.assertEqual(rqlst.as_string(),
- "Any A,C WHERE A documented_by C?, A is Affaire "
- "WITH C BEING "
- "(Any C WHERE EXISTS(C in_state B, D in_group F, G require_state B, G name 'read', "
- "G require_group F), D eid %(A)s, C is Card)")
+ u'Any A,C WHERE A documented_by C?, A is Affaire '
+ 'WITH C BEING '
+ '(Any C WHERE EXISTS(C in_state B, D in_group F, G require_state B, G name "read", '
+ 'G require_group F), D eid %(A)s, C is Card)')
def test_optional_var_2(self):
constraint = ('X in_state S, U in_group G, P require_state S,'
'P name "read", P require_group G')
- rqlst = parse('Any A,C,T WHERE A documented_by C?, C title T')
+ rqlst = parse(u'Any A,C,T WHERE A documented_by C?, C title T')
rewrite(rqlst, {('C', 'X'): (constraint,)}, {})
self.assertEqual(rqlst.as_string(),
- "Any A,C,T WHERE A documented_by C?, A is Affaire "
- "WITH C,T BEING "
- "(Any C,T WHERE C title T, EXISTS(C in_state B, D in_group F, "
- "G require_state B, G name 'read', G require_group F), "
- "D eid %(A)s, C is Card)")
+ u'Any A,C,T WHERE A documented_by C?, A is Affaire '
+ 'WITH C,T BEING '
+ '(Any C,T WHERE C title T, EXISTS(C in_state B, D in_group F, '
+ 'G require_state B, G name "read", G require_group F), '
+ 'D eid %(A)s, C is Card)')
def test_optional_var_3(self):
constraint1 = ('X in_state S, U in_group G, P require_state S,'
'P name "read", P require_group G')
constraint2 = 'X in_state S, S name "public"'
- rqlst = parse('Any A,C,T WHERE A documented_by C?, C title T')
+ rqlst = parse(u'Any A,C,T WHERE A documented_by C?, C title T')
rewrite(rqlst, {('C', 'X'): (constraint1, constraint2)}, {})
self.assertEqual(rqlst.as_string(),
- "Any A,C,T WHERE A documented_by C?, A is Affaire "
- "WITH C,T BEING (Any C,T WHERE C title T, "
- "(EXISTS(C in_state B, D in_group F, G require_state B, G name 'read', G require_group F)) "
- "OR (EXISTS(C in_state E, E name 'public')), "
- "D eid %(A)s, C is Card)")
+ u'Any A,C,T WHERE A documented_by C?, A is Affaire '
+ 'WITH C,T BEING (Any C,T WHERE C title T, '
+ '(EXISTS(C in_state B, D in_group F, G require_state B, G name "read", G require_group F)) '
+ 'OR (EXISTS(C in_state E, E name "public")), '
+ 'D eid %(A)s, C is Card)')
def test_optional_var_4(self):
constraint1 = 'A created_by U, X documented_by A'
constraint2 = 'A created_by U, X concerne A'
constraint3 = 'X created_by U'
- rqlst = parse('Any X,LA,Y WHERE LA? documented_by X, LA concerne Y')
+ rqlst = parse(u'Any X,LA,Y WHERE LA? documented_by X, LA concerne Y')
rewrite(rqlst, {('LA', 'X'): (constraint1, constraint2),
('X', 'X'): (constraint3,),
('Y', 'X'): (constraint3,)}, {})
@@ -208,7 +210,7 @@
# see test of the same name in RewriteFullTC: original problem is
# unreproducible here because it actually lies in
# RQLRewriter.insert_local_checks
- rqlst = parse('Any A,AR,X,CD WHERE A concerne X?, A ref AR, A eid %(a)s, X creation_date CD')
+ rqlst = parse(u'Any A,AR,X,CD WHERE A concerne X?, A ref AR, A eid %(a)s, X creation_date CD')
rewrite(rqlst, {('X', 'X'): ('X created_by U',),}, {'a': 3})
self.assertEqual(rqlst.as_string(),
u'Any A,AR,X,CD WHERE A concerne X?, A ref AR, A eid %(a)s WITH X,CD BEING (Any X,CD WHERE X creation_date CD, EXISTS(X created_by B), B eid %(A)s, X is IN(Division, Note, Societe))')
@@ -216,7 +218,7 @@
def test_optional_var_inlined(self):
c1 = ('X require_permission P')
c2 = ('X inlined_card O, O require_permission P')
- rqlst = parse('Any C,A,R WHERE A? inlined_card C, A ref R')
+ rqlst = parse(u'Any C,A,R WHERE A? inlined_card C, A ref R')
rewrite(rqlst, {('C', 'X'): (c1,),
('A', 'X'): (c2,),
}, {})
@@ -231,7 +233,7 @@
# def test_optional_var_inlined_has_perm(self):
# c1 = ('X require_permission P')
# c2 = ('X inlined_card O, U has_read_permission O')
- # rqlst = parse('Any C,A,R WHERE A? inlined_card C, A ref R')
+ # rqlst = parse(u'Any C,A,R WHERE A? inlined_card C, A ref R')
# rewrite(rqlst, {('C', 'X'): (c1,),
# ('A', 'X'): (c2,),
# }, {})
@@ -241,7 +243,7 @@
def test_optional_var_inlined_imbricated_error(self):
c1 = ('X require_permission P')
c2 = ('X inlined_card O, O require_permission P')
- rqlst = parse('Any C,A,R,A2,R2 WHERE A? inlined_card C, A ref R,A2? inlined_card C, A2 ref R2')
+ rqlst = parse(u'Any C,A,R,A2,R2 WHERE A? inlined_card C, A ref R,A2? inlined_card C, A2 ref R2')
self.assertRaises(BadSchemaDefinition,
rewrite, rqlst, {('C', 'X'): (c1,),
('A', 'X'): (c2,),
@@ -251,7 +253,7 @@
def test_optional_var_inlined_linked(self):
c1 = ('X require_permission P')
c2 = ('X inlined_card O, O require_permission P')
- rqlst = parse('Any A,W WHERE A inlined_card C?, C inlined_note N, '
+ rqlst = parse(u'Any A,W WHERE A inlined_card C?, C inlined_note N, '
'N inlined_affaire W')
rewrite(rqlst, {('C', 'X'): (c1,)}, {})
self.assertEqual(rqlst.as_string(),
@@ -265,70 +267,70 @@
# relation used in the rql expression can be ignored and S replaced by
# the variable from the incoming query
snippet = ('X in_state S, S name "hop"')
- rqlst = parse('Card C WHERE C in_state STATE')
+ rqlst = parse(u'Card C WHERE C in_state STATE')
rewrite(rqlst, {('C', 'X'): (snippet,)}, {})
self.assertEqual(rqlst.as_string(),
- "Any C WHERE C in_state STATE, C is Card, "
- "EXISTS(STATE name 'hop'), STATE is State")
+ 'Any C WHERE C in_state STATE, C is Card, '
+ 'EXISTS(STATE name "hop"), STATE is State')
def test_relation_optimization_1_rhs(self):
snippet = ('TW subworkflow_exit X, TW name "hop"')
- rqlst = parse('WorkflowTransition C WHERE C subworkflow_exit EXIT')
+ rqlst = parse(u'WorkflowTransition C WHERE C subworkflow_exit EXIT')
rewrite(rqlst, {('EXIT', 'X'): (snippet,)}, {})
self.assertEqual(rqlst.as_string(),
- "Any C WHERE C subworkflow_exit EXIT, C is WorkflowTransition, "
- "EXISTS(C name 'hop'), EXIT is SubWorkflowExitPoint")
+ 'Any C WHERE C subworkflow_exit EXIT, C is WorkflowTransition, '
+ 'EXISTS(C name "hop"), EXIT is SubWorkflowExitPoint')
def test_relation_optimization_2_lhs(self):
# optional relation can be shared if also optional in the snippet
snippet = ('X in_state S?, S name "hop"')
- rqlst = parse('Card C WHERE C in_state STATE?')
+ rqlst = parse(u'Card C WHERE C in_state STATE?')
rewrite(rqlst, {('C', 'X'): (snippet,)}, {})
self.assertEqual(rqlst.as_string(),
- "Any C WHERE C in_state STATE?, C is Card, "
- "EXISTS(STATE name 'hop'), STATE is State")
+ 'Any C WHERE C in_state STATE?, C is Card, '
+ 'EXISTS(STATE name "hop"), STATE is State')
def test_relation_optimization_2_rhs(self):
snippet = ('TW? subworkflow_exit X, TW name "hop"')
- rqlst = parse('SubWorkflowExitPoint EXIT WHERE C? subworkflow_exit EXIT')
+ rqlst = parse(u'SubWorkflowExitPoint EXIT WHERE C? subworkflow_exit EXIT')
rewrite(rqlst, {('EXIT', 'X'): (snippet,)}, {})
self.assertEqual(rqlst.as_string(),
- "Any EXIT WHERE C? subworkflow_exit EXIT, EXIT is SubWorkflowExitPoint, "
- "EXISTS(C name 'hop'), C is WorkflowTransition")
+ 'Any EXIT WHERE C? subworkflow_exit EXIT, EXIT is SubWorkflowExitPoint, '
+ 'EXISTS(C name "hop"), C is WorkflowTransition')
def test_relation_optimization_3_lhs(self):
# optional relation in the snippet but not in the orig tree can be shared
snippet = ('X in_state S?, S name "hop"')
- rqlst = parse('Card C WHERE C in_state STATE')
+ rqlst = parse(u'Card C WHERE C in_state STATE')
rewrite(rqlst, {('C', 'X'): (snippet,)}, {})
self.assertEqual(rqlst.as_string(),
- "Any C WHERE C in_state STATE, C is Card, "
- "EXISTS(STATE name 'hop'), STATE is State")
+ 'Any C WHERE C in_state STATE, C is Card, '
+ 'EXISTS(STATE name "hop"), STATE is State')
def test_relation_optimization_3_rhs(self):
snippet = ('TW? subworkflow_exit X, TW name "hop"')
- rqlst = parse('WorkflowTransition C WHERE C subworkflow_exit EXIT')
+ rqlst = parse(u'WorkflowTransition C WHERE C subworkflow_exit EXIT')
rewrite(rqlst, {('EXIT', 'X'): (snippet,)}, {})
self.assertEqual(rqlst.as_string(),
- "Any C WHERE C subworkflow_exit EXIT, C is WorkflowTransition, "
- "EXISTS(C name 'hop'), EXIT is SubWorkflowExitPoint")
+ 'Any C WHERE C subworkflow_exit EXIT, C is WorkflowTransition, '
+ 'EXISTS(C name "hop"), EXIT is SubWorkflowExitPoint')
def test_relation_non_optimization_1_lhs(self):
# but optional relation in the orig tree but not in the snippet can't be shared
snippet = ('X in_state S, S name "hop"')
- rqlst = parse('Card C WHERE C in_state STATE?')
+ rqlst = parse(u'Card C WHERE C in_state STATE?')
rewrite(rqlst, {('C', 'X'): (snippet,)}, {})
self.assertEqual(rqlst.as_string(),
- "Any C WHERE C in_state STATE?, C is Card, "
- "EXISTS(C in_state A, A name 'hop', A is State), STATE is State")
+ 'Any C WHERE C in_state STATE?, C is Card, '
+ 'EXISTS(C in_state A, A name "hop", A is State), STATE is State')
def test_relation_non_optimization_1_rhs(self):
snippet = ('TW subworkflow_exit X, TW name "hop"')
- rqlst = parse('SubWorkflowExitPoint EXIT WHERE C? subworkflow_exit EXIT')
+ rqlst = parse(u'SubWorkflowExitPoint EXIT WHERE C? subworkflow_exit EXIT')
rewrite(rqlst, {('EXIT', 'X'): (snippet,)}, {})
self.assertEqual(rqlst.as_string(),
- "Any EXIT WHERE C? subworkflow_exit EXIT, EXIT is SubWorkflowExitPoint, "
- "EXISTS(A subworkflow_exit EXIT, A name 'hop', A is WorkflowTransition), "
- "C is WorkflowTransition")
+ 'Any EXIT WHERE C? subworkflow_exit EXIT, EXIT is SubWorkflowExitPoint, '
+ 'EXISTS(A subworkflow_exit EXIT, A name "hop", A is WorkflowTransition), '
+ 'C is WorkflowTransition')
def test_relation_non_optimization_2(self):
"""See #3024730"""
@@ -336,7 +338,7 @@
# previously inserted, else this may introduce duplicated results, as N
# will then be shared by multiple EXISTS and so at SQL generation time,
# the table will be in the FROM clause of the outermost query
- rqlst = parse('Any A,C WHERE A inlined_card C')
+ rqlst = parse(u'Any A,C WHERE A inlined_card C')
rewrite(rqlst, {('A', 'X'): ('X inlined_card C, C inlined_note N, N owned_by U',),
('C', 'X'): ('X inlined_note N, N owned_by U',)}, {})
self.assertEqual(rqlst.as_string(),
@@ -348,35 +350,35 @@
def test_unsupported_constraint_1(self):
# CWUser doesn't have require_permission
trinfo_constraint = ('X wf_info_for Y, Y require_permission P, P name "read"')
- rqlst = parse('Any U,T WHERE U is CWUser, T wf_info_for U')
+ rqlst = parse(u'Any U,T WHERE U is CWUser, T wf_info_for U')
self.assertRaises(Unauthorized, rewrite, rqlst, {('T', 'X'): (trinfo_constraint,)}, {})
def test_unsupported_constraint_2(self):
trinfo_constraint = ('X wf_info_for Y, Y require_permission P, P name "read"')
- rqlst = parse('Any U,T WHERE U is CWUser, T wf_info_for U')
+ rqlst = parse(u'Any U,T WHERE U is CWUser, T wf_info_for U')
rewrite(rqlst, {('T', 'X'): (trinfo_constraint, 'X wf_info_for Y, Y in_group G, G name "managers"')}, {})
self.assertEqual(rqlst.as_string(),
- u"Any U,T WHERE U is CWUser, T wf_info_for U, "
- "EXISTS(U in_group B, B name 'managers', B is CWGroup), T is TrInfo")
+ u'Any U,T WHERE U is CWUser, T wf_info_for U, '
+ 'EXISTS(U in_group B, B name "managers", B is CWGroup), T is TrInfo')
def test_unsupported_constraint_3(self):
self.skipTest('raise unauthorized for now')
trinfo_constraint = ('X wf_info_for Y, Y require_permission P, P name "read"')
- rqlst = parse('Any T WHERE T wf_info_for X')
+ rqlst = parse(u'Any T WHERE T wf_info_for X')
rewrite(rqlst, {('T', 'X'): (trinfo_constraint, 'X in_group G, G name "managers"')}, {})
self.assertEqual(rqlst.as_string(),
u'XXX dunno what should be generated')
def test_add_ambiguity_exists(self):
constraint = ('X concerne Y')
- rqlst = parse('Affaire X')
+ rqlst = parse(u'Affaire X')
rewrite(rqlst, {('X', 'X'): (constraint,)}, {})
self.assertEqual(rqlst.as_string(),
u"Any X WHERE X is Affaire, ((EXISTS(X concerne A, A is Division)) OR (EXISTS(X concerne C, C is Societe))) OR (EXISTS(X concerne B, B is Note))")
def test_add_ambiguity_outerjoin(self):
constraint = ('X concerne Y')
- rqlst = parse('Any X,C WHERE X? documented_by C')
+ rqlst = parse(u'Any X,C WHERE X? documented_by C')
rewrite(rqlst, {('X', 'X'): (constraint,)}, {})
# ambiguity are kept in the sub-query, no need to be resolved using OR
self.assertEqual(rqlst.as_string(),
@@ -385,76 +387,76 @@
def test_rrqlexpr_nonexistant_subject_1(self):
constraint = RRQLExpression('S owned_by U')
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'S'): (constraint,)}, {}, 'SU')
self.assertEqual(rqlst.as_string(),
u"Any C WHERE C is Card, A eid %(B)s, EXISTS(C owned_by A)")
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'S'): (constraint,)}, {}, 'OU')
self.assertEqual(rqlst.as_string(),
u"Any C WHERE C is Card")
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'S'): (constraint,)}, {}, 'SOU')
self.assertEqual(rqlst.as_string(),
u"Any C WHERE C is Card, A eid %(B)s, EXISTS(C owned_by A)")
def test_rrqlexpr_nonexistant_subject_2(self):
constraint = RRQLExpression('S owned_by U, O owned_by U, O is Card')
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'S'): (constraint,)}, {}, 'SU')
self.assertEqual(rqlst.as_string(),
'Any C WHERE C is Card, A eid %(B)s, EXISTS(C owned_by A)')
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'S'): (constraint,)}, {}, 'OU')
self.assertEqual(rqlst.as_string(),
'Any C WHERE C is Card, B eid %(D)s, EXISTS(A owned_by B, A is Card)')
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'S'): (constraint,)}, {}, 'SOU')
self.assertEqual(rqlst.as_string(),
'Any C WHERE C is Card, A eid %(B)s, EXISTS(C owned_by A, D owned_by A, D is Card)')
def test_rrqlexpr_nonexistant_subject_3(self):
constraint = RRQLExpression('U in_group G, G name "users"')
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'S'): (constraint,)}, {}, 'SU')
self.assertEqual(rqlst.as_string(),
u'Any C WHERE C is Card, A eid %(B)s, EXISTS(A in_group D, D name "users", D is CWGroup)')
def test_rrqlexpr_nonexistant_subject_4(self):
constraint = RRQLExpression('U in_group G, G name "users", S owned_by U')
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'S'): (constraint,)}, {}, 'SU')
self.assertEqual(rqlst.as_string(),
u'Any C WHERE C is Card, A eid %(B)s, EXISTS(A in_group D, D name "users", C owned_by A, D is CWGroup)')
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'S'): (constraint,)}, {}, 'OU')
self.assertEqual(rqlst.as_string(),
u'Any C WHERE C is Card, A eid %(B)s, EXISTS(A in_group D, D name "users", D is CWGroup)')
def test_rrqlexpr_nonexistant_subject_5(self):
constraint = RRQLExpression('S owned_by Z, O owned_by Z, O is Card')
- rqlst = parse('Card C')
+ rqlst = parse(u'Card C')
rewrite(rqlst, {('C', 'S'): (constraint,)}, {}, 'S')
self.assertEqual(rqlst.as_string(),
u"Any C WHERE C is Card, EXISTS(C owned_by A, A is CWUser)")
def test_rqlexpr_not_relation_1_1(self):
constraint = ERQLExpression('X owned_by Z, Z login "hop"', 'X')
- rqlst = parse('Affaire A WHERE NOT EXISTS(A documented_by C)')
+ rqlst = parse(u'Affaire A WHERE NOT EXISTS(A documented_by C)')
rewrite(rqlst, {('C', 'X'): (constraint,)}, {}, 'X')
self.assertEqual(rqlst.as_string(),
u'Any A WHERE NOT EXISTS(A documented_by C, EXISTS(C owned_by B, B login "hop", B is CWUser), C is Card), A is Affaire')
def test_rqlexpr_not_relation_1_2(self):
constraint = ERQLExpression('X owned_by Z, Z login "hop"', 'X')
- rqlst = parse('Affaire A WHERE NOT EXISTS(A documented_by C)')
+ rqlst = parse(u'Affaire A WHERE NOT EXISTS(A documented_by C)')
rewrite(rqlst, {('A', 'X'): (constraint,)}, {}, 'X')
self.assertEqual(rqlst.as_string(),
u'Any A WHERE NOT EXISTS(A documented_by C, C is Card), A is Affaire, EXISTS(A owned_by B, B login "hop", B is CWUser)')
def test_rqlexpr_not_relation_2(self):
constraint = ERQLExpression('X owned_by Z, Z login "hop"', 'X')
- rqlst = rqlhelper.parse('Affaire A WHERE NOT A documented_by C', annotate=False)
+ rqlst = rqlhelper.parse(u'Affaire A WHERE NOT A documented_by C', annotate=False)
rewrite(rqlst, {('C', 'X'): (constraint,)}, {}, 'X')
self.assertEqual(rqlst.as_string(),
u'Any A WHERE NOT EXISTS(A documented_by C, EXISTS(C owned_by B, B login "hop", B is CWUser), C is Card), A is Affaire')
@@ -463,7 +465,7 @@
c1 = ERQLExpression('X owned_by Z, Z login "hop"', 'X')
c2 = ERQLExpression('X owned_by Z, Z login "hip"', 'X')
c3 = ERQLExpression('X owned_by Z, Z login "momo"', 'X')
- rqlst = rqlhelper.parse('Any A WHERE A documented_by C?', annotate=False)
+ rqlst = rqlhelper.parse(u'Any A WHERE A documented_by C?', annotate=False)
rewrite(rqlst, {('C', 'X'): (c1, c2, c3)}, {}, 'X')
self.assertEqual(rqlst.as_string(),
u'Any A WHERE A documented_by C?, A is Affaire '
@@ -484,12 +486,12 @@
# 4. this variable require a rewrite
c_bad = ERQLExpression('X documented_by R, A in_state R')
- rqlst = parse('Any A, R WHERE A ref R, S is Affaire')
+ rqlst = parse(u'Any A, R WHERE A ref R, S is Affaire')
rewrite(rqlst, {('A', 'X'): (c_ok, c_bad)}, {})
def test_nonregr_is_instance_of(self):
user_expr = ERQLExpression('NOT X in_group AF, AF name "guests"')
- rqlst = parse('Any O WHERE S use_email O, S is CWUser, O is_instance_of EmailAddress')
+ rqlst = parse(u'Any O WHERE S use_email O, S is CWUser, O is_instance_of EmailAddress')
rewrite(rqlst, {('S', 'X'): (user_expr,)}, {})
self.assertEqual(rqlst.as_string(),
'Any O WHERE S use_email O, S is CWUser, O is EmailAddress, '
@@ -600,7 +602,7 @@
# Basic tests
def test_base_rule(self):
rules = {'participated_in': 'S contributor O'}
- rqlst = rqlhelper.parse('Any X WHERE X participated_in S')
+ rqlst = rqlhelper.parse(u'Any X WHERE X participated_in S')
rule_rewrite(rqlst, rules)
self.assertEqual('Any X WHERE X contributor S',
rqlst.as_string())
@@ -609,7 +611,7 @@
rules = {'illustrator_of': ('C is Contribution, C contributor S, '
'C manifestation O, C role R, '
'R name "illustrator"')}
- rqlst = rqlhelper.parse('Any A,B WHERE A illustrator_of B')
+ rqlst = rqlhelper.parse(u'Any A,B WHERE A illustrator_of B')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE C is Contribution, '
'C contributor A, C manifestation B, '
@@ -620,7 +622,7 @@
rules = {'illustrator_of': ('C is Contribution, C contributor S, '
'C manifestation O, C role R, '
'R name "illustrator"')}
- rqlst = rqlhelper.parse('Any A WHERE EXISTS(A illustrator_of B)')
+ rqlst = rqlhelper.parse(u'Any A WHERE EXISTS(A illustrator_of B)')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A WHERE EXISTS(C is Contribution, '
'C contributor A, C manifestation B, '
@@ -631,7 +633,7 @@
def test_rewrite2(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('Any A,B WHERE A illustrator_of B, C require_permission R, S'
+ rqlst = rqlhelper.parse(u'Any A,B WHERE A illustrator_of B, C require_permission R, S'
'require_state O')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE C require_permission R, S require_state O, '
@@ -642,7 +644,7 @@
def test_rewrite3(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('Any A,B WHERE E require_permission T, A illustrator_of B')
+ rqlst = rqlhelper.parse(u'Any A,B WHERE E require_permission T, A illustrator_of B')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE E require_permission T, '
'C is Contribution, C contributor A, C manifestation B, '
@@ -652,7 +654,7 @@
def test_rewrite4(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('Any A,B WHERE C require_permission R, A illustrator_of B')
+ rqlst = rqlhelper.parse(u'Any A,B WHERE C require_permission R, A illustrator_of B')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE C require_permission R, '
'D is Contribution, D contributor A, D manifestation B, '
@@ -662,7 +664,7 @@
def test_rewrite5(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('Any A,B WHERE C require_permission R, A illustrator_of B, '
+ rqlst = rqlhelper.parse(u'Any A,B WHERE C require_permission R, A illustrator_of B, '
'S require_state O')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE C require_permission R, S require_state O, '
@@ -674,7 +676,7 @@
def test_rewrite_with(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('Any A,B WITH A, B BEING(Any X, Y WHERE X illustrator_of Y)')
+ rqlst = rqlhelper.parse(u'Any A,B WITH A, B BEING(Any X, Y WHERE X illustrator_of Y)')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WITH A,B BEING '
'(Any X,Y WHERE A is Contribution, A contributor X, '
@@ -684,7 +686,7 @@
def test_rewrite_with2(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('Any A,B WHERE T require_permission C WITH A, B BEING(Any X, Y WHERE X illustrator_of Y)')
+ rqlst = rqlhelper.parse(u'Any A,B WHERE T require_permission C WITH A, B BEING(Any X, Y WHERE X illustrator_of Y)')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE T require_permission C '
'WITH A,B BEING (Any X,Y WHERE A is Contribution, '
@@ -693,7 +695,7 @@
def test_rewrite_with3(self):
rules = {'participated_in': 'S contributor O'}
- rqlst = rqlhelper.parse('Any A,B WHERE A participated_in B '
+ rqlst = rqlhelper.parse(u'Any A,B WHERE A participated_in B '
'WITH A, B BEING(Any X,Y WHERE X contributor Y)')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE A contributor B WITH A,B BEING '
@@ -703,7 +705,7 @@
def test_rewrite_with4(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('Any A,B WHERE A illustrator_of B '
+ rqlst = rqlhelper.parse(u'Any A,B WHERE A illustrator_of B '
'WITH A, B BEING(Any X, Y WHERE X illustrator_of Y)')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE C is Contribution, '
@@ -717,7 +719,7 @@
def test_rewrite_union(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('(Any A,B WHERE A illustrator_of B) UNION'
+ rqlst = rqlhelper.parse(u'(Any A,B WHERE A illustrator_of B) UNION'
'(Any X,Y WHERE X is CWUser, Z manifestation Y)')
rule_rewrite(rqlst, rules)
self.assertEqual('(Any A,B WHERE C is Contribution, '
@@ -728,7 +730,7 @@
def test_rewrite_union2(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('(Any Y WHERE Y match W) UNION '
+ rqlst = rqlhelper.parse(u'(Any Y WHERE Y match W) UNION '
'(Any A WHERE A illustrator_of B) UNION '
'(Any Y WHERE Y is ArtWork)')
rule_rewrite(rqlst, rules)
@@ -742,7 +744,7 @@
def test_rewrite_exists(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('(Any A,B WHERE A illustrator_of B, '
+ rqlst = rqlhelper.parse(u'(Any A,B WHERE A illustrator_of B, '
'EXISTS(B is ArtWork))')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE EXISTS(B is ArtWork), '
@@ -753,7 +755,7 @@
def test_rewrite_exists2(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('(Any A,B WHERE B contributor A, EXISTS(A illustrator_of W))')
+ rqlst = rqlhelper.parse(u'(Any A,B WHERE B contributor A, EXISTS(A illustrator_of W))')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE B contributor A, '
'EXISTS(C is Contribution, C contributor A, C manifestation W, '
@@ -763,7 +765,7 @@
def test_rewrite_exists3(self):
rules = {'illustrator_of': 'C is Contribution, C contributor S, '
'C manifestation O, C role R, R name "illustrator"'}
- rqlst = rqlhelper.parse('(Any A,B WHERE A illustrator_of B, EXISTS(A illustrator_of W))')
+ rqlst = rqlhelper.parse(u'(Any A,B WHERE A illustrator_of B, EXISTS(A illustrator_of W))')
rule_rewrite(rqlst, rules)
self.assertEqual('Any A,B WHERE EXISTS(C is Contribution, C contributor A, '
'C manifestation W, C role D, D name "illustrator"), '
@@ -774,7 +776,7 @@
# Test for GROUPBY
def test_rewrite_groupby(self):
rules = {'participated_in': 'S contributor O'}
- rqlst = rqlhelper.parse('Any SUM(SA) GROUPBY S WHERE P participated_in S, P manifestation SA')
+ rqlst = rqlhelper.parse(u'Any SUM(SA) GROUPBY S WHERE P participated_in S, P manifestation SA')
rule_rewrite(rqlst, rules)
self.assertEqual('Any SUM(SA) GROUPBY S WHERE P manifestation SA, P contributor S',
rqlst.as_string())
--- a/test/unittest_rset.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_rset.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,8 +18,9 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""unit tests for module cubicweb.utils"""
-from urlparse import urlsplit
-import pickle
+from six import string_types
+from six.moves import cPickle as pickle
+from six.moves.urllib.parse import urlsplit
from rql import parse
@@ -100,7 +101,11 @@
def test_pickle(self):
del self.rset.req
- self.assertEqual(len(pickle.dumps(self.rset)), 376)
+ rs2 = pickle.loads(pickle.dumps(self.rset))
+ self.assertEqual(self.rset.rows, rs2.rows)
+ self.assertEqual(self.rset.rowcount, rs2.rowcount)
+ self.assertEqual(self.rset.rql, rs2.rql)
+ self.assertEqual(self.rset.description, rs2.description)
def test_build_url(self):
with self.admin_access.web_request() as req:
@@ -274,7 +279,7 @@
"""make sure syntax tree is cached"""
rqlst1 = self.rset.syntax_tree()
rqlst2 = self.rset.syntax_tree()
- self.assert_(rqlst1 is rqlst2)
+ self.assertIs(rqlst1, rqlst2)
def test_get_entity_simple(self):
with self.admin_access.web_request() as req:
@@ -550,19 +555,32 @@
def test_str(self):
with self.admin_access.web_request() as req:
rset = req.execute('(Any X,N WHERE X is CWGroup, X name N)')
- self.assertIsInstance(str(rset), basestring)
+ self.assertIsInstance(str(rset), string_types)
self.assertEqual(len(str(rset).splitlines()), 1)
def test_repr(self):
with self.admin_access.web_request() as req:
rset = req.execute('(Any X,N WHERE X is CWGroup, X name N)')
- self.assertIsInstance(repr(rset), basestring)
+ self.assertIsInstance(repr(rset), string_types)
self.assertTrue(len(repr(rset).splitlines()) > 1)
rset = req.execute('(Any X WHERE X is CWGroup, X name "managers")')
- self.assertIsInstance(str(rset), basestring)
+ self.assertIsInstance(str(rset), string_types)
self.assertEqual(len(str(rset).splitlines()), 1)
+ def test_slice(self):
+ rs = ResultSet([[12000, 'adim', u'Adim chez les pinguins'],
+ [12000, 'adim', u'Jardiner facile'],
+ [13000, 'syt', u'Le carrelage en 42 leçons'],
+ [14000, 'nico', u'La tarte tatin en 15 minutes'],
+ [14000, 'nico', u"L'épluchage du castor commun"]],
+ 'Any U, L, T WHERE U is CWUser, U login L,'\
+ 'D created_by U, D title T',
+ description=[['CWUser', 'String', 'String']] * 5)
+ self.assertEqual(rs[1::2],
+ [[12000, 'adim', u'Jardiner facile'],
+ [14000, 'nico', u'La tarte tatin en 15 minutes']])
+
def test_nonregr_symmetric_relation(self):
# see https://www.cubicweb.org/ticket/4739253
with self.admin_access.client_cnx() as cnx:
--- a/test/unittest_schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -105,10 +105,9 @@
#
# isinstance(cstr, RQLConstraint)
# -> expected to return RQLConstraint instances but not
- # RRQLVocabularyConstraint and QLUniqueConstraint
+ # RQLVocabularyConstraint and RQLUniqueConstraint
self.assertFalse(issubclass(RQLUniqueConstraint, RQLVocabularyConstraint))
self.assertFalse(issubclass(RQLUniqueConstraint, RQLConstraint))
- self.assertTrue(issubclass(RQLConstraint, RQLVocabularyConstraint))
def test_entity_perms(self):
self.assertEqual(eperson.get_groups('read'), set(('managers', 'users', 'guests')))
@@ -158,7 +157,7 @@
def test_knownValues_load_schema(self):
schema = loader.load(config)
- self.assert_(isinstance(schema, CubicWebSchema))
+ self.assertIsInstance(schema, CubicWebSchema)
self.assertEqual(schema.name, 'data')
entities = sorted([str(e) for e in schema.entities()])
expected_entities = ['Ami', 'BaseTransition', 'BigInt', 'Bookmark', 'Boolean', 'Bytes', 'Card',
@@ -273,11 +272,13 @@
config = TestConfiguration('data', apphome=join(dirname(__file__), 'data_schemareader'))
config.bootstrap_cubes()
schema = loader.load(config)
- self.assertEqual(schema['in_group'].rdefs.values()[0].permissions,
+ rdef = next(iter(schema['in_group'].rdefs.values()))
+ self.assertEqual(rdef.permissions,
{'read': ('managers',),
'add': ('managers',),
'delete': ('managers',)})
- self.assertEqual(schema['cw_for_source'].rdefs.values()[0].permissions,
+ rdef = next(iter(schema['cw_for_source'].rdefs.values()))
+ self.assertEqual(rdef.permissions,
{'read': ('managers', 'users'),
'add': ('managers',),
'delete': ('managers',)})
@@ -355,11 +356,11 @@
# check object/subject type
self.assertEqual([('Person','Service')],
- schema['produces_and_buys'].rdefs.keys())
+ list(schema['produces_and_buys'].rdefs.keys()))
self.assertEqual([('Person','Service')],
- schema['produces_and_buys2'].rdefs.keys())
+ list(schema['produces_and_buys2'].rdefs.keys()))
self.assertCountEqual([('Company', 'Service'), ('Person', 'Service')],
- schema['reproduce'].rdefs.keys())
+ list(schema['reproduce'].rdefs.keys()))
# check relation definitions are marked infered
rdef = schema['produces_and_buys'].rdefs[('Person','Service')]
self.assertTrue(rdef.infered)
@@ -426,7 +427,9 @@
def test(self):
self.assertEqual(normalize_expression('X bla Y,Y blur Z , Z zigoulou X '),
- 'X bla Y, Y blur Z, Z zigoulou X')
+ 'X bla Y, Y blur Z, Z zigoulou X')
+ self.assertEqual(normalize_expression('X bla Y, Y name "x,y"'),
+ 'X bla Y, Y name "x,y"')
class RQLExpressionTC(TestCase):
@@ -553,7 +556,7 @@
self.set_description('composite rdefs for %s' % etype)
yield self.assertEqual, self.composites[etype], \
sorted([(r.rtype.type, r.subject.type, r.object.type, role)
- for r, role in sorted(schema[etype].composite_rdef_roles)])
+ for r, role in schema[etype].composite_rdef_roles])
if __name__ == '__main__':
--- a/test/unittest_uilib.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_uilib.py Wed Dec 16 11:23:48 2015 +0100
@@ -200,4 +200,3 @@
if __name__ == '__main__':
unittest_main()
-
--- a/test/unittest_utils.py Tue Dec 15 19:05:41 2015 +0100
+++ b/test/unittest_utils.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,12 +21,13 @@
import decimal
import datetime
+from six.moves import range
from logilab.common.testlib import TestCase, DocTest, unittest_main
from cubicweb.devtools.testlib import CubicWebTC
-from cubicweb.utils import (make_uid, UStringIO, SizeConstrainedList,
- RepeatList, HTMLHead, QueryCache, parse_repo_uri)
+from cubicweb.utils import (make_uid, UStringIO, RepeatList, HTMLHead,
+ QueryCache, parse_repo_uri)
from cubicweb.entity import Entity
try:
@@ -67,7 +68,7 @@
def test_querycache(self):
c = QueryCache(ceiling=20)
# write only
- for x in xrange(10):
+ for x in range(10):
c[x] = x
self.assertEqual(c._usage_report(),
{'transientcount': 0,
@@ -75,7 +76,7 @@
'permanentcount': 0})
c = QueryCache(ceiling=10)
# we should also get a warning
- for x in xrange(20):
+ for x in range(20):
c[x] = x
self.assertEqual(c._usage_report(),
{'transientcount': 0,
@@ -83,8 +84,8 @@
'permanentcount': 0})
# write + reads
c = QueryCache(ceiling=20)
- for n in xrange(4):
- for x in xrange(10):
+ for n in range(4):
+ for x in range(10):
c[x] = x
c[x]
self.assertEqual(c._usage_report(),
@@ -92,8 +93,8 @@
'itemcount': 10,
'permanentcount': 0})
c = QueryCache(ceiling=20)
- for n in xrange(17):
- for x in xrange(10):
+ for n in range(17):
+ for x in range(10):
c[x] = x
c[x]
self.assertEqual(c._usage_report(),
@@ -101,8 +102,8 @@
'itemcount': 10,
'permanentcount': 10})
c = QueryCache(ceiling=20)
- for n in xrange(17):
- for x in xrange(10):
+ for n in range(17):
+ for x in range(10):
c[x] = x
if n % 2:
c[x]
@@ -115,7 +116,7 @@
class UStringIOTC(TestCase):
def test_boolean_value(self):
- self.assert_(UStringIO())
+ self.assertTrue(UStringIO())
class RepeatListTC(TestCase):
@@ -165,25 +166,6 @@
self.assertEqual(l, [(1, 3)]*2)
-class SizeConstrainedListTC(TestCase):
-
- def test_append(self):
- l = SizeConstrainedList(10)
- for i in xrange(12):
- l.append(i)
- self.assertEqual(l, range(2, 12))
-
- def test_extend(self):
- testdata = [(range(5), range(5)),
- (range(10), range(10)),
- (range(12), range(2, 12)),
- ]
- for extension, expected in testdata:
- l = SizeConstrainedList(10)
- l.extend(extension)
- yield self.assertEqual, l, expected
-
-
class JSONEncoderTC(TestCase):
def setUp(self):
if json is None:
--- a/toolsutils.py Tue Dec 15 19:05:41 2015 +0100
+++ b/toolsutils.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,11 +16,13 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""some utilities for cubicweb command line tools"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
# XXX move most of this in logilab.common (shellutils ?)
+import io
import os, sys
import subprocess
from os import listdir, makedirs, environ, chmod, walk, remove
@@ -37,6 +39,8 @@
def symlink(*args):
raise NotImplementedError
+from six import add_metaclass
+
from logilab.common.clcommands import Command as BaseCommand
from logilab.common.shellutils import ASK
@@ -62,29 +66,29 @@
"""create a directory if it doesn't exist yet"""
try:
makedirs(directory)
- print '-> created directory %s' % directory
+ print('-> created directory %s' % directory)
except OSError as ex:
import errno
if ex.errno != errno.EEXIST:
raise
- print '-> no need to create existing directory %s' % directory
+ print('-> no need to create existing directory %s' % directory)
def create_symlink(source, target):
"""create a symbolic link"""
if exists(target):
remove(target)
symlink(source, target)
- print '[symlink] %s <-- %s' % (target, source)
+ print('[symlink] %s <-- %s' % (target, source))
def create_copy(source, target):
import shutil
- print '[copy] %s <-- %s' % (target, source)
+ print('[copy] %s <-- %s' % (target, source))
shutil.copy2(source, target)
def rm(whatever):
import shutil
shutil.rmtree(whatever)
- print '-> removed %s' % whatever
+ print('-> removed %s' % whatever)
def show_diffs(appl_file, ref_file, askconfirm=True):
"""interactivly replace the old file with the new file according to
@@ -95,8 +99,8 @@
diffs = pipe.stdout.read()
if diffs:
if askconfirm:
- print
- print diffs
+ print()
+ print(diffs)
action = ASK.ask('Replace ?', ('Y', 'n', 'q'), 'Y').lower()
else:
action = 'y'
@@ -106,17 +110,17 @@
except IOError:
os.system('chmod a+w %s' % appl_file)
shutil.copyfile(ref_file, appl_file)
- print 'replaced'
+ print('replaced')
elif action == 'q':
sys.exit(0)
else:
copy_file = appl_file + '.default'
- copy = file(copy_file, 'w')
+ copy = open(copy_file, 'w')
copy.write(open(ref_file).read())
copy.close()
- print 'keep current version, the new file has been written to', copy_file
+ print('keep current version, the new file has been written to', copy_file)
else:
- print 'no diff between %s and %s' % (appl_file, ref_file)
+ print('no diff between %s and %s' % (appl_file, ref_file))
SKEL_EXCLUDE = ('*.py[co]', '*.orig', '*~', '*_flymake.py')
def copy_skeleton(skeldir, targetdir, context,
@@ -143,25 +147,25 @@
if not askconfirm or not exists(tfpath) or \
ASK.confirm('%s exists, overwrite?' % tfpath):
fill_templated_file(fpath, tfpath, context)
- print '[generate] %s <-- %s' % (tfpath, fpath)
+ print('[generate] %s <-- %s' % (tfpath, fpath))
elif exists(tfpath):
show_diffs(tfpath, fpath, askconfirm)
else:
shutil.copyfile(fpath, tfpath)
def fill_templated_file(fpath, tfpath, context):
- fobj = file(tfpath, 'w')
- templated = file(fpath).read()
- fobj.write(templated % context)
- fobj.close()
+ with io.open(fpath, encoding='ascii') as fobj:
+ template = fobj.read()
+ with io.open(tfpath, 'w', encoding='ascii') as fobj:
+ fobj.write(template % context)
def restrict_perms_to_user(filepath, log=None):
"""set -rw------- permission on the given file"""
if log:
log('set permissions to 0600 for %s', filepath)
else:
- print '-> set permissions to 0600 for %s' % filepath
- chmod(filepath, 0600)
+ print('-> set permissions to 0600 for %s' % filepath)
+ chmod(filepath, 0o600)
def read_config(config_file, raise_if_unreadable=False):
"""read some simple configuration from `config_file` and return it as a
@@ -209,12 +213,13 @@
return cls
+@add_metaclass(metacmdhandler)
class CommandHandler(object):
"""configuration specific helper for cubicweb-ctl commands"""
- __metaclass__ = metacmdhandler
def __init__(self, config):
self.config = config
+
class Command(BaseCommand):
"""base class for cubicweb-ctl commands"""
@@ -234,7 +239,7 @@
raise ConfigurationError(msg)
def fail(self, reason):
- print "command failed:", reason
+ print("command failed:", reason)
sys.exit(1)
--- a/tox.ini Tue Dec 15 19:05:41 2015 +0100
+++ b/tox.ini Wed Dec 16 11:23:48 2015 +0100
@@ -1,50 +1,26 @@
[tox]
-env = py27
+envlist = cubicweb,dataimport,devtools,entities,etwist,ext,hooks,server,sobjects,web,wsgi
[testenv]
sitepackages = True
-commands = pytest -t {envname}/test {posargs}
+deps =
+ cubicweb: -r{toxinidir}/test/requirements.txt
+ devtools: -r{toxinidir}/devtools/test/requirements.txt
+ entities: -r{toxinidir}/entities/test/requirements.txt
+ etwist: -r{toxinidir}/etwist/test/requirements.txt
+ ext: -r{toxinidir}/ext/test/requirements.txt
+ hooks: -r{toxinidir}/hooks/test/requirements.txt
+ server: -r{toxinidir}/server/test/requirements.txt
+ sobjects: -r{toxinidir}/sobjects/test/requirements.txt
+ web: -r{toxinidir}/web/test/requirements.txt
+ wsgi: -r{toxinidir}/wsgi/test/requirements.txt
+commands =
+ {envpython} -c 'from logilab.common import pytest; pytest.run()' -t {toxinidir}/{envname}/test {posargs}
[testenv:cubicweb]
-deps =
- -r{toxinidir}/test/requirements.txt
-commands = pytest -t test {posargs}
-
-[testenv:dataimport]
-
-[testenv:devtools]
-deps =
- -r{toxinidir}/devtools/test/requirements.txt
-
-[testenv:entities]
-deps =
- -r{toxinidir}/entities/test/requirements.txt
-
-[testenv:etwist]
-deps =
- -r{toxinidir}/etwist/test/requirements.txt
-
-[testenv:ext]
-deps =
- -r{toxinidir}/ext/test/requirements.txt
-
-[testenv:hooks]
-
-[testenv:server]
-deps =
- -r{toxinidir}/server/test/requirements.txt
-
-[testenv:sobjects]
-deps =
- -r{toxinidir}/sobjects/test/requirements.txt
-
-[testenv:web]
-deps =
- -r{toxinidir}/web/test/requirements.txt
-
-[testenv:wsgi]
-deps =
- -r{toxinidir}/wsgi/test/requirements.txt
+commands =
+ {envpython} -m pip install --upgrade --no-deps --quiet git+git://github.com/logilab/yapps@master#egg=yapps
+ {envpython} -c 'from logilab.common import pytest; pytest.run()' -t {toxinidir}/test {posargs}
[testenv:doc]
changedir = doc
--- a/transaction.py Tue Dec 15 19:05:41 2015 +0100
+++ b/transaction.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,7 +17,7 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
""" undoable transaction objects. """
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from cubicweb import RepositoryError
--- a/uilib.py Tue Dec 15 19:05:41 2015 +0100
+++ b/uilib.py Wed Dec 16 11:23:48 2015 +0100
@@ -26,12 +26,15 @@
import csv
import re
-from StringIO import StringIO
+from io import StringIO
+
+from six import PY2, PY3, text_type, binary_type, string_types, integer_types
from logilab.mtconverter import xml_escape, html_unescape
from logilab.common.date import ustrftime
from logilab.common.deprecation import deprecated
+from cubicweb import _
from cubicweb.utils import js_dumps
@@ -62,7 +65,7 @@
return value
def print_int(value, req, props, displaytime=True):
- return unicode(value)
+ return text_type(value)
def print_date(value, req, props, displaytime=True):
return ustrftime(value, req.property_value('ui.date-format'))
@@ -92,7 +95,7 @@
_('%d seconds')
def print_timedelta(value, req, props, displaytime=True):
- if isinstance(value, (int, long)):
+ if isinstance(value, integer_types):
# `date - date`, unlike `datetime - datetime` gives an int
# (number of days), not a timedelta
# XXX should rql be fixed to return Int instead of Interval in
@@ -122,7 +125,7 @@
return req._('no')
def print_float(value, req, props, displaytime=True):
- return unicode(req.property_value('ui.float-format') % value)
+ return text_type(req.property_value('ui.float-format') % value) # XXX cast needed ?
PRINTERS = {
'Bytes': print_bytes,
@@ -337,9 +340,8 @@
def __unicode__(self):
if self.parent:
return u'%s.%s' % (self.parent, self.id)
- return unicode(self.id)
- def __str__(self):
- return unicode(self).encode('utf8')
+ return text_type(self.id)
+ __str__ = __unicode__ if PY3 else lambda self: self.__unicode__().encode('utf-8')
def __getattr__(self, attr):
return _JSId(attr, self)
def __call__(self, *args):
@@ -357,6 +359,7 @@
if self.parent:
return u'%s(%s)' % (self.parent, ','.join(args))
return ','.join(args)
+ __str__ = __unicode__ if PY3 else lambda self: self.__unicode__().encode('utf-8')
class _JS(object):
def __getattr__(self, attr):
@@ -389,7 +392,7 @@
'img', 'area', 'input', 'col'))
def sgml_attributes(attrs):
- return u' '.join(u'%s="%s"' % (attr, xml_escape(unicode(value)))
+ return u' '.join(u'%s="%s"' % (attr, xml_escape(text_type(value)))
for attr, value in sorted(attrs.items())
if value is not None)
@@ -407,7 +410,7 @@
value += u' ' + sgml_attributes(attrs)
if content:
if escapecontent:
- content = xml_escape(unicode(content))
+ content = xml_escape(text_type(content))
value += u'>%s</%s>' % (content, tag)
else:
if tag in HTML4_EMPTY_TAGS:
@@ -436,8 +439,8 @@
stream = StringIO() #UStringIO() don't want unicode assertion
formater.format(layout, stream)
res = stream.getvalue()
- if isinstance(res, str):
- res = unicode(res, 'UTF8')
+ if isinstance(res, binary_type):
+ res = res.decode('UTF8')
return res
# traceback formatting ########################################################
@@ -445,14 +448,17 @@
import traceback
def exc_message(ex, encoding):
- try:
- excmsg = unicode(ex)
- except Exception:
+ if PY3:
+ excmsg = str(ex)
+ else:
try:
- excmsg = unicode(str(ex), encoding, 'replace')
+ excmsg = unicode(ex)
except Exception:
- excmsg = unicode(repr(ex), encoding, 'replace')
- exctype = unicode(ex.__class__.__name__)
+ try:
+ excmsg = unicode(str(ex), encoding, 'replace')
+ except Exception:
+ excmsg = unicode(repr(ex), encoding, 'replace')
+ exctype = ex.__class__.__name__
return u'%s: %s' % (exctype, excmsg)
@@ -462,7 +468,10 @@
for stackentry in traceback.extract_tb(info[2]):
res.append(u'\tFile %s, line %s, function %s' % tuple(stackentry[:3]))
if stackentry[3]:
- res.append(u'\t %s' % stackentry[3].decode('utf-8', 'replace'))
+ data = xml_escape(stackentry[3])
+ if PY2:
+ data = data.decode('utf-8', 'replace')
+ res.append(u'\t %s' % data)
res.append(u'\n')
try:
res.append(u'\t Error: %s\n' % exception)
@@ -496,14 +505,16 @@
u'<b class="function">%s</b>:<br/>'%(
xml_escape(stackentry[0]), stackentry[1], xml_escape(stackentry[2])))
if stackentry[3]:
- string = xml_escape(stackentry[3]).decode('utf-8', 'replace')
+ string = xml_escape(stackentry[3])
+ if PY2:
+ string = string.decode('utf-8', 'replace')
strings.append(u'  %s<br/>\n' % (string))
# add locals info for each entry
try:
local_context = tcbk.tb_frame.f_locals
html_info = []
chars = 0
- for name, value in local_context.iteritems():
+ for name, value in local_context.items():
value = xml_escape(repr(value))
info = u'<span class="name">%s</span>=%s, ' % (name, value)
line_length = len(name) + len(value)
@@ -526,7 +537,9 @@
# csv files / unicode support #################################################
class UnicodeCSVWriter:
- """proxies calls to csv.writer.writerow to be able to deal with unicode"""
+ """proxies calls to csv.writer.writerow to be able to deal with unicode
+
+ Under Python 3, this code no longer encodes anything."""
def __init__(self, wfunc, encoding, **kwargs):
self.writer = csv.writer(self, **kwargs)
@@ -537,9 +550,12 @@
self.wfunc(data)
def writerow(self, row):
+ if PY3:
+ self.writer.writerow(row)
+ return
csvrow = []
for elt in row:
- if isinstance(elt, unicode):
+ if isinstance(elt, text_type):
csvrow.append(elt.encode(self.encoding))
else:
csvrow.append(str(elt))
@@ -559,7 +575,7 @@
def __call__(self, function):
def newfunc(*args, **kwargs):
ret = function(*args, **kwargs)
- if isinstance(ret, basestring):
+ if isinstance(ret, string_types):
return ret[:self.maxsize]
return ret
return newfunc
@@ -568,6 +584,6 @@
def htmlescape(function):
def newfunc(*args, **kwargs):
ret = function(*args, **kwargs)
- assert isinstance(ret, basestring)
+ assert isinstance(ret, string_types)
return xml_escape(ret)
return newfunc
--- a/utils.py Tue Dec 15 19:05:41 2015 +0100
+++ b/utils.py Wed Dec 16 11:23:48 2015 +0100
@@ -33,9 +33,10 @@
from uuid import uuid4
from warnings import warn
from threading import Lock
-from urlparse import urlparse
+from logging import getLogger
-from logging import getLogger
+from six import text_type
+from six.moves.urllib.parse import urlparse
from logilab.mtconverter import xml_escape
from logilab.common.deprecation import deprecated
@@ -100,7 +101,7 @@
"""
def __init__(self, w, tag, closetag=None):
self.written = False
- self.tag = unicode(tag)
+ self.tag = text_type(tag)
self.closetag = closetag
self.w = w
@@ -116,7 +117,7 @@
def __exit__(self, exctype, value, traceback):
if self.written is True:
if self.closetag:
- self.w(unicode(self.closetag))
+ self.w(text_type(self.closetag))
else:
self.w(self.tag.replace('<', '</', 1))
@@ -138,39 +139,6 @@
yield subchild
-class SizeConstrainedList(list):
- """simple list that makes sure the list does not get bigger than a given
- size.
-
- when the list is full and a new element is added, the first element of the
- list is removed before appending the new one
-
- >>> l = SizeConstrainedList(2)
- >>> l.append(1)
- >>> l.append(2)
- >>> l
- [1, 2]
- >>> l.append(3)
- >>> l
- [2, 3]
- """
- def __init__(self, maxsize):
- self.maxsize = maxsize
-
- def append(self, element):
- if len(self) == self.maxsize:
- del self[0]
- super(SizeConstrainedList, self).append(element)
-
- def extend(self, sequence):
- super(SizeConstrainedList, self).extend(sequence)
- keepafter = len(self) - self.maxsize
- if keepafter > 0:
- del self[:keepafter]
-
- __iadd__ = extend
-
-
class RepeatList(object):
"""fake a list with the same element in each row"""
__slots__ = ('_size', '_item')
@@ -185,13 +153,13 @@
def __iter__(self):
return repeat(self._item, self._size)
def __getitem__(self, index):
+ if isinstance(index, slice):
+ # XXX could be more efficient, but do we bother?
+ return ([self._item] * self._size)[index]
return self._item
def __delitem__(self, idc):
assert self._size > 0
self._size -= 1
- def __getslice__(self, i, j):
- # XXX could be more efficient, but do we bother?
- return ([self._item] * self._size)[i:j]
def __add__(self, other):
if isinstance(other, RepeatList):
if other._item == self._item:
@@ -208,8 +176,10 @@
if isinstance(other, RepeatList):
return other._size == self._size and other._item == self._item
return self[:] == other
- # py3k future warning "Overriding __eq__ blocks inheritance of __hash__ in 3.x"
- # is annoying but won't go away because we don't want to hash() the repeatlist
+ def __ne__(self, other):
+ return not (self == other)
+ def __hash__(self):
+ raise NotImplementedError
def pop(self, i):
self._size -= 1
@@ -223,11 +193,13 @@
self.tracewrites = tracewrites
super(UStringIO, self).__init__(*args, **kwargs)
- def __nonzero__(self):
+ def __bool__(self):
return True
+ __nonzero__ = __bool__
+
def write(self, value):
- assert isinstance(value, unicode), u"unicode required not %s : %s"\
+ assert isinstance(value, text_type), u"unicode required not %s : %s"\
% (type(value).__name__, repr(value))
if self.tracewrites:
from traceback import format_stack
@@ -553,9 +525,9 @@
def _dict2js(d, predictable=False):
if predictable:
- it = sorted(d.iteritems())
+ it = sorted(d.items())
else:
- it = d.iteritems()
+ it = d.items()
res = [key + ': ' + js_dumps(val, predictable)
for key, val in it]
return '{%s}' % ', '.join(res)
--- a/view.py Tue Dec 15 19:05:41 2015 +0100
+++ b/view.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,12 +18,14 @@
"""abstract views and templates classes for CubicWeb web client"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from io import BytesIO
from warnings import warn
from functools import partial
+from six.moves import range
+
from logilab.common.deprecation import deprecated
from logilab.common.registry import yes
from logilab.mtconverter import xml_escape
@@ -173,7 +175,7 @@
# specific view
if rset.rowcount != 1:
kwargs.setdefault('initargs', self.cw_extra_kwargs)
- for i in xrange(len(rset)):
+ for i in range(len(rset)):
if wrap:
self.w(u'<div class="section">')
self.wview(self.__regid__, rset, row=i, **kwargs)
@@ -213,7 +215,7 @@
return self._cw.build_url('view', vid=self.__regid__)
coltypes = rset.column_types(0)
if len(coltypes) == 1:
- etype = iter(coltypes).next()
+ etype = next(iter(coltypes))
if not self._cw.vreg.schema.eschema(etype).final:
if len(rset) == 1:
entity = rset.get_entity(0, 0)
@@ -281,7 +283,7 @@
else :
etypes = rset.column_types(0)
if len(etypes) == 1:
- etype = iter(etypes).next()
+ etype = next(iter(etypes))
clabel = display_name(self._cw, etype, 'plural')
else :
clabel = u'#[*] (%s)' % vtitle
@@ -394,7 +396,7 @@
if rset is None:
rset = self.cw_rset = self._cw.execute(self.startup_rql())
if rset:
- for i in xrange(len(rset)):
+ for i in range(len(rset)):
self.wview(self.__regid__, rset, row=i, **kwargs)
else:
self.no_entities(**kwargs)
--- a/web/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,10 +20,9 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
-from urllib import quote as urlquote
-
+from six.moves.urllib.parse import quote as urlquote
from logilab.common.deprecation import deprecated
from cubicweb.web._exceptions import *
--- a/web/_exceptions.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/_exceptions.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,7 +20,7 @@
__docformat__ = "restructuredtext en"
-import httplib
+from six.moves import http_client
from cubicweb._exceptions import *
from cubicweb.utils import json_dumps
@@ -41,7 +41,7 @@
"""base class for publishing related exception"""
def __init__(self, *args, **kwargs):
- self.status = kwargs.pop('status', httplib.OK)
+ self.status = kwargs.pop('status', http_client.OK)
super(PublishException, self).__init__(*args, **kwargs)
class LogOut(PublishException):
@@ -52,7 +52,7 @@
class Redirect(PublishException):
"""raised to redirect the http request"""
- def __init__(self, location, status=httplib.SEE_OTHER):
+ def __init__(self, location, status=http_client.SEE_OTHER):
super(Redirect, self).__init__(status=status)
self.location = location
@@ -71,7 +71,7 @@
"""raised when a request can't be served because of a bad input"""
def __init__(self, *args, **kwargs):
- kwargs.setdefault('status', httplib.BAD_REQUEST)
+ kwargs.setdefault('status', http_client.BAD_REQUEST)
super(RequestError, self).__init__(*args, **kwargs)
@@ -79,14 +79,14 @@
"""raised when an edit request doesn't specify any eid to edit"""
def __init__(self, *args, **kwargs):
- kwargs.setdefault('status', httplib.BAD_REQUEST)
+ kwargs.setdefault('status', http_client.BAD_REQUEST)
super(NothingToEdit, self).__init__(*args, **kwargs)
class ProcessFormError(RequestError):
"""raised when posted data can't be processed by the corresponding field
"""
def __init__(self, *args, **kwargs):
- kwargs.setdefault('status', httplib.BAD_REQUEST)
+ kwargs.setdefault('status', http_client.BAD_REQUEST)
super(ProcessFormError, self).__init__(*args, **kwargs)
class NotFound(RequestError):
@@ -94,13 +94,13 @@
a 404 error should be returned"""
def __init__(self, *args, **kwargs):
- kwargs.setdefault('status', httplib.NOT_FOUND)
+ kwargs.setdefault('status', http_client.NOT_FOUND)
super(NotFound, self).__init__(*args, **kwargs)
class RemoteCallFailed(RequestError):
"""raised when a json remote call fails
"""
- def __init__(self, reason='', status=httplib.INTERNAL_SERVER_ERROR):
+ def __init__(self, reason='', status=http_client.INTERNAL_SERVER_ERROR):
super(RemoteCallFailed, self).__init__(reason, status=status)
self.reason = reason
--- a/web/action.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/action.py Wed Dec 16 11:23:48 2015 +0100
@@ -33,7 +33,7 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from cubicweb import target
from cubicweb.predicates import (partial_relation_possible, match_search_state,
@@ -91,7 +91,7 @@
"""base class for actions consisting to create a new object with an initial
relation set to an entity.
- Additionaly to EntityAction behaviour, this class is parametrized using
+ Additionally to EntityAction behaviour, this class is parametrized using
.rtype, .role and .target_etype attributes to check if the action apply and
if the logged user has access to it (see
:class:`~cubicweb.selectors.partial_relation_possible` selector
@@ -111,4 +111,3 @@
return self._cw.vreg["etypes"].etype_class(ttype).cw_create_url(self._cw,
__redirectpath=entity.rest_path(), __linkto=linkto,
__redirectvid=self._cw.form.get('__redirectvid', ''))
-
--- a/web/application.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/application.py Wed Dec 16 11:23:48 2015 +0100
@@ -25,7 +25,8 @@
from warnings import warn
import json
-import httplib
+from six import text_type, binary_type
+from six.moves import http_client
from logilab.common.deprecation import deprecated
@@ -68,8 +69,8 @@
def __init__(self, appli):
self.repo = appli.repo
self.vreg = appli.vreg
- self.session_manager = self.vreg['components'].select('sessionmanager',
- repo=self.repo)
+ self.session_manager = self.vreg['sessions'].select('sessionmanager',
+ repo=self.repo)
global SESSION_MANAGER
SESSION_MANAGER = self.session_manager
if self.vreg.config.mode != 'test':
@@ -80,8 +81,8 @@
def reset_session_manager(self):
data = self.session_manager.dump_data()
- self.session_manager = self.vreg['components'].select('sessionmanager',
- repo=self.repo)
+ self.session_manager = self.vreg['sessions'].select('sessionmanager',
+ repo=self.repo)
self.session_manager.restore_data(data)
global SESSION_MANAGER
SESSION_MANAGER = self.session_manager
@@ -256,7 +257,7 @@
# activate realm-based auth
realm = self.vreg.config['realm']
req.set_header('WWW-Authenticate', [('Basic', {'realm' : realm })], raw=False)
- content = ''
+ content = b''
try:
try:
session = self.get_session(req)
@@ -290,7 +291,7 @@
if self.vreg.config['auth-mode'] == 'cookie' and ex.url:
req.headers_out.setHeader('location', str(ex.url))
if ex.status is not None:
- req.status_out = httplib.SEE_OTHER
+ req.status_out = http_client.SEE_OTHER
# When the authentification is handled by http we must
# explicitly ask for authentification to flush current http
# authentification information
@@ -310,23 +311,24 @@
# the request does not use https, redirect to login form
https_url = self.vreg.config['https-url']
if https_url and req.base_url() != https_url:
- req.status_out = httplib.SEE_OTHER
+ req.status_out = http_client.SEE_OTHER
req.headers_out.setHeader('location', https_url + 'login')
else:
# We assume here that in http auth mode the user *May* provide
# Authentification Credential if asked kindly.
if self.vreg.config['auth-mode'] == 'http':
- req.status_out = httplib.UNAUTHORIZED
+ req.status_out = http_client.UNAUTHORIZED
# In the other case (coky auth) we assume that there is no way
# for the user to provide them...
# XXX But WHY ?
else:
- req.status_out = httplib.FORBIDDEN
+ req.status_out = http_client.FORBIDDEN
# If previous error handling already generated a custom content
# do not overwrite it. This is used by LogOut Except
# XXX ensure we don't actually serve content
if not content:
content = self.need_login_content(req)
+ assert isinstance(content, binary_type)
return content
@@ -368,7 +370,7 @@
except cors.CORSPreflight:
# Return directly an empty 200
req.status_out = 200
- result = ''
+ result = b''
except StatusResponse as ex:
warn('[3.16] StatusResponse is deprecated use req.status_out',
DeprecationWarning, stacklevel=2)
@@ -394,12 +396,12 @@
except Unauthorized as ex:
req.data['errmsg'] = req._('You\'re not authorized to access this page. '
'If you think you should, please contact the site administrator.')
- req.status_out = httplib.FORBIDDEN
+ req.status_out = http_client.FORBIDDEN
result = self.error_handler(req, ex, tb=False)
except Forbidden as ex:
req.data['errmsg'] = req._('This action is forbidden. '
'If you think it should be allowed, please contact the site administrator.')
- req.status_out = httplib.FORBIDDEN
+ req.status_out = http_client.FORBIDDEN
result = self.error_handler(req, ex, tb=False)
except (BadRQLQuery, RequestError) as ex:
result = self.error_handler(req, ex, tb=False)
@@ -413,7 +415,7 @@
raise
### Last defense line
except BaseException as ex:
- req.status_out = httplib.INTERNAL_SERVER_ERROR
+ req.status_out = http_client.INTERNAL_SERVER_ERROR
result = self.error_handler(req, ex, tb=True)
finally:
if req.cnx and not commited:
@@ -437,7 +439,7 @@
req.headers_out.setHeader('location', str(ex.location))
assert 300 <= ex.status < 400
req.status_out = ex.status
- return ''
+ return b''
def validation_error_handler(self, req, ex):
ex.translate(req._) # translate messages using ui language
@@ -453,9 +455,9 @@
# messages.
location = req.form['__errorurl'].rsplit('#', 1)[0]
req.headers_out.setHeader('location', str(location))
- req.status_out = httplib.SEE_OTHER
- return ''
- req.status_out = httplib.CONFLICT
+ req.status_out = http_client.SEE_OTHER
+ return b''
+ req.status_out = http_client.CONFLICT
return self.error_handler(req, ex, tb=False)
def error_handler(self, req, ex, tb=False):
@@ -491,14 +493,14 @@
def ajax_error_handler(self, req, ex):
req.set_header('content-type', 'application/json')
- status = httplib.INTERNAL_SERVER_ERROR
+ status = http_client.INTERNAL_SERVER_ERROR
if isinstance(ex, PublishException) and ex.status is not None:
status = ex.status
if req.status_out < 400:
# don't overwrite it if it's already set
req.status_out = status
- json_dumper = getattr(ex, 'dumps', lambda : json.dumps({'reason': unicode(ex)}))
- return json_dumper()
+ json_dumper = getattr(ex, 'dumps', lambda : json.dumps({'reason': text_type(ex)}))
+ return json_dumper().encode('utf-8')
# special case handling
--- a/web/box.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/box.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,9 @@
"""abstract box classes for CubicWeb web client"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six import add_metaclass
from logilab.mtconverter import xml_escape
from logilab.common.deprecation import class_deprecated, class_renamed
@@ -41,7 +43,7 @@
actions_by_cat.setdefault(action.category, []).append(
(action.title, action) )
for key, values in actions_by_cat.items():
- actions_by_cat[key] = [act for title, act in sorted(values)]
+ actions_by_cat[key] = [act for title, act in sorted(values, key=lambda x: x[0])]
if categories_in_order:
for cat in categories_in_order:
if cat in actions_by_cat:
@@ -53,6 +55,7 @@
# old box system, deprecated ###################################################
+@add_metaclass(class_deprecated)
class BoxTemplate(View):
"""base template for boxes, usually a (contextual) list of possible
actions. Various classes attributes may be used to control the box
@@ -66,7 +69,6 @@
box.render(self.w)
"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.10] *BoxTemplate classes are deprecated, use *CtxComponent instead (%(cls)s)'
__registry__ = 'ctxcomponents'
@@ -193,4 +195,3 @@
AjaxEditRelationBoxTemplate = class_renamed(
'AjaxEditRelationBoxTemplate', AjaxEditRelationCtxComponent,
'[3.10] AjaxEditRelationBoxTemplate has been renamed to AjaxEditRelationCtxComponent (%(cls)s)')
-
--- a/web/captcha.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/captcha.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,7 +22,9 @@
__docformat__ = "restructuredtext en"
from random import randint, choice
-from cStringIO import StringIO
+from io import BytesIO
+
+from six.moves import range
from PIL import Image, ImageFont, ImageDraw, ImageFilter
@@ -51,7 +53,7 @@
draw = ImageDraw.Draw(img)
# draw 100 random colored boxes on the background
x, y = img.size
- for num in xrange(100):
+ for num in range(100):
draw.rectangle((randint(0, x), randint(0, y),
randint(0, x), randint(0, y)),
fill=randint(0, 0xffffff))
@@ -67,7 +69,7 @@
"""
text = u''.join(choice('QWERTYUOPASDFGHJKLZXCVBNM') for i in range(size))
img = pil_captcha(text, fontfile, fontsize)
- out = StringIO()
+ out = BytesIO()
img.save(out, format)
out.seek(0)
return text, out
--- a/web/component.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/component.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,10 +20,12 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
+from six import PY3, add_metaclass, text_type
+
from logilab.common.deprecation import class_deprecated, class_renamed, deprecated
from logilab.mtconverter import xml_escape
@@ -215,6 +217,9 @@
def __unicode__(self):
return tags.a(self.label, href=self.href, **self.attrs)
+ if PY3:
+ __str__ = __unicode__
+
def render(self, w):
w(tags.a(self.label, href=self.href, **self.attrs))
@@ -425,7 +430,7 @@
@property
def domid(self):
- return domid(self.__regid__) + unicode(self.entity.eid)
+ return domid(self.__regid__) + text_type(self.entity.eid)
def lazy_view_holder(self, w, entity, oid, registry='views'):
"""add a holder and return a URL that may be used to replace this
@@ -498,7 +503,7 @@
args['subject'],
args['object'])
return u'[<a href="javascript: %s" class="action">%s</a>] %s' % (
- xml_escape(unicode(jscall)), label, etarget.view('incontext'))
+ xml_escape(text_type(jscall)), label, etarget.view('incontext'))
def related_boxitems(self, entity):
return [self.box_item(entity, etarget, 'delete_relation', u'-')
@@ -515,7 +520,7 @@
"""returns the list of unrelated entities, using the entity's
appropriate vocabulary function
"""
- skip = set(unicode(e.eid) for e in entity.related(self.rtype, role(self),
+ skip = set(text_type(e.eid) for e in entity.related(self.rtype, role(self),
entities=True))
skip.add(None)
skip.add(INTERNAL_FIELD_VALUE)
@@ -571,7 +576,7 @@
# to be defined in concrete classes
rtype = role = target_etype = None
- # class attributes below *must* be set in concrete classes (additionaly to
+ # class attributes below *must* be set in concrete classes (additionally to
# rtype / role [/ target_etype]. They should correspond to js_* methods on
# the json controller
@@ -633,7 +638,7 @@
if maydel:
if not js_css_added:
js_css_added = self.add_js_css()
- jscall = unicode(js.ajaxBoxRemoveLinkedEntity(
+ jscall = text_type(js.ajaxBoxRemoveLinkedEntity(
self.__regid__, entity.eid, rentity.eid,
self.fname_remove,
self.removed_msg and _(self.removed_msg)))
@@ -648,7 +653,7 @@
if mayadd:
multiple = self.rdef.role_cardinality(self.role) in '*+'
w(u'<table><tr><td>')
- jscall = unicode(js.ajaxBoxShowSelector(
+ jscall = text_type(js.ajaxBoxShowSelector(
self.__regid__, entity.eid, self.fname_vocabulary,
self.fname_validate, self.added_msg and _(self.added_msg),
_(stdmsgs.BUTTON_OK[0]), _(stdmsgs.BUTTON_CANCEL[0]),
@@ -677,6 +682,7 @@
# old contextual components, deprecated ########################################
+@add_metaclass(class_deprecated)
class EntityVComponent(Component):
"""abstract base class for additinal components displayed in content
headers and footer according to:
@@ -687,7 +693,6 @@
it should be configured using .accepts, .etype, .rtype, .target and
.context class attributes
"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.10] *VComponent classes are deprecated, use *CtxComponent instead (%(cls)s)'
__registry__ = 'ctxcomponents'
--- a/web/controller.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/controller.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,6 +19,8 @@
__docformat__ = "restructuredtext en"
+from six import PY2
+
from logilab.mtconverter import xml_escape
from logilab.common.registry import yes
from logilab.common.deprecation import deprecated
@@ -87,7 +89,7 @@
rql = req.form.get('rql')
if rql:
req.ensure_ro_rql(rql)
- if not isinstance(rql, unicode):
+ if PY2 and not isinstance(rql, unicode):
rql = unicode(rql, req.encoding)
pp = req.vreg['components'].select_or_none('magicsearch', req)
if pp is not None:
@@ -132,8 +134,6 @@
newparams['_cwmsgid'] = self._cw.set_redirect_message(msg)
if '__action_apply' in self._cw.form:
self._return_to_edition_view(newparams)
- if '__action_cancel' in self._cw.form:
- self._return_to_lastpage(newparams)
else:
self._return_to_original_view(newparams)
@@ -155,7 +155,7 @@
and '_cwmsgid' in newparams):
# are we here on creation or modification?
if any(eid == self._edited_entity.eid
- for eid in self._cw.data.get('eidmap', {}).itervalues()):
+ for eid in self._cw.data.get('eidmap', {}).values()):
msg = self._cw._('click here to see created entity')
else:
msg = self._cw._('click here to see edited entity')
@@ -201,11 +201,9 @@
raise Redirect(self._cw.build_url(path, **newparams))
- def _return_to_lastpage(self, newparams):
- """cancel-button case: in this case we are always expecting to go back
- where we came from, and this is not easy. Currently we suppose that
- __redirectpath is specifying that place if found, else we look in the
- request breadcrumbs for the last visited page.
+ def _redirect(self, newparams):
+ """Raise a redirect. We use __redirectpath if it specified, else we
+ return to the home page.
"""
if '__redirectpath' in self._cw.form:
# if redirect path was explicitly specified in the form, use it
@@ -213,7 +211,7 @@
url = self._cw.build_url(path)
url = append_url_params(url, self._cw.form.get('__redirectparams'))
else:
- url = self._cw.last_visited_page()
+ url = self._cw.base_url()
# The newparams must update the params in all cases
url = self._cw.rebuild_url(url, **newparams)
raise Redirect(url)
@@ -221,4 +219,3 @@
from cubicweb import set_log_methods
set_log_methods(Controller, LOGGER)
-
--- a/web/cors.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/cors.py Wed Dec 16 11:23:48 2015 +0100
@@ -14,7 +14,7 @@
"""
-import urlparse
+from six.moves.urllib.parse import urlsplit
from cubicweb.web import LOGGER
info = LOGGER.info
@@ -37,7 +37,7 @@
In case of non-compliance, no CORS-related header is set.
"""
- base_url = urlparse.urlsplit(req.base_url())
+ base_url = urlsplit(req.base_url())
expected_host = '://'.join((base_url.scheme, base_url.netloc))
if not req.get_header('Origin') or req.get_header('Origin') == expected_host:
# not a CORS request, nothing to do
@@ -50,7 +50,7 @@
process_preflight(req, config)
else: # Simple CORS or actual request
process_simple(req, config)
- except CORSFailed, exc:
+ except CORSFailed as exc:
info('Cross origin resource sharing failed: %s' % exc)
except CORSPreflight:
info('Cross origin resource sharing: valid Preflight request %s')
@@ -101,7 +101,7 @@
if '*' not in allowed_origins and origin not in allowed_origins:
raise CORSFailed('Origin is not allowed')
# bit of sanity check; see "6.3 Security"
- myhost = urlparse.urlsplit(req.base_url()).netloc
+ myhost = urlsplit(req.base_url()).netloc
host = req.get_header('Host')
if host != myhost:
info('cross origin resource sharing detected possible '
@@ -111,4 +111,3 @@
# include "Vary: Origin" header (see 6.4)
req.headers_out.addHeader('Vary', 'Origin')
return origin
-
--- a/web/data/cubicweb.edition.js Tue Dec 15 19:05:41 2015 +0100
+++ b/web/data/cubicweb.edition.js Wed Dec 16 11:23:48 2015 +0100
@@ -537,6 +537,24 @@
}
/**
+ * Cancel the operations done on the given form.
+ *
+ */
+$(function () {
+ $(document).on('click', '.cwjs-edition-cancel', function (evt) {
+ var $mynode = $(evt.currentTarget),
+ $form = $mynode.closest('form'),
+ $error = $form.find(':input[name="__errorurl"]'),
+ errorurl = $error.attr('value'),
+ args = ajaxFuncArgs('cancel_edition', null, errorurl);
+ loadRemote(AJAX_BASE_URL, args, 'POST', true);
+ history.back();
+ return false;
+ });
+});
+
+
+/**
* .. function:: validateForm(formid, action, onsuccess, onfailure)
*
* called on traditionnal form submission : the idea is to try
--- a/web/facet.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/facet.py Wed Dec 16 11:23:48 2015 +0100
@@ -50,13 +50,15 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from functools import reduce
from warnings import warn
from copy import deepcopy
from datetime import datetime, timedelta
+from six import text_type, string_types
+
from logilab.mtconverter import xml_escape
from logilab.common.graph import has_path
from logilab.common.decorators import cached, cachedproperty
@@ -80,7 +82,7 @@
ptypes = facet.cw_rset.column_types(0)
if len(ptypes) == 1:
return display_name(facet._cw, facet.rtype, form=facet.role,
- context=iter(ptypes).next())
+ context=next(iter(ptypes)))
return display_name(facet._cw, facet.rtype, form=facet.role)
def get_facet(req, facetid, select, filtered_variable):
@@ -133,7 +135,7 @@
or the first variable selected in column 0
"""
if mainvar is None:
- vref = select.selection[0].iget_nodes(nodes.VariableRef).next()
+ vref = next(select.selection[0].iget_nodes(nodes.VariableRef))
return vref.variable
return select.defined_vars[mainvar]
@@ -156,7 +158,7 @@
for term in select.selection[:]:
select.remove_selected(term)
# remove unbound variables which only have some type restriction
- for dvar in list(select.defined_vars.itervalues()):
+ for dvar in list(select.defined_vars.values()):
if not (dvar is filtered_variable or dvar.stinfo['relations']):
select.undefine_variable(dvar)
# global tree config: DISTINCT, LIMIT, OFFSET
@@ -305,7 +307,7 @@
# optional relation
return ovar
if all(rdef.cardinality[cardidx] in '1+'
- for rdef in rschema.rdefs.itervalues()):
+ for rdef in rschema.rdefs.values()):
# mandatory relation without any restriction on the other variable
for orel in ovar.stinfo['relations']:
if rel is orel:
@@ -670,7 +672,7 @@
insert_attr_select_relation(
select, self.filtered_variable, self.rtype, self.role,
self.target_attr, select_target_entity=False)
- values = [unicode(x) for x, in self.rqlexec(select.as_string())]
+ values = [text_type(x) for x, in self.rqlexec(select.as_string())]
except Exception:
self.exception('while computing values for %s', self)
return []
@@ -719,14 +721,14 @@
def rset_vocabulary(self, rset):
if self.i18nable:
- _ = self._cw._
+ tr = self._cw._
else:
- _ = unicode
+ tr = text_type
if self.rql_sort:
- values = [(_(label), eid) for eid, label in rset]
+ values = [(tr(label), eid) for eid, label in rset]
else:
if self.label_vid is None:
- values = [(_(label), eid) for eid, label in rset]
+ values = [(tr(label), eid) for eid, label in rset]
else:
values = [(entity.view(self.label_vid), entity.eid)
for entity in rset.entities()]
@@ -754,7 +756,7 @@
# XXX handle rel is None case in RQLPathFacet?
if self.restr_attr != 'eid':
self.select.set_distinct(True)
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
# only one value selected
if value:
self.select.add_constant_restriction(
@@ -808,7 +810,7 @@
rschema = self._cw.vreg.schema.rschema(self.rtype)
# XXX when called via ajax, no rset to compute possible types
possibletypes = self.cw_rset and self.cw_rset.column_types(0)
- for rdef in rschema.rdefs.itervalues():
+ for rdef in rschema.rdefs.values():
if possibletypes is not None:
if self.role == 'subject':
if rdef.subject not in possibletypes:
@@ -829,13 +831,13 @@
if self._cw.vreg.schema.rschema(self.rtype).final:
return False
if self.role == 'object':
- subj = utils.rqlvar_maker(defined=self.select.defined_vars,
- aliases=self.select.aliases).next()
+ subj = next(utils.rqlvar_maker(defined=self.select.defined_vars,
+ aliases=self.select.aliases))
obj = self.filtered_variable.name
else:
subj = self.filtered_variable.name
- obj = utils.rqlvar_maker(defined=self.select.defined_vars,
- aliases=self.select.aliases).next()
+ obj = next(utils.rqlvar_maker(defined=self.select.defined_vars,
+ aliases=self.select.aliases))
restrictions = []
if self.select.where:
restrictions.append(self.select.where.as_string())
@@ -916,15 +918,13 @@
def rset_vocabulary(self, rset):
if self.i18nable:
- _ = self._cw._
+ tr = self._cw._
else:
- _ = unicode
+ tr = text_type
if self.rql_sort:
- return [(_(value), value) for value, in rset]
- values = [(_(value), value) for value, in rset]
- if self.sortasc:
- return sorted(values)
- return reversed(sorted(values))
+ return [(tr(value), value) for value, in rset]
+ values = [(tr(value), value) for value, in rset]
+ return sorted(values, reverse=not self.sortasc)
class AttributeFacet(RelationAttributeFacet):
@@ -1073,7 +1073,7 @@
assert self.path and isinstance(self.path, (list, tuple)), \
'path should be a list of 3-uples, not %s' % self.path
for part in self.path:
- if isinstance(part, basestring):
+ if isinstance(part, string_types):
part = part.split()
assert len(part) == 3, \
'path should be a list of 3-uples, not %s' % part
@@ -1126,7 +1126,7 @@
cleanup_select(select, self.filtered_variable)
varmap, restrvar = self.add_path_to_select(skiplabel=True)
select.append_selected(nodes.VariableRef(restrvar))
- values = [unicode(x) for x, in self.rqlexec(select.as_string())]
+ values = [text_type(x) for x, in self.rqlexec(select.as_string())]
except Exception:
self.exception('while computing values for %s', self)
return []
@@ -1149,7 +1149,7 @@
varmap = {'X': self.filtered_variable}
actual_filter_variable = None
for part in self.path:
- if isinstance(part, basestring):
+ if isinstance(part, string_types):
part = part.split()
subject, rtype, object = part
if skiplabel and object == self.label_variable:
@@ -1165,7 +1165,7 @@
if len(attrtypes) > 1:
raise Exception('ambigous attribute %s, specify attrtype on %s'
% (rtype, self.__class__))
- self.restr_attr_type = iter(attrtypes).next()
+ self.restr_attr_type = next(iter(attrtypes))
if skipattrfilter:
actual_filter_variable = subject
continue
@@ -1253,7 +1253,7 @@
rset = self._range_rset()
if rset:
minv, maxv = rset[0]
- return [(unicode(minv), minv), (unicode(maxv), maxv)]
+ return [(text_type(minv), minv), (text_type(maxv), maxv)]
return []
def possible_values(self):
@@ -1272,7 +1272,7 @@
def formatvalue(self, value):
"""format `value` before in order to insert it in the RQL query"""
- return unicode(value)
+ return text_type(value)
def infvalue(self, min=False):
if min:
@@ -1373,7 +1373,7 @@
# *list* (see rqlexec implementation)
if rset:
minv, maxv = rset[0]
- return [(unicode(minv), minv), (unicode(maxv), maxv)]
+ return [(text_type(minv), minv), (text_type(maxv), maxv)]
return []
@@ -1392,7 +1392,7 @@
skiplabel=True, skipattrfilter=True)
restrel = None
for part in self.path:
- if isinstance(part, basestring):
+ if isinstance(part, string_types):
part = part.split()
subject, rtype, object = part
if object == self.filter_variable:
@@ -1516,7 +1516,7 @@
if not val or val & mask])
def possible_values(self):
- return [unicode(val) for label, val in self.vocabulary()]
+ return [text_type(val) for label, val in self.vocabulary()]
## html widets ################################################################
@@ -1595,7 +1595,7 @@
if selected:
cssclass += ' facetValueSelected'
w(u'<div class="%s" cubicweb:value="%s">\n'
- % (cssclass, xml_escape(unicode(value))))
+ % (cssclass, xml_escape(text_type(value))))
# If it is overflowed one must add padding to compensate for the vertical
# scrollbar; given current css values, 4 blanks work perfectly ...
padding = u' ' * self.scrollbar_padding_factor if overflow else u''
@@ -1754,7 +1754,7 @@
imgsrc = self._cw.data_url(self.unselected_img)
imgalt = self._cw._('not selected')
w(u'<div class="%s" cubicweb:value="%s">\n'
- % (cssclass, xml_escape(unicode(self.value))))
+ % (cssclass, xml_escape(text_type(self.value))))
w(u'<div>')
w(u'<img src="%s" alt="%s" cubicweb:unselimg="true" /> ' % (imgsrc, imgalt))
w(u'<label class="facetTitle" cubicweb:facetName="%s">%s</label>'
--- a/web/form.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/form.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,6 +20,8 @@
from warnings import warn
+from six import add_metaclass
+
from logilab.common.decorators import iclassmethod
from logilab.common.deprecation import deprecated
@@ -74,8 +76,8 @@
found
"""
+@add_metaclass(metafieldsform)
class Form(AppObject):
- __metaclass__ = metafieldsform
__registry__ = 'forms'
parent_form = None
@@ -120,7 +122,7 @@
extrakw = {}
# search for navigation parameters and customization of existing
# attributes; remaining stuff goes in extrakwargs
- for key, val in kwargs.iteritems():
+ for key, val in kwargs.items():
if key in controller.NAV_FORM_PARAMETERS:
hiddens.append( (key, val) )
elif key == 'redirect_path':
@@ -280,4 +282,3 @@
def remaining_errors(self):
return sorted(self.form_valerror.errors.items())
-
--- a/web/formfields.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/formfields.py Wed Dec 16 11:23:48 2015 +0100
@@ -66,6 +66,8 @@
from warnings import warn
from datetime import datetime, timedelta
+from six import PY2, text_type, string_types
+
from logilab.mtconverter import xml_escape
from logilab.common import nullobject
from logilab.common.date import ustrftime
@@ -159,7 +161,7 @@
:attr:`order`
key used by automatic forms to sort fields
:attr:`ignore_req_params`
- when true, this field won't consider value potentialy specified using
+ when true, this field won't consider value potentially specified using
request's form parameters (eg you won't be able to specify a value using for
instance url like http://mywebsite.com/form?field=value)
@@ -231,11 +233,14 @@
def __unicode__(self):
return self.as_string(False)
- def __str__(self):
- return self.as_string(False).encode('UTF8')
+ if PY2:
+ def __str__(self):
+ return self.as_string(False).encode('UTF8')
+ else:
+ __str__ = __unicode__
def __repr__(self):
- return self.as_string(True).encode('UTF8')
+ return self.as_string(True)
def init_widget(self, widget):
if widget is not None:
@@ -279,7 +284,7 @@
return u''
if value is True:
return u'1'
- return unicode(value)
+ return text_type(value)
def get_widget(self, form):
"""return the widget instance associated to this field"""
@@ -381,7 +386,7 @@
assert self.choices is not None
if callable(self.choices):
# pylint: disable=E1102
- if getattr(self.choices, 'im_self', None) is self:
+ if getattr(self.choices, '__self__', None) is self:
vocab = self.choices(form=form, **kwargs)
else:
vocab = self.choices(form=form, field=self, **kwargs)
@@ -508,7 +513,7 @@
class StringField(Field):
"""Use this field to edit unicode string (`String` yams type). This field
- additionaly support a `max_length` attribute that specify a maximum size for
+ additionally support a `max_length` attribute that specify a maximum size for
the string (`None` meaning no limit).
Unless explicitly specified, the widget for this field will be:
@@ -780,7 +785,7 @@
If the stream format is one of text/plain, text/html, text/rest,
text/markdown
- then a :class:`~cubicweb.web.formwidgets.TextArea` will be additionaly
+ then a :class:`~cubicweb.web.formwidgets.TextArea` will be additionally
displayed, allowing to directly the file's content when desired, instead
of choosing a file from user's file system.
"""
@@ -794,7 +799,7 @@
if data:
encoding = self.encoding(form)
try:
- form.formvalues[(self, form)] = unicode(data.getvalue(), encoding)
+ form.formvalues[(self, form)] = data.getvalue().decode(encoding)
except UnicodeError:
pass
else:
@@ -815,7 +820,7 @@
def _process_form_value(self, form):
value = form._cw.form.get(self.input_name(form))
- if isinstance(value, unicode):
+ if isinstance(value, text_type):
# file modified using a text widget
return Binary(value.encode(self.encoding(form)))
return super(EditableFileField, self)._process_form_value(form)
@@ -823,7 +828,7 @@
class BigIntField(Field):
"""Use this field to edit big integers (`BigInt` yams type). This field
- additionaly support `min` and `max` attributes that specify a minimum and/or
+ additionally support `min` and `max` attributes that specify a minimum and/or
maximum value for the integer (`None` meaning no boundary).
Unless explicitly specified, the widget for this field will be a
@@ -842,7 +847,7 @@
self.widget.attrs.setdefault('size', self.default_text_input_size)
def _ensure_correctly_typed(self, form, value):
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
value = value.strip()
if not value:
return None
@@ -907,7 +912,7 @@
class FloatField(IntField):
- """Use this field to edit floats (`Float` yams type). This field additionaly
+ """Use this field to edit floats (`Float` yams type). This field additionally
support `min` and `max` attributes as the
:class:`~cubicweb.web.formfields.IntField`.
@@ -924,7 +929,7 @@
return self.format_single_value(req, 1.234)
def _ensure_correctly_typed(self, form, value):
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
value = value.strip()
if not value:
return None
@@ -946,7 +951,7 @@
def format_single_value(self, req, value):
if value:
value = format_time(value.days * 24 * 3600 + value.seconds)
- return unicode(value)
+ return text_type(value)
return u''
def example_format(self, req):
@@ -956,7 +961,7 @@
return u'20s, 10min, 24h, 4d'
def _ensure_correctly_typed(self, form, value):
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
value = value.strip()
if not value:
return None
@@ -986,14 +991,14 @@
return self.format_single_value(req, datetime.now())
def _ensure_correctly_typed(self, form, value):
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
value = value.strip()
if not value:
return None
try:
value = form._cw.parse_datetime(value, self.etype)
except ValueError as ex:
- raise ProcessFormError(unicode(ex))
+ raise ProcessFormError(text_type(ex))
return value
@@ -1083,7 +1088,7 @@
linkedto = form.linked_to.get((self.name, self.role))
if linkedto:
buildent = form._cw.entity_from_eid
- return [(buildent(eid).view('combobox'), unicode(eid))
+ return [(buildent(eid).view('combobox'), text_type(eid))
for eid in linkedto]
return []
@@ -1095,7 +1100,7 @@
# vocabulary doesn't include current values, add them
if form.edited_entity.has_eid():
rset = form.edited_entity.related(self.name, self.role)
- vocab += [(e.view('combobox'), unicode(e.eid))
+ vocab += [(e.view('combobox'), text_type(e.eid))
for e in rset.entities()]
return vocab
@@ -1129,11 +1134,11 @@
if entity.eid in done:
continue
done.add(entity.eid)
- res.append((entity.view('combobox'), unicode(entity.eid)))
+ res.append((entity.view('combobox'), text_type(entity.eid)))
return res
def format_single_value(self, req, value):
- return unicode(value)
+ return text_type(value)
def process_form_value(self, form):
"""process posted form and return correctly typed value"""
--- a/web/formwidgets.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/formwidgets.py Wed Dec 16 11:23:48 2015 +0100
@@ -99,6 +99,8 @@
from datetime import date
from warnings import warn
+from six import text_type, string_types
+
from logilab.mtconverter import xml_escape
from logilab.common.deprecation import deprecated
from logilab.common.date import todatetime
@@ -282,7 +284,7 @@
"""
posted = form._cw.form
val = posted.get(field.input_name(form, self.suffix))
- if isinstance(val, basestring):
+ if isinstance(val, string_types):
val = val.strip()
return val
@@ -416,7 +418,7 @@
lines = value.splitlines()
linecount = len(lines)
for line in lines:
- linecount += len(line) / self._columns
+ linecount += len(line) // self._columns
attrs.setdefault('cols', self._columns)
attrs.setdefault('rows', min(self._maxrows, linecount + self._minrows))
return tags.textarea(value, name=field.input_name(form, self.suffix),
@@ -474,7 +476,7 @@
options.append(u'</optgroup>')
if not 'size' in attrs:
if self._multiple:
- size = unicode(min(self.default_size, len(vocab) or 1))
+ size = text_type(min(self.default_size, len(vocab) or 1))
else:
size = u'1'
attrs['size'] = size
@@ -616,7 +618,7 @@
iattrs['checked'] = u'checked'
tag = tags.input(name=field.input_name(form, self.suffix),
type=self.type, value=value, **iattrs)
- options.append(u'%s %s' % (tag, label))
+ options.append(u'<label>%s %s</label>' % (tag, xml_escape(label)))
return sep.join(options)
@@ -706,7 +708,7 @@
else:
value = self.value
attrs = self.attributes(form, field)
- attrs.setdefault('size', unicode(self.default_size))
+ attrs.setdefault('size', text_type(self.default_size))
return tags.input(name=field.input_name(form, self.suffix),
value=value, type='text', **attrs)
@@ -779,13 +781,13 @@
try:
date = todatetime(req.parse_datetime(datestr, 'Date'))
except ValueError as exc:
- raise ProcessFormError(unicode(exc))
+ raise ProcessFormError(text_type(exc))
if timestr is None:
return date
try:
time = req.parse_datetime(timestr, 'Time')
except ValueError as exc:
- raise ProcessFormError(unicode(exc))
+ raise ProcessFormError(text_type(exc))
return date.replace(hour=time.hour, minute=time.minute, second=time.second)
@@ -993,12 +995,12 @@
req = form._cw
values = {}
path = req.form.get(field.input_name(form, 'path'))
- if isinstance(path, basestring):
+ if isinstance(path, string_types):
path = path.strip()
if path is None:
path = u''
fqs = req.form.get(field.input_name(form, 'fqs'))
- if isinstance(fqs, basestring):
+ if isinstance(fqs, string_types):
fqs = fqs.strip() or None
if fqs:
for i, line in enumerate(fqs.split('\n')):
@@ -1009,7 +1011,7 @@
except ValueError:
raise ProcessFormError(req._("wrong query parameter line %s") % (i+1))
# value will be url quoted by build_url_params
- values.setdefault(key.encode(req.encoding), []).append(val)
+ values.setdefault(key, []).append(val)
if not values:
return path
return u'%s?%s' % (path, req.build_url_params(**values))
@@ -1094,4 +1096,3 @@
'<img src="%(imgsrc)s" alt="%(label)s"/>%(label)s</a>' % {
'label': label, 'imgsrc': imgsrc,
'domid': self.domid, 'href': self.href}
-
--- a/web/htmlwidgets.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/htmlwidgets.py Wed Dec 16 11:23:48 2015 +0100
@@ -24,6 +24,9 @@
import random
from math import floor
+from six import add_metaclass
+from six.moves import range
+
from logilab.mtconverter import xml_escape
from logilab.common.deprecation import class_deprecated
@@ -115,9 +118,9 @@
self.w(u'</div>')
+@add_metaclass(class_deprecated)
class SideBoxWidget(BoxWidget):
"""default CubicWeb's sidebox widget"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.10] class %(cls)s is deprecated'
title_class = u'sideBoxTitle'
@@ -207,9 +210,9 @@
self.w(u'</ul></div></div>')
+@add_metaclass(class_deprecated)
class BoxField(HTMLWidget):
"""couples label / value meant to be displayed in a box"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.10] class %(cls)s is deprecated'
def __init__(self, label, value):
self.label = label
@@ -220,18 +223,19 @@
u'<span class="value">%s</span></div></li>'
% (self.label, self.value))
+
+@add_metaclass(class_deprecated)
class BoxSeparator(HTMLWidget):
"""a menu separator"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.10] class %(cls)s is deprecated'
def _render(self):
self.w(u'</ul><hr class="boxSeparator"/><ul>')
+@add_metaclass(class_deprecated)
class BoxLink(HTMLWidget):
"""a link in a box"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.10] class %(cls)s is deprecated'
def __init__(self, href, label, _class='', title='', ident='', escape=False):
self.href = href
@@ -252,9 +256,9 @@
self.w(u'<li class="%s">%s</li>\n' % (self._class, link))
+@add_metaclass(class_deprecated)
class BoxHtml(HTMLWidget):
"""a form in a box"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.10] class %(cls)s is deprecated'
def __init__(self, rawhtml):
self.rawhtml = rawhtml
@@ -339,17 +343,17 @@
self.w(u'<thead>')
self.w(u'<tr class="header">')
for column in self.columns:
- attrs = ('%s="%s"' % (name, value) for name, value in column.cell_attrs.iteritems())
+ attrs = ('%s="%s"' % (name, value) for name, value in column.cell_attrs.items())
self.w(u'<th %s>%s</th>' % (' '.join(attrs), column.name or u''))
self.w(u'</tr>')
self.w(u'</thead><tbody>')
- for rowindex in xrange(len(self.model.get_rows())):
+ for rowindex in range(len(self.model.get_rows())):
klass = (rowindex%2==1) and 'odd' or 'even'
self.w(u'<tr class="%s" %s>' % (klass, self.highlight))
for column, sortvalue in self.itercols(rowindex):
attrs = dict(column.cell_attrs)
attrs["cubicweb:sortvalue"] = sortvalue
- attrs = ('%s="%s"' % (name, value) for name, value in attrs.iteritems())
+ attrs = ('%s="%s"' % (name, value) for name, value in attrs.items())
self.w(u'<td %s>' % (' '.join(attrs)))
for cellvid, colindex in column.cellrenderers:
self.model.render_cell(cellvid, rowindex, colindex, w=self.w)
@@ -361,5 +365,3 @@
def itercols(self, rowindex):
for column in self.columns:
yield column, self.model.sortvalue(rowindex, column.rset_sortcol)
-
-
--- a/web/http_headers.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/http_headers.py Wed Dec 16 11:23:48 2015 +0100
@@ -2,11 +2,14 @@
# http://twistedmatrix.com/trac/wiki/TwistedWeb2
-import types, time
+import time
from calendar import timegm
import base64
import re
-import urlparse
+
+from six import string_types
+from six.moves.urllib.parse import urlparse
+
def dashCapitalize(s):
''' Capitalize a string, making sure to treat - as a word seperator '''
@@ -295,9 +298,9 @@
cur = cur+1
if qpair:
- raise ValueError, "Missing character after '\\'"
+ raise ValueError("Missing character after '\\'")
if quoted:
- raise ValueError, "Missing end quote"
+ raise ValueError("Missing end quote")
if start != cur:
if foldCase:
@@ -347,7 +350,7 @@
##### parser utilities:
def checkSingleToken(tokens):
if len(tokens) != 1:
- raise ValueError, "Expected single token, not %s." % (tokens,)
+ raise ValueError("Expected single token, not %s." % (tokens,))
return tokens[0]
def parseKeyValue(val):
@@ -355,11 +358,11 @@
return val[0], None
elif len(val) == 3 and val[1] == Token('='):
return val[0], val[2]
- raise ValueError, "Expected key or key=value, but got %s." % (val,)
+ raise ValueError("Expected key or key=value, but got %s." % (val,))
def parseArgs(field):
args = split(field, Token(';'))
- val = args.next()
+ val = next(args)
args = [parseKeyValue(arg) for arg in args]
return val, args
@@ -380,7 +383,7 @@
def unique(seq):
'''if seq is not a string, check it's a sequence of one element and return it'''
- if isinstance(seq, basestring):
+ if isinstance(seq, string_types):
return seq
if len(seq) != 1:
raise ValueError('single value required, not %s' % seq)
@@ -398,7 +401,7 @@
"""Ensure origin is a valid URL-base stuff, or null"""
if origin == 'null':
return origin
- p = urlparse.urlparse(origin)
+ p = urlparse(origin)
if p.params or p.query or p.username or p.path not in ('', '/'):
raise ValueError('Incorrect Accept-Control-Allow-Origin value %s' % origin)
if p.scheme not in ('http', 'https'):
@@ -452,14 +455,15 @@
"""
if (value in (True, 1) or
- isinstance(value, basestring) and value.lower() == 'true'):
+ isinstance(value, string_types) and value.lower() == 'true'):
return 'true'
if (value in (False, 0) or
- isinstance(value, basestring) and value.lower() == 'false'):
+ isinstance(value, string_types) and value.lower() == 'false'):
return 'false'
raise ValueError("Invalid true/false header value: %s" % value)
class MimeType(object):
+ @classmethod
def fromString(klass, mimeTypeString):
"""Generate a MimeType object from the given string.
@@ -469,8 +473,6 @@
"""
return DefaultHTTPHandler.parse('content-type', [mimeTypeString])
- fromString = classmethod(fromString)
-
def __init__(self, mediaType, mediaSubtype, params={}, **kwargs):
"""
@type mediaType: C{str}
@@ -499,14 +501,14 @@
return "MimeType(%r, %r, %r)" % (self.mediaType, self.mediaSubtype, self.params)
def __hash__(self):
- return hash(self.mediaType)^hash(self.mediaSubtype)^hash(tuple(self.params.iteritems()))
+ return hash(self.mediaType)^hash(self.mediaSubtype)^hash(tuple(self.params.items()))
##### Specific header parsers.
def parseAccept(field):
type, args = parseArgs(field)
if len(type) != 3 or type[1] != Token('/'):
- raise ValueError, "MIME Type "+str(type)+" invalid."
+ raise ValueError("MIME Type "+str(type)+" invalid.")
# okay, this spec is screwy. A 'q' parameter is used as the separator
# between MIME parameters and (as yet undefined) additional HTTP
@@ -569,7 +571,7 @@
type, args = parseArgs(header)
if len(type) != 3 or type[1] != Token('/'):
- raise ValueError, "MIME Type "+str(type)+" invalid."
+ raise ValueError("MIME Type "+str(type)+" invalid.")
args = [(kv[0].lower(), kv[1]) for kv in args]
@@ -730,7 +732,7 @@
out ="%s/%s"%(mimeType.mediaType, mimeType.mediaSubtype)
if mimeType.params:
- out+=';'+generateKeyValues(mimeType.params.iteritems())
+ out+=';'+generateKeyValues(mimeType.params.items())
if q != 1.0:
out+=(';q=%.3f' % (q,)).rstrip('0').rstrip('.')
@@ -766,7 +768,8 @@
v = [field.strip().lower() for field in v.split(',')]
return k, v
-def generateCacheControl((k, v)):
+def generateCacheControl(args):
+ k, v = args
if v is None:
return str(k)
else:
@@ -833,7 +836,7 @@
def generateContentType(mimeType):
out = "%s/%s" % (mimeType.mediaType, mimeType.mediaSubtype)
if mimeType.params:
- out += ';' + generateKeyValues(mimeType.params.iteritems())
+ out += ';' + generateKeyValues(mimeType.params.items())
return out
def generateIfRange(dateOrETag):
@@ -854,7 +857,7 @@
try:
l = []
- for k, v in dict(challenge).iteritems():
+ for k, v in dict(challenge).items():
l.append("%s=%s" % (k, quoteString(v)))
_generated.append("%s %s" % (scheme, ", ".join(l)))
@@ -864,7 +867,7 @@
return _generated
def generateAuthorization(seq):
- return [' '.join(seq)]
+ return [' '.join(str(v) for v in seq)]
####
@@ -1326,10 +1329,10 @@
self._headers = {}
self.handler = handler
if headers is not None:
- for key, value in headers.iteritems():
+ for key, value in headers.items():
self.setHeader(key, value)
if rawHeaders is not None:
- for key, value in rawHeaders.iteritems():
+ for key, value in rawHeaders.items():
self.setRawHeaders(key, value)
def _setRawHeaders(self, headers):
@@ -1458,7 +1461,7 @@
"""Return an iterator of key, value pairs of all headers
contained in this object, as strings. The keys are capitalized
in canonical capitalization."""
- for k, v in self._raw_headers.iteritems():
+ for k, v in self._raw_headers.items():
if v is _RecalcNeeded:
v = self._toRaw(k)
yield self.canonicalNameCaps(k), v
@@ -1480,7 +1483,7 @@
is strictly an error, but we're nice.).
"""
-iteritems = lambda x: x.iteritems()
+iteritems = lambda x: x.items()
parser_general_headers = {
--- a/web/httpcache.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/httpcache.py Wed Dec 16 11:23:48 2015 +0100
@@ -31,6 +31,7 @@
def set_headers(self):
self.req.set_header('Cache-control', 'no-cache')
+ self.req.set_header('Expires', 'Sat, 01 Jan 2000 00:00:00 GMT')
class MaxAgeHTTPCacheManager(NoHTTPCacheManager):
@@ -68,7 +69,7 @@
try:
req.set_header('Etag', '"%s"' % self.etag())
except NoEtag:
- self.req.set_header('Cache-control', 'no-cache')
+ super(EtagHTTPCacheManager, self).set_headers()
return
req.set_header('Cache-control',
'must-revalidate,max-age=%s' % self.max_age())
@@ -178,4 +179,3 @@
('if-none-match', if_none_match),
#('if-modified-since', if_modified_since),
]
-
--- a/web/propertysheet.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/propertysheet.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,6 +22,8 @@
import re
import os
import os.path as osp
+import tempfile
+
TYPE_CHECKS = [('STYLESHEETS', list), ('JAVASCRIPTS', list),
('STYLESHEETS_IE', list), ('STYLESHEETS_PRINT', list),
@@ -51,13 +53,13 @@
self.clear()
self._ordered_propfiles = []
self._propfile_mtime = {}
- self._sourcefile_mtime = {}
- self._cache = {}
def load(self, fpath):
scriptglobals = self.context.copy()
scriptglobals['__file__'] = fpath
- execfile(fpath, scriptglobals, self)
+ with open(fpath, 'rb') as fobj:
+ code = compile(fobj.read(), fpath, 'exec')
+ exec(code, scriptglobals, self)
for name, type in TYPE_CHECKS:
if name in self:
if not isinstance(self[name], type):
@@ -67,10 +69,7 @@
self._ordered_propfiles.append(fpath)
def need_reload(self):
- for rid, (adirectory, rdirectory, mtime) in self._cache.items():
- if os.stat(osp.join(rdirectory, rid)).st_mtime > mtime:
- del self._cache[rid]
- for fpath, mtime in self._propfile_mtime.iteritems():
+ for fpath, mtime in self._propfile_mtime.items():
if os.stat(fpath).st_mtime > mtime:
return True
return False
@@ -86,31 +85,29 @@
self.reload()
def process_resource(self, rdirectory, rid):
+ cachefile = osp.join(self._cache_directory, rid)
+ self.debug('processing %s/%s into %s',
+ rdirectory, rid, cachefile)
+ rcachedir = osp.dirname(cachefile)
+ if not osp.exists(rcachedir):
+ os.makedirs(rcachedir)
+ sourcefile = osp.join(rdirectory, rid)
+ with open(sourcefile) as f:
+ content = f.read()
+ # XXX replace % not followed by a paren by %% to avoid having to do
+ # this in the source css file ?
try:
- return self._cache[rid][0]
- except KeyError:
- cachefile = osp.join(self._cache_directory, rid)
- self.debug('caching processed %s/%s into %s',
- rdirectory, rid, cachefile)
- rcachedir = osp.dirname(cachefile)
- if not osp.exists(rcachedir):
- os.makedirs(rcachedir)
- sourcefile = osp.join(rdirectory, rid)
- content = file(sourcefile).read()
- # XXX replace % not followed by a paren by %% to avoid having to do
- # this in the source css file ?
- try:
- content = self.compile(content)
- except ValueError as ex:
- self.error("can't process %s/%s: %s", rdirectory, rid, ex)
- adirectory = rdirectory
- else:
- stream = file(cachefile, 'w')
+ content = self.compile(content)
+ except ValueError as ex:
+ self.error("can't process %s/%s: %s", rdirectory, rid, ex)
+ adirectory = rdirectory
+ else:
+ tmpfd, tmpfile = tempfile.mkstemp(dir=rcachedir, prefix=osp.basename(cachefile))
+ with os.fdopen(tmpfd, 'w') as stream:
stream.write(content)
- stream.close()
- adirectory = self._cache_directory
- self._cache[rid] = (adirectory, rdirectory, os.stat(sourcefile).st_mtime)
- return adirectory
+ os.rename(tmpfile, cachefile)
+ adirectory = self._cache_directory
+ return adirectory
def compile(self, content):
return self._percent_rgx.sub('%%', content) % self
--- a/web/request.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/request.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,15 +22,16 @@
import time
import random
import base64
-import urllib
-from StringIO import StringIO
from hashlib import sha1 # pylint: disable=E0611
-from Cookie import SimpleCookie
from calendar import timegm
from datetime import date, datetime
-from urlparse import urlsplit
-import httplib
from warnings import warn
+from io import BytesIO
+
+from six import PY2, binary_type, text_type, string_types
+from six.moves import http_client
+from six.moves.urllib.parse import urlsplit, quote as urlquote
+from six.moves.http_cookies import SimpleCookie
from rql.utils import rqlvar_maker
@@ -41,7 +42,7 @@
from cubicweb import AuthenticationError
from cubicweb.req import RequestSessionBase
from cubicweb.uilib import remove_html_tags, js
-from cubicweb.utils import SizeConstrainedList, HTMLHead, make_uid
+from cubicweb.utils import HTMLHead, make_uid
from cubicweb.view import TRANSITIONAL_DOCTYPE_NOEXT
from cubicweb.web import (INTERNAL_FIELD_VALUE, LOGGER, NothingToEdit,
RequestError, StatusResponse)
@@ -51,7 +52,7 @@
_MARKER = object()
def build_cb_uid(seed):
- sha = sha1('%s%s%s' % (time.time(), seed, random.random()))
+ sha = sha1(('%s%s%s' % (time.time(), seed, random.random())).encode('ascii'))
return 'cb_%s' % (sha.hexdigest())
@@ -137,12 +138,12 @@
#: received headers
self._headers_in = Headers()
if headers is not None:
- for k, v in headers.iteritems():
+ for k, v in headers.items():
self._headers_in.addRawHeader(k, v)
#: form parameters
self.setup_params(form)
#: received body
- self.content = StringIO()
+ self.content = BytesIO()
# prepare output header
#: Header used for the final response
self.headers_out = Headers()
@@ -242,7 +243,7 @@
'__redirectvid', '__redirectrql'))
def setup_params(self, params):
- """WARNING: we're intentionaly leaving INTERNAL_FIELD_VALUE here
+ """WARNING: we're intentionally leaving INTERNAL_FIELD_VALUE here
subclasses should overrides to
"""
@@ -250,12 +251,13 @@
if params is None:
return
encoding = self.encoding
- for param, val in params.iteritems():
+ for param, val in params.items():
if isinstance(val, (tuple, list)):
- val = [unicode(x, encoding) for x in val]
+ if PY2:
+ val = [unicode(x, encoding) for x in val]
if len(val) == 1:
val = val[0]
- elif isinstance(val, str):
+ elif PY2 and isinstance(val, str):
val = unicode(val, encoding)
if param in self.no_script_form_params and val:
val = self.no_script_form_param(param, val)
@@ -317,7 +319,7 @@
return None
def set_message(self, msg):
- assert isinstance(msg, unicode)
+ assert isinstance(msg, text_type)
self.reset_message()
self._msg = msg
@@ -330,7 +332,7 @@
def set_redirect_message(self, msg):
# TODO - this should probably be merged with append_to_redirect_message
- assert isinstance(msg, unicode)
+ assert isinstance(msg, text_type)
msgid = self.redirect_message_id()
self.session.data[msgid] = msg
return msgid
@@ -396,26 +398,6 @@
return False
return True
- def update_breadcrumbs(self):
- """stores the last visisted page in session data"""
- searchstate = self.search_state[0]
- if searchstate == 'normal':
- breadcrumbs = self.session.data.get('breadcrumbs')
- if breadcrumbs is None:
- breadcrumbs = SizeConstrainedList(10)
- self.session.data['breadcrumbs'] = breadcrumbs
- breadcrumbs.append(self.url())
- else:
- url = self.url()
- if breadcrumbs and breadcrumbs[-1] != url:
- breadcrumbs.append(url)
-
- def last_visited_page(self):
- breadcrumbs = self.session.data.get('breadcrumbs')
- if breadcrumbs:
- return breadcrumbs.pop()
- return self.base_url()
-
# web edition helpers #####################################################
@cached # so it's writed only once
@@ -437,7 +419,7 @@
eids = form['eid']
except KeyError:
raise NothingToEdit(self._('no selected entities'))
- if isinstance(eids, basestring):
+ if isinstance(eids, string_types):
eids = (eids,)
for peid in eids:
if withtype:
@@ -569,18 +551,18 @@
header = [disposition]
unicode_filename = None
try:
- ascii_filename = filename.encode('ascii')
+ ascii_filename = filename.encode('ascii').decode('ascii')
except UnicodeEncodeError:
# fallback filename for very old browser
unicode_filename = filename
- ascii_filename = filename.encode('ascii', 'ignore')
+ ascii_filename = filename.encode('ascii', 'ignore').decode('ascii')
# escape " and \
# see http://greenbytes.de/tech/tc2231/#attwithfilenameandextparamescaped
ascii_filename = ascii_filename.replace('\x5c', r'\\').replace('"', r'\"')
header.append('filename="%s"' % ascii_filename)
if unicode_filename is not None:
# encoded filename according RFC5987
- urlquoted_filename = urllib.quote(unicode_filename.encode('utf-8'), '')
+ urlquoted_filename = urlquote(unicode_filename.encode('utf-8'), '')
header.append("filename*=utf-8''" + urlquoted_filename)
self.set_header('content-disposition', ';'.join(header))
@@ -596,7 +578,7 @@
:param localfile: if True, the default data dir prefix is added to the
JS filename
"""
- if isinstance(jsfiles, basestring):
+ if isinstance(jsfiles, string_types):
jsfiles = (jsfiles,)
for jsfile in jsfiles:
if localfile:
@@ -616,7 +598,7 @@
the css inclusion. cf:
http://msdn.microsoft.com/en-us/library/ms537512(VS.85).aspx
"""
- if isinstance(cssfiles, basestring):
+ if isinstance(cssfiles, string_types):
cssfiles = (cssfiles,)
if ieonly:
if self.ie_browser():
@@ -702,7 +684,7 @@
return urlsplit(self.base_url())[2]
def data_url(self, relpath):
- """returns the absolute path for a data resouce"""
+ """returns the absolute path for a data resource"""
return self.datadir_url + relpath
@cached
@@ -722,25 +704,19 @@
Some response cache headers may be set by this method.
"""
modified = True
- if self.get_header('Cache-Control') not in ('max-age=0', 'no-cache'):
- # Here, we search for any invalid 'not modified' condition
- # see http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.3
- validators = get_validators(self._headers_in)
- if validators: # if we have no
- modified = any(func(val, self.headers_out) for func, val in validators)
+ # Here, we search for any invalid 'not modified' condition
+ # see http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.3
+ validators = get_validators(self._headers_in)
+ if validators: # if we have no
+ modified = any(func(val, self.headers_out) for func, val in validators)
# Forge expected response
- if modified:
- if 'Expires' not in self.headers_out:
- # Expires header seems to be required by IE7 -- Are you sure ?
- self.add_header('Expires', 'Sat, 01 Jan 2000 00:00:00 GMT')
- # /!\ no raise, the function returns and we keep processing the request
- else:
+ if not modified:
# overwrite headers_out to forge a brand new not-modified response
self.headers_out = self._forge_cached_headers()
if self.http_method() in ('HEAD', 'GET'):
- self.status_out = httplib.NOT_MODIFIED
+ self.status_out = http_client.NOT_MODIFIED
else:
- self.status_out = httplib.PRECONDITION_FAILED
+ self.status_out = http_client.PRECONDITION_FAILED
# XXX replace by True once validate_cache bw compat method is dropped
return self.status_out
# XXX replace by False once validate_cache bw compat method is dropped
@@ -800,7 +776,7 @@
def header_accept_language(self):
"""returns an ordered list of preferred languages"""
acceptedlangs = self.get_header('Accept-Language', raw=False) or {}
- for lang, _ in sorted(acceptedlangs.iteritems(), key=lambda x: x[1],
+ for lang, _ in sorted(acceptedlangs.items(), key=lambda x: x[1],
reverse=True):
lang = lang.split('-')[0]
yield lang
@@ -844,7 +820,7 @@
scheme = scheme.lower()
try:
assert scheme == "basic"
- user, passwd = base64.decodestring(rest).split(":", 1)
+ user, passwd = base64.decodestring(rest.encode('ascii')).split(b":", 1)
# XXX HTTP header encoding: use email.Header?
return user.decode('UTF8'), passwd
except Exception as ex:
@@ -966,8 +942,10 @@
def __getattribute__(self, attr):
raise AuthenticationError()
- def __nonzero__(self):
+ def __bool__(self):
return False
+
+ __nonzero__ = __bool__
class _MockAnonymousSession(object):
sessionid = 'thisisnotarealsession'
@@ -1023,8 +1001,8 @@
self.set_language(lang)
except KeyError:
# this occurs usually during test execution
- self._ = self.__ = unicode
- self.pgettext = lambda x, y: unicode(y)
+ self._ = self.__ = text_type
+ self.pgettext = lambda x, y: text_type(y)
entity_metas = _cnx_func('entity_metas')
source_defs = _cnx_func('source_defs')
--- a/web/schemaviewer.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/schemaviewer.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,9 @@
"""an helper class to display CubicWeb schema using ureports"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six import string_types
from logilab.common.ureports import Section, Title, Table, Link, Span, Text
@@ -218,7 +220,7 @@
elif prop == 'constraints':
val = ', '.join([c.expression for c in val])
elif isinstance(val, dict):
- for key, value in val.iteritems():
+ for key, value in val.items():
if isinstance(value, (list, tuple)):
val[key] = ', '.join(sorted( str(v) for v in value))
val = str(val)
@@ -226,7 +228,7 @@
elif isinstance(val, (list, tuple)):
val = sorted(val)
val = ', '.join(str(v) for v in val)
- elif val and isinstance(val, basestring):
+ elif val and isinstance(val, string_types):
val = _(val)
else:
val = str(val)
--- a/web/test/data/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/data/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,6 +20,9 @@
String, Int, Datetime, Boolean, Float)
from yams.constraints import IntervalBoundConstraint
+from cubicweb import _
+
+
class Salesterm(EntityType):
described_by_test = SubjectRelation('File', cardinality='1*',
composite='subject', inlined=True)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/data/static/jstests/ajax_url0.html Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,3 @@
+<div id="ajaxroot">
+ <h1>Hello</h1>
+</div>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/data/static/jstests/ajax_url1.html Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,6 @@
+<div id="ajaxroot">
+ <div class="ajaxHtmlHead">
+ <cubicweb:script src="http://foo.js" type="text/javascript"> </cubicweb:script>
+ </div>
+ <h1>Hello</h1>
+</div>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/data/static/jstests/ajaxresult.json Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,1 @@
+["foo", "bar"]
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/data/static/jstests/test_ajax.html Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,26 @@
+<html>
+ <head>
+ <!-- dependencies -->
+ <script type="text/javascript">
+ var JSON_BASE_URL = '';
+ </script>
+ <script type="text/javascript" src="/data/jquery.js"></script>
+ <script src="/data/cubicweb.js" type="text/javascript"></script>
+ <script src="/data/cubicweb.htmlhelpers.js" type="text/javascript"></script>
+ <script src="/data/cubicweb.python.js" type="text/javascript"></script>
+ <script src="/data/cubicweb.compat.js" type="text/javascript"></script>
+ <script src="/data/cubicweb.ajax.js" type="text/javascript"></script>
+ <!-- qunit files -->
+ <script type="text/javascript" src="/devtools/qunit.js"></script>
+ <link rel="stylesheet" type="text/css" media="all" href="/devtools/qunit.css" />
+ <!-- test suite -->
+ <script src="/devtools/cwmock.js" type="text/javascript"></script>
+ <script src="test_ajax.js" type="text/javascript"></script>
+ </head>
+ <body>
+ <div id="main"> </div>
+ <h1 id="qunit-header">cubicweb.ajax.js functions tests</h1>
+ <h2 id="qunit-banner"></h2>
+ <ol id="qunit-tests">
+ </body>
+</html>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/data/static/jstests/test_ajax.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,274 @@
+$(document).ready(function() {
+
+ QUnit.module("ajax", {
+ setup: function() {
+ this.scriptsLength = $('head script[src]').length-1;
+ this.cssLength = $('head link[rel=stylesheet]').length-1;
+ // re-initialize cw loaded cache so that each tests run in a
+ // clean environment, have a lookt at _loadAjaxHtmlHead implementation
+ // in cubicweb.ajax.js for more information.
+ cw.loaded_scripts = [];
+ cw.loaded_links = [];
+ },
+ teardown: function() {
+ $('head script[src]:lt(' + ($('head script[src]').length - 1 - this.scriptsLength) + ')').remove();
+ $('head link[rel=stylesheet]:gt(' + this.cssLength + ')').remove();
+ }
+ });
+
+ function jsSources() {
+ return $.map($('head script[src]'), function(script) {
+ return script.getAttribute('src');
+ });
+ }
+
+ QUnit.test('test simple h1 inclusion (ajax_url0.html)', function (assert) {
+ assert.expect(3);
+ assert.equal($('#qunit-fixture').children().length, 0);
+ var done = assert.async();
+ $('#qunit-fixture').loadxhtml('static/jstests/ajax_url0.html', null, 'GET')
+ .addCallback(function() {
+ try {
+ assert.equal($('#qunit-fixture').children().length, 1);
+ assert.equal($('#qunit-fixture h1').html(), 'Hello');
+ } finally {
+ done();
+ };
+ }
+ );
+ });
+
+ QUnit.test('test simple html head inclusion (ajax_url1.html)', function (assert) {
+ assert.expect(6);
+ var scriptsIncluded = jsSources();
+ assert.equal(jQuery.inArray('http://foo.js', scriptsIncluded), - 1);
+ var done = assert.async();
+ $('#qunit-fixture').loadxhtml('static/jstests/ajax_url1.html', null, 'GET')
+ .addCallback(function() {
+ try {
+ var origLength = scriptsIncluded.length;
+ scriptsIncluded = jsSources();
+ // check that foo.js has been prepended to <head>
+ assert.equal(scriptsIncluded.length, origLength + 1);
+ assert.equal(scriptsIncluded.indexOf('http://foo.js'), 0);
+ // check that <div class="ajaxHtmlHead"> has been removed
+ assert.equal($('#qunit-fixture').children().length, 1);
+ assert.equal($('div.ajaxHtmlHead').length, 0);
+ assert.equal($('#qunit-fixture h1').html(), 'Hello');
+ } finally {
+ done();
+ };
+ }
+ );
+ });
+
+ QUnit.test('test addCallback', function (assert) {
+ assert.expect(3);
+ assert.equal($('#qunit-fixture').children().length, 0);
+ var done = assert.async();
+ var d = $('#qunit-fixture').loadxhtml('static/jstests/ajax_url0.html', null, 'GET');
+ d.addCallback(function() {
+ try {
+ assert.equal($('#qunit-fixture').children().length, 1);
+ assert.equal($('#qunit-fixture h1').html(), 'Hello');
+ } finally {
+ done();
+ };
+ });
+ });
+
+ QUnit.test('test callback after synchronous request', function (assert) {
+ assert.expect(1);
+ var deferred = new Deferred();
+ var result = jQuery.ajax({
+ url: 'static/jstests/ajax_url0.html',
+ async: false,
+ beforeSend: function(xhr) {
+ deferred._req = xhr;
+ },
+ success: function(data, status) {
+ deferred.success(data);
+ }
+ });
+ var done = assert.async();
+ deferred.addCallback(function() {
+ try {
+ // add an assertion to ensure the callback is executed
+ assert.ok(true, "callback is executed");
+ } finally {
+ done();
+ };
+ });
+ });
+
+ QUnit.test('test addCallback with parameters', function (assert) {
+ assert.expect(3);
+ assert.equal($('#qunit-fixture').children().length, 0);
+ var done = assert.async();
+ var d = $('#qunit-fixture').loadxhtml('static/jstests/ajax_url0.html', null, 'GET');
+ d.addCallback(function(data, req, arg1, arg2) {
+ try {
+ assert.equal(arg1, 'Hello');
+ assert.equal(arg2, 'world');
+ } finally {
+ done();
+ };
+ },
+ 'Hello', 'world');
+ });
+
+ QUnit.test('test callback after synchronous request with parameters', function (assert) {
+ assert.expect(3);
+ var deferred = new Deferred();
+ deferred.addCallback(function(data, req, arg1, arg2) {
+ // add an assertion to ensure the callback is executed
+ try {
+ assert.ok(true, "callback is executed");
+ assert.equal(arg1, 'Hello');
+ assert.equal(arg2, 'world');
+ } finally {
+ done();
+ };
+ },
+ 'Hello', 'world');
+ deferred.addErrback(function() {
+ // throw an exception to start errback chain
+ try {
+ throw this._error;
+ } finally {
+ done();
+ };
+ });
+ var done = assert.async();
+ var result = jQuery.ajax({
+ url: 'static/jstests/ajax_url0.html',
+ async: false,
+ beforeSend: function(xhr) {
+ deferred._req = xhr;
+ },
+ success: function(data, status) {
+ deferred.success(data);
+ }
+ });
+ });
+
+ QUnit.test('test addErrback', function (assert) {
+ assert.expect(1);
+ var done = assert.async();
+ var d = $('#qunit-fixture').loadxhtml('static/jstests/nonexistent.html', null, 'GET');
+ d.addCallback(function() {
+ // should not be executed
+ assert.ok(false, "callback is executed");
+ });
+ d.addErrback(function() {
+ try {
+ assert.ok(true, "errback is executed");
+ } finally {
+ done();
+ };
+ });
+ });
+
+ QUnit.test('test callback execution order', function (assert) {
+ assert.expect(3);
+ var counter = 0;
+ var done = assert.async();
+ var d = $('#qunit-fixture').loadxhtml('static/jstests/ajax_url0.html', null, 'GET');
+ d.addCallback(function() {
+ assert.equal(++counter, 1); // should be executed first
+ });
+ d.addCallback(function() {
+ assert.equal(++counter, 2);
+ });
+ d.addCallback(function() {
+ try {
+ assert.equal(++counter, 3);
+ } finally {
+ done();
+ }
+ });
+ });
+
+ QUnit.test('test already included resources are ignored (ajax_url1.html)', function (assert) {
+ assert.expect(10);
+ var scriptsIncluded = jsSources();
+ // NOTE:
+ assert.equal(jQuery.inArray('http://foo.js', scriptsIncluded), -1);
+ assert.equal($('head link').length, 1);
+ /* use endswith because in pytest context we have an absolute path */
+ assert.ok($('head link').attr('href').endswith('/qunit.css'), 'qunit.css is loaded');
+ var done = assert.async();
+ $('#qunit-fixture').loadxhtml('static/jstests/ajax_url1.html', null, 'GET')
+ .addCallback(function() {
+ var origLength = scriptsIncluded.length;
+ scriptsIncluded = jsSources();
+ try {
+ // check that foo.js has been inserted in <head>
+ assert.equal(scriptsIncluded.length, origLength + 1);
+ assert.equal(scriptsIncluded.indexOf('http://foo.js'), 0);
+ // check that <div class="ajaxHtmlHead"> has been removed
+ assert.equal($('#qunit-fixture').children().length, 1);
+ assert.equal($('div.ajaxHtmlHead').length, 0);
+ assert.equal($('#qunit-fixture h1').html(), 'Hello');
+ // qunit.css is not added twice
+ assert.equal($('head link').length, 1);
+ /* use endswith because in pytest context we have an absolute path */
+ assert.ok($('head link').attr('href').endswith('/qunit.css'), 'qunit.css is loaded');
+ } finally {
+ done();
+ }
+ }
+ );
+ });
+
+ QUnit.test('test synchronous request loadRemote', function (assert) {
+ var res = loadRemote('static/jstests/ajaxresult.json', {},
+ 'GET', true);
+ assert.deepEqual(res, ['foo', 'bar']);
+ });
+
+ QUnit.test('test event on CubicWeb', function (assert) {
+ assert.expect(1);
+ var done = assert.async();
+ var events = null;
+ $(CubicWeb).bind('server-response', function() {
+ // check that server-response event on CubicWeb is triggered
+ events = 'CubicWeb';
+ });
+ $('#qunit-fixture').loadxhtml('static/jstests/ajax_url0.html', null, 'GET')
+ .addCallback(function() {
+ try {
+ assert.equal(events, 'CubicWeb');
+ } finally {
+ done();
+ };
+ }
+ );
+ });
+
+ QUnit.test('test event on node', function (assert) {
+ assert.expect(3);
+ var done = assert.async();
+ var nodes = [];
+ $('#qunit-fixture').bind('server-response', function() {
+ nodes.push('node');
+ });
+ $(CubicWeb).bind('server-response', function() {
+ nodes.push('CubicWeb');
+ });
+ $('#qunit-fixture').loadxhtml('static/jstests/ajax_url0.html', null, 'GET')
+ .addCallback(function() {
+ try {
+ assert.equal(nodes.length, 2);
+ // check that server-response event on CubicWeb is triggered
+ // only once and event server-response on node is triggered
+ assert.equal(nodes[0], 'CubicWeb');
+ assert.equal(nodes[1], 'node');
+ } finally {
+ done();
+ };
+ }
+ );
+ });
+});
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/data/static/jstests/test_htmlhelpers.html Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,22 @@
+<html>
+ <head>
+ <!-- dependencies -->
+ <script type="text/javascript" src="../../data/jquery.js"></script>
+ <script src="../../data/cubicweb.python.js" type="text/javascript"></script>
+ <script src="../../data/cubicweb.js" type="text/javascript"></script>
+ <script src="../../data/cubicweb.compat.js" type="text/javascript"></script>
+ <script src="../../data/cubicweb.htmlhelpers.js" type="text/javascript"></script>
+ <!-- qunit files -->
+ <script type="text/javascript" src="../../../devtools/data/qunit.js"></script>
+ <link rel="stylesheet" type="text/css" media="all" href="../../../devtools/data/qunit.css" />
+ <!-- test suite -->
+ <script src="cwmock.js" type="text/javascript"></script>
+ <script src="test_htmlhelpers.js" type="text/javascript"></script>
+ </head>
+ <body>
+ <div id="main"> </div>
+ <h1 id="qunit-header">cubicweb.htmlhelpers.js functions tests</h1>
+ <h2 id="qunit-banner"></h2>
+ <ol id="qunit-tests">
+ </body>
+</html>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/data/static/jstests/test_htmlhelpers.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,36 @@
+$(document).ready(function() {
+
+ QUnit.module("module2", {
+ setup: function() {
+ $('#qunit-fixture').append('<select id="theselect" multiple="multiple" size="2">' +
+ '</select>');
+ }
+ });
+
+ QUnit.test("test first selected", function (assert) {
+ $('#theselect').append('<option value="foo">foo</option>' +
+ '<option selected="selected" value="bar">bar</option>' +
+ '<option value="baz">baz</option>' +
+ '<option selected="selecetd"value="spam">spam</option>');
+ var selected = firstSelected(document.getElementById("theselect"));
+ assert.equal(selected.value, 'bar');
+ });
+
+ QUnit.test("test first selected 2", function (assert) {
+ $('#theselect').append('<option value="foo">foo</option>' +
+ '<option value="bar">bar</option>' +
+ '<option value="baz">baz</option>' +
+ '<option value="spam">spam</option>');
+ var selected = firstSelected(document.getElementById("theselect"));
+ assert.equal(selected, null);
+ });
+
+ QUnit.module("visibilty");
+ QUnit.test('toggleVisibility', function (assert) {
+ $('#qunit-fixture').append('<div id="foo"></div>');
+ toggleVisibility('foo');
+ assert.ok($('#foo').hasClass('hidden'), 'check hidden class is set');
+ });
+
+});
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/data/static/jstests/test_utils.html Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,22 @@
+<html>
+ <head>
+ <!-- dependencies -->
+ <script type="text/javascript" src="utils.js"></script>
+ <script type="text/javascript" src="../../data/jquery.js"></script>
+ <script src="../../data/cubicweb.python.js" type="text/javascript"></script>
+ <script src="../../data/cubicweb.js" type="text/javascript"></script>
+ <script src="../../data/cubicweb.compat.js" type="text/javascript"></script>
+ <!-- qunit files -->
+ <script type="text/javascript" src="../../../devtools/data/qunit.js"></script>
+ <link rel="stylesheet" type="text/css" media="all" href="../../../devtools/data/qunit.css" />
+ <!-- test suite -->
+ <script src="cwmock.js" type="text/javascript"></script>
+ <script src="test_utils.js" type="text/javascript"></script>
+ </head>
+ <body>
+ <div id="main"> </div>
+ <h1 id="qunit-header">cw.utils functions tests</h1>
+ <h2 id="qunit-banner"></h2>
+ <ol id="qunit-tests">
+ </body>
+</html>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/data/static/jstests/test_utils.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,92 @@
+$(document).ready(function() {
+
+ QUnit.module("datetime");
+
+ QUnit.test("test full datetime", function (assert) {
+ assert.equal(cw.utils.toISOTimestamp(new Date(1986, 3, 18, 10, 30, 0, 0)),
+ '1986-04-18 10:30:00');
+ });
+
+ QUnit.test("test only date", function (assert) {
+ assert.equal(cw.utils.toISOTimestamp(new Date(1986, 3, 18)), '1986-04-18 00:00:00');
+ });
+
+ QUnit.test("test null", function (assert) {
+ assert.equal(cw.utils.toISOTimestamp(null), null);
+ });
+
+ QUnit.module("parsing");
+ QUnit.test("test basic number parsing", function (assert) {
+ var d = strptime('2008/08/08', '%Y/%m/%d');
+ assert.deepEqual(datetuple(d), [2008, 8, 8, 0, 0]);
+ d = strptime('2008/8/8', '%Y/%m/%d');
+ assert.deepEqual(datetuple(d), [2008, 8, 8, 0, 0]);
+ d = strptime('8/8/8', '%Y/%m/%d');
+ assert.deepEqual(datetuple(d), [8, 8, 8, 0, 0]);
+ d = strptime('0/8/8', '%Y/%m/%d');
+ assert.deepEqual(datetuple(d), [0, 8, 8, 0, 0]);
+ d = strptime('-10/8/8', '%Y/%m/%d');
+ assert.deepEqual(datetuple(d), [-10, 8, 8, 0, 0]);
+ d = strptime('-35000', '%Y');
+ assert.deepEqual(datetuple(d), [-35000, 1, 1, 0, 0]);
+ });
+
+ QUnit.test("test custom format parsing", function (assert) {
+ var d = strptime('2008-08-08', '%Y-%m-%d');
+ assert.deepEqual(datetuple(d), [2008, 8, 8, 0, 0]);
+ d = strptime('2008 - ! 08: 08', '%Y - ! %m: %d');
+ assert.deepEqual(datetuple(d), [2008, 8, 8, 0, 0]);
+ d = strptime('2008-08-08 12:14', '%Y-%m-%d %H:%M');
+ assert.deepEqual(datetuple(d), [2008, 8, 8, 12, 14]);
+ d = strptime('2008-08-08 1:14', '%Y-%m-%d %H:%M');
+ assert.deepEqual(datetuple(d), [2008, 8, 8, 1, 14]);
+ d = strptime('2008-08-08 01:14', '%Y-%m-%d %H:%M');
+ assert.deepEqual(datetuple(d), [2008, 8, 8, 1, 14]);
+ });
+
+ QUnit.module("sliceList");
+ QUnit.test("test slicelist", function (assert) {
+ var list = ['a', 'b', 'c', 'd', 'e', 'f'];
+ assert.deepEqual(cw.utils.sliceList(list, 2), ['c', 'd', 'e', 'f']);
+ assert.deepEqual(cw.utils.sliceList(list, 2, -2), ['c', 'd']);
+ assert.deepEqual(cw.utils.sliceList(list, -3), ['d', 'e', 'f']);
+ assert.deepEqual(cw.utils.sliceList(list, 0, -2), ['a', 'b', 'c', 'd']);
+ assert.deepEqual(cw.utils.sliceList(list), list);
+ });
+
+ QUnit.module("formContents", {
+ setup: function() {
+ $('#qunit-fixture').append('<form id="test-form"></form>');
+ }
+ });
+ // XXX test fckeditor
+ QUnit.test("test formContents", function (assert) {
+ $('#test-form').append('<input name="input-text" ' +
+ 'type="text" value="toto" />');
+ $('#test-form').append('<textarea rows="10" cols="30" '+
+ 'name="mytextarea">Hello World!</textarea> ');
+ $('#test-form').append('<input name="choice" type="radio" ' +
+ 'value="yes" />');
+ $('#test-form').append('<input name="choice" type="radio" ' +
+ 'value="no" checked="checked"/>');
+ $('#test-form').append('<input name="check" type="checkbox" ' +
+ 'value="yes" />');
+ $('#test-form').append('<input name="check" type="checkbox" ' +
+ 'value="no" checked="checked"/>');
+ $('#test-form').append('<select id="theselect" name="theselect" ' +
+ 'multiple="multiple" size="2"></select>');
+ $('#theselect').append('<option selected="selected" ' +
+ 'value="foo">foo</option>' +
+ '<option value="bar">bar</option>');
+ //Append an unchecked radio input : should not be in formContents list
+ $('#test-form').append('<input name="unchecked-choice" type="radio" ' +
+ 'value="one" />');
+ $('#test-form').append('<input name="unchecked-choice" type="radio" ' +
+ 'value="two"/>');
+ assert.deepEqual(cw.utils.formContents($('#test-form')[0]), [
+ ['input-text', 'mytextarea', 'choice', 'check', 'theselect'],
+ ['toto', 'Hello World!', 'no', 'no', 'foo']
+ ]);
+ });
+});
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/web/test/data/static/jstests/utils.js Wed Dec 16 11:23:48 2015 +0100
@@ -0,0 +1,29 @@
+function datetuple(d) {
+ return [d.getFullYear(), d.getMonth()+1, d.getDate(),
+ d.getHours(), d.getMinutes()];
+}
+
+function pprint(obj) {
+ print('{');
+ for(k in obj) {
+ print(' ' + k + ' = ' + obj[k]);
+ }
+ print('}');
+}
+
+function arrayrepr(array) {
+ return '[' + array.join(', ') + ']';
+}
+
+function assertArrayEquals(array1, array2) {
+ if (array1.length != array2.length) {
+ throw new crosscheck.AssertionFailure(array1.join(', ') + ' != ' + array2.join(', '));
+ }
+ for (var i=0; i<array1.length; i++) {
+ if (array1[i] != array2[i]) {
+
+ throw new crosscheck.AssertionFailure(arrayrepr(array1) + ' and ' + arrayrepr(array2)
+ + ' differs at index ' + i);
+ }
+ }
+}
--- a/web/test/data/views.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/data/views.py Wed Dec 16 11:23:48 2015 +0100
@@ -42,10 +42,10 @@
"""
try:
result_dict = {}
- for key, value in self._cw.form.iteritems():
+ for key, value in self._cw.form.items():
result_dict[key] = _recursive_replace_stream_by_content(value)
return result_dict
- except Exception, ex:
+ except Exception as ex:
import traceback as tb
tb.print_exc(ex)
--- a/web/test/jstests/ajax_url0.html Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,3 +0,0 @@
-<div id="ajaxroot">
- <h1>Hello</h1>
-</div>
--- a/web/test/jstests/ajax_url1.html Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,6 +0,0 @@
-<div id="ajaxroot">
- <div class="ajaxHtmlHead">
- <cubicweb:script src="http://foo.js" type="text/javascript"> </cubicweb:script>
- </div>
- <h1>Hello</h1>
-</div>
--- a/web/test/jstests/ajax_url2.html Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,7 +0,0 @@
-<div id="ajaxroot">
- <div class="ajaxHtmlHead">
- <script src="http://foo.js" type="text/javascript"> </script>
- <link rel="stylesheet" type="text/css" media="all" href="qunit.css" />
- </div>
- <h1>Hello</h1>
-</div>
--- a/web/test/jstests/ajaxresult.json Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,1 +0,0 @@
-["foo", "bar"]
--- a/web/test/jstests/test_ajax.html Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,26 +0,0 @@
-<html>
- <head>
- <!-- dependencies -->
- <script type="text/javascript">
- var JSON_BASE_URL = '';
- </script>
- <script type="text/javascript" src="../../data/jquery.js"></script>
- <script src="../../data/cubicweb.js" type="text/javascript"></script>
- <script src="../../data/cubicweb.htmlhelpers.js" type="text/javascript"></script>
- <script src="../../data/cubicweb.python.js" type="text/javascript"></script>
- <script src="../../data/cubicweb.compat.js" type="text/javascript"></script>
- <script src="../../data/cubicweb.ajax.js" type="text/javascript"></script>
- <!-- qunit files -->
- <script type="text/javascript" src="../../../devtools/data/qunit.js"></script>
- <link rel="stylesheet" type="text/css" media="all" href="../../../devtools/data/qunit.css" />
- <!-- test suite -->
- <script src="cwmock.js" type="text/javascript"></script>
- <script src="test_ajax.js" type="text/javascript"></script>
- </head>
- <body>
- <div id="main"> </div>
- <h1 id="qunit-header">cubicweb.ajax.js functions tests</h1>
- <h2 id="qunit-banner"></h2>
- <ol id="qunit-tests">
- </body>
-</html>
--- a/web/test/jstests/test_ajax.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,274 +0,0 @@
-$(document).ready(function() {
-
- QUnit.module("ajax", {
- setup: function() {
- this.scriptsLength = $('head script[src]').length-1;
- this.cssLength = $('head link[rel=stylesheet]').length-1;
- // re-initialize cw loaded cache so that each tests run in a
- // clean environment, have a lookt at _loadAjaxHtmlHead implementation
- // in cubicweb.ajax.js for more information.
- cw.loaded_scripts = [];
- cw.loaded_links = [];
- },
- teardown: function() {
- $('head script[src]:lt(' + ($('head script[src]').length - 1 - this.scriptsLength) + ')').remove();
- $('head link[rel=stylesheet]:gt(' + this.cssLength + ')').remove();
- }
- });
-
- function jsSources() {
- return $.map($('head script[src]'), function(script) {
- return script.getAttribute('src');
- });
- }
-
- QUnit.test('test simple h1 inclusion (ajax_url0.html)', function (assert) {
- assert.expect(3);
- assert.equal($('#qunit-fixture').children().length, 0);
- var done = assert.async();
- $('#qunit-fixture').loadxhtml(BASE_URL + 'cwsoftwareroot/web/test/jstests/ajax_url0.html')
- .addCallback(function() {
- try {
- assert.equal($('#qunit-fixture').children().length, 1);
- assert.equal($('#qunit-fixture h1').html(), 'Hello');
- } finally {
- done();
- };
- }
- );
- });
-
- QUnit.test('test simple html head inclusion (ajax_url1.html)', function (assert) {
- assert.expect(6);
- var scriptsIncluded = jsSources();
- assert.equal(jQuery.inArray('http://foo.js', scriptsIncluded), - 1);
- var done = assert.async();
- $('#qunit-fixture').loadxhtml(BASE_URL + 'cwsoftwareroot/web/test/jstests/ajax_url1.html')
- .addCallback(function() {
- try {
- var origLength = scriptsIncluded.length;
- scriptsIncluded = jsSources();
- // check that foo.js has been prepended to <head>
- assert.equal(scriptsIncluded.length, origLength + 1);
- assert.equal(scriptsIncluded.indexOf('http://foo.js'), 0);
- // check that <div class="ajaxHtmlHead"> has been removed
- assert.equal($('#qunit-fixture').children().length, 1);
- assert.equal($('div.ajaxHtmlHead').length, 0);
- assert.equal($('#qunit-fixture h1').html(), 'Hello');
- } finally {
- done();
- };
- }
- );
- });
-
- QUnit.test('test addCallback', function (assert) {
- assert.expect(3);
- assert.equal($('#qunit-fixture').children().length, 0);
- var done = assert.async();
- var d = $('#qunit-fixture').loadxhtml(BASE_URL + 'cwsoftwareroot/web/test/jstests/ajax_url0.html');
- d.addCallback(function() {
- try {
- assert.equal($('#qunit-fixture').children().length, 1);
- assert.equal($('#qunit-fixture h1').html(), 'Hello');
- } finally {
- done();
- };
- });
- });
-
- QUnit.test('test callback after synchronous request', function (assert) {
- assert.expect(1);
- var deferred = new Deferred();
- var result = jQuery.ajax({
- url: BASE_URL + 'cwsoftwareroot/web/test/jstests/ajax_url0.html',
- async: false,
- beforeSend: function(xhr) {
- deferred._req = xhr;
- },
- success: function(data, status) {
- deferred.success(data);
- }
- });
- var done = assert.async();
- deferred.addCallback(function() {
- try {
- // add an assertion to ensure the callback is executed
- assert.ok(true, "callback is executed");
- } finally {
- done();
- };
- });
- });
-
- QUnit.test('test addCallback with parameters', function (assert) {
- assert.expect(3);
- assert.equal($('#qunit-fixture').children().length, 0);
- var done = assert.async();
- var d = $('#qunit-fixture').loadxhtml(BASE_URL + 'cwsoftwareroot/web/test/jstests/ajax_url0.html');
- d.addCallback(function(data, req, arg1, arg2) {
- try {
- assert.equal(arg1, 'Hello');
- assert.equal(arg2, 'world');
- } finally {
- done();
- };
- },
- 'Hello', 'world');
- });
-
- QUnit.test('test callback after synchronous request with parameters', function (assert) {
- assert.expect(3);
- var deferred = new Deferred();
- deferred.addCallback(function(data, req, arg1, arg2) {
- // add an assertion to ensure the callback is executed
- try {
- assert.ok(true, "callback is executed");
- assert.equal(arg1, 'Hello');
- assert.equal(arg2, 'world');
- } finally {
- done();
- };
- },
- 'Hello', 'world');
- deferred.addErrback(function() {
- // throw an exception to start errback chain
- try {
- throw this._error;
- } finally {
- done();
- };
- });
- var done = assert.async();
- var result = jQuery.ajax({
- url: BASE_URL + 'cwsoftwareroot/web/test/jstests/ajax_url0.html',
- async: false,
- beforeSend: function(xhr) {
- deferred._req = xhr;
- },
- success: function(data, status) {
- deferred.success(data);
- }
- });
- });
-
- QUnit.test('test addErrback', function (assert) {
- assert.expect(1);
- var done = assert.async();
- var d = $('#qunit-fixture').loadxhtml(BASE_URL + 'cwsoftwareroot/web/test/jstests/nonexistent.html');
- d.addCallback(function() {
- // should not be executed
- assert.ok(false, "callback is executed");
- });
- d.addErrback(function() {
- try {
- assert.ok(true, "errback is executed");
- } finally {
- done();
- };
- });
- });
-
- QUnit.test('test callback execution order', function (assert) {
- assert.expect(3);
- var counter = 0;
- var done = assert.async();
- var d = $('#qunit-fixture').loadxhtml(BASE_URL + 'cwsoftwareroot/web/test/jstests/ajax_url0.html');
- d.addCallback(function() {
- assert.equal(++counter, 1); // should be executed first
- });
- d.addCallback(function() {
- assert.equal(++counter, 2);
- });
- d.addCallback(function() {
- try {
- assert.equal(++counter, 3);
- } finally {
- done();
- }
- });
- });
-
- QUnit.test('test already included resources are ignored (ajax_url1.html)', function (assert) {
- assert.expect(10);
- var scriptsIncluded = jsSources();
- // NOTE:
- assert.equal(jQuery.inArray('http://foo.js', scriptsIncluded), -1);
- assert.equal($('head link').length, 1);
- /* use endswith because in pytest context we have an absolute path */
- assert.ok($('head link').attr('href').endswith('/qunit.css'), 'qunit.css is loaded');
- var done = assert.async();
- $('#qunit-fixture').loadxhtml(BASE_URL + 'cwsoftwareroot/web/test/jstests/ajax_url1.html')
- .addCallback(function() {
- var origLength = scriptsIncluded.length;
- scriptsIncluded = jsSources();
- try {
- // check that foo.js has been inserted in <head>
- assert.equal(scriptsIncluded.length, origLength + 1);
- assert.equal(scriptsIncluded.indexOf('http://foo.js'), 0);
- // check that <div class="ajaxHtmlHead"> has been removed
- assert.equal($('#qunit-fixture').children().length, 1);
- assert.equal($('div.ajaxHtmlHead').length, 0);
- assert.equal($('#qunit-fixture h1').html(), 'Hello');
- // qunit.css is not added twice
- assert.equal($('head link').length, 1);
- /* use endswith because in pytest context we have an absolute path */
- assert.ok($('head link').attr('href').endswith('/qunit.css'), 'qunit.css is loaded');
- } finally {
- done();
- }
- }
- );
- });
-
- QUnit.test('test synchronous request loadRemote', function (assert) {
- var res = loadRemote(BASE_URL + 'cwsoftwareroot/web/test/jstests/ajaxresult.json', {},
- 'GET', true);
- assert.deepEqual(res, ['foo', 'bar']);
- });
-
- QUnit.test('test event on CubicWeb', function (assert) {
- assert.expect(1);
- var done = assert.async();
- var events = null;
- $(CubicWeb).bind('server-response', function() {
- // check that server-response event on CubicWeb is triggered
- events = 'CubicWeb';
- });
- $('#qunit-fixture').loadxhtml(BASE_URL + 'cwsoftwareroot/web/test/jstests/ajax_url0.html')
- .addCallback(function() {
- try {
- assert.equal(events, 'CubicWeb');
- } finally {
- done();
- };
- }
- );
- });
-
- QUnit.test('test event on node', function (assert) {
- assert.expect(3);
- var done = assert.async();
- var nodes = [];
- $('#qunit-fixture').bind('server-response', function() {
- nodes.push('node');
- });
- $(CubicWeb).bind('server-response', function() {
- nodes.push('CubicWeb');
- });
- $('#qunit-fixture').loadxhtml(BASE_URL + 'cwsoftwareroot/web/test/jstests/ajax_url0.html')
- .addCallback(function() {
- try {
- assert.equal(nodes.length, 2);
- // check that server-response event on CubicWeb is triggered
- // only once and event server-response on node is triggered
- assert.equal(nodes[0], 'CubicWeb');
- assert.equal(nodes[1], 'node');
- } finally {
- done();
- };
- }
- );
- });
-});
-
--- a/web/test/jstests/test_htmlhelpers.html Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,22 +0,0 @@
-<html>
- <head>
- <!-- dependencies -->
- <script type="text/javascript" src="../../data/jquery.js"></script>
- <script src="../../data/cubicweb.python.js" type="text/javascript"></script>
- <script src="../../data/cubicweb.js" type="text/javascript"></script>
- <script src="../../data/cubicweb.compat.js" type="text/javascript"></script>
- <script src="../../data/cubicweb.htmlhelpers.js" type="text/javascript"></script>
- <!-- qunit files -->
- <script type="text/javascript" src="../../../devtools/data/qunit.js"></script>
- <link rel="stylesheet" type="text/css" media="all" href="../../../devtools/data/qunit.css" />
- <!-- test suite -->
- <script src="cwmock.js" type="text/javascript"></script>
- <script src="test_htmlhelpers.js" type="text/javascript"></script>
- </head>
- <body>
- <div id="main"> </div>
- <h1 id="qunit-header">cubicweb.htmlhelpers.js functions tests</h1>
- <h2 id="qunit-banner"></h2>
- <ol id="qunit-tests">
- </body>
-</html>
--- a/web/test/jstests/test_htmlhelpers.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,36 +0,0 @@
-$(document).ready(function() {
-
- QUnit.module("module2", {
- setup: function() {
- $('#qunit-fixture').append('<select id="theselect" multiple="multiple" size="2">' +
- '</select>');
- }
- });
-
- QUnit.test("test first selected", function (assert) {
- $('#theselect').append('<option value="foo">foo</option>' +
- '<option selected="selected" value="bar">bar</option>' +
- '<option value="baz">baz</option>' +
- '<option selected="selecetd"value="spam">spam</option>');
- var selected = firstSelected(document.getElementById("theselect"));
- assert.equal(selected.value, 'bar');
- });
-
- QUnit.test("test first selected 2", function (assert) {
- $('#theselect').append('<option value="foo">foo</option>' +
- '<option value="bar">bar</option>' +
- '<option value="baz">baz</option>' +
- '<option value="spam">spam</option>');
- var selected = firstSelected(document.getElementById("theselect"));
- assert.equal(selected, null);
- });
-
- QUnit.module("visibilty");
- QUnit.test('toggleVisibility', function (assert) {
- $('#qunit-fixture').append('<div id="foo"></div>');
- toggleVisibility('foo');
- assert.ok($('#foo').hasClass('hidden'), 'check hidden class is set');
- });
-
-});
-
--- a/web/test/jstests/test_utils.html Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,22 +0,0 @@
-<html>
- <head>
- <!-- dependencies -->
- <script type="text/javascript" src="utils.js"></script>
- <script type="text/javascript" src="../../data/jquery.js"></script>
- <script src="../../data/cubicweb.python.js" type="text/javascript"></script>
- <script src="../../data/cubicweb.js" type="text/javascript"></script>
- <script src="../../data/cubicweb.compat.js" type="text/javascript"></script>
- <!-- qunit files -->
- <script type="text/javascript" src="../../../devtools/data/qunit.js"></script>
- <link rel="stylesheet" type="text/css" media="all" href="../../../devtools/data/qunit.css" />
- <!-- test suite -->
- <script src="cwmock.js" type="text/javascript"></script>
- <script src="test_utils.js" type="text/javascript"></script>
- </head>
- <body>
- <div id="main"> </div>
- <h1 id="qunit-header">cw.utils functions tests</h1>
- <h2 id="qunit-banner"></h2>
- <ol id="qunit-tests">
- </body>
-</html>
--- a/web/test/jstests/test_utils.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,92 +0,0 @@
-$(document).ready(function() {
-
- QUnit.module("datetime");
-
- QUnit.test("test full datetime", function (assert) {
- assert.equal(cw.utils.toISOTimestamp(new Date(1986, 3, 18, 10, 30, 0, 0)),
- '1986-04-18 10:30:00');
- });
-
- QUnit.test("test only date", function (assert) {
- assert.equal(cw.utils.toISOTimestamp(new Date(1986, 3, 18)), '1986-04-18 00:00:00');
- });
-
- QUnit.test("test null", function (assert) {
- assert.equal(cw.utils.toISOTimestamp(null), null);
- });
-
- QUnit.module("parsing");
- QUnit.test("test basic number parsing", function (assert) {
- var d = strptime('2008/08/08', '%Y/%m/%d');
- assert.deepEqual(datetuple(d), [2008, 8, 8, 0, 0]);
- d = strptime('2008/8/8', '%Y/%m/%d');
- assert.deepEqual(datetuple(d), [2008, 8, 8, 0, 0]);
- d = strptime('8/8/8', '%Y/%m/%d');
- assert.deepEqual(datetuple(d), [8, 8, 8, 0, 0]);
- d = strptime('0/8/8', '%Y/%m/%d');
- assert.deepEqual(datetuple(d), [0, 8, 8, 0, 0]);
- d = strptime('-10/8/8', '%Y/%m/%d');
- assert.deepEqual(datetuple(d), [-10, 8, 8, 0, 0]);
- d = strptime('-35000', '%Y');
- assert.deepEqual(datetuple(d), [-35000, 1, 1, 0, 0]);
- });
-
- QUnit.test("test custom format parsing", function (assert) {
- var d = strptime('2008-08-08', '%Y-%m-%d');
- assert.deepEqual(datetuple(d), [2008, 8, 8, 0, 0]);
- d = strptime('2008 - ! 08: 08', '%Y - ! %m: %d');
- assert.deepEqual(datetuple(d), [2008, 8, 8, 0, 0]);
- d = strptime('2008-08-08 12:14', '%Y-%m-%d %H:%M');
- assert.deepEqual(datetuple(d), [2008, 8, 8, 12, 14]);
- d = strptime('2008-08-08 1:14', '%Y-%m-%d %H:%M');
- assert.deepEqual(datetuple(d), [2008, 8, 8, 1, 14]);
- d = strptime('2008-08-08 01:14', '%Y-%m-%d %H:%M');
- assert.deepEqual(datetuple(d), [2008, 8, 8, 1, 14]);
- });
-
- QUnit.module("sliceList");
- QUnit.test("test slicelist", function (assert) {
- var list = ['a', 'b', 'c', 'd', 'e', 'f'];
- assert.deepEqual(cw.utils.sliceList(list, 2), ['c', 'd', 'e', 'f']);
- assert.deepEqual(cw.utils.sliceList(list, 2, -2), ['c', 'd']);
- assert.deepEqual(cw.utils.sliceList(list, -3), ['d', 'e', 'f']);
- assert.deepEqual(cw.utils.sliceList(list, 0, -2), ['a', 'b', 'c', 'd']);
- assert.deepEqual(cw.utils.sliceList(list), list);
- });
-
- QUnit.module("formContents", {
- setup: function() {
- $('#qunit-fixture').append('<form id="test-form"></form>');
- }
- });
- // XXX test fckeditor
- QUnit.test("test formContents", function (assert) {
- $('#test-form').append('<input name="input-text" ' +
- 'type="text" value="toto" />');
- $('#test-form').append('<textarea rows="10" cols="30" '+
- 'name="mytextarea">Hello World!</textarea> ');
- $('#test-form').append('<input name="choice" type="radio" ' +
- 'value="yes" />');
- $('#test-form').append('<input name="choice" type="radio" ' +
- 'value="no" checked="checked"/>');
- $('#test-form').append('<input name="check" type="checkbox" ' +
- 'value="yes" />');
- $('#test-form').append('<input name="check" type="checkbox" ' +
- 'value="no" checked="checked"/>');
- $('#test-form').append('<select id="theselect" name="theselect" ' +
- 'multiple="multiple" size="2"></select>');
- $('#theselect').append('<option selected="selected" ' +
- 'value="foo">foo</option>' +
- '<option value="bar">bar</option>');
- //Append an unchecked radio input : should not be in formContents list
- $('#test-form').append('<input name="unchecked-choice" type="radio" ' +
- 'value="one" />');
- $('#test-form').append('<input name="unchecked-choice" type="radio" ' +
- 'value="two"/>');
- assert.deepEqual(cw.utils.formContents($('#test-form')[0]), [
- ['input-text', 'mytextarea', 'choice', 'check', 'theselect'],
- ['toto', 'Hello World!', 'no', 'no', 'foo']
- ]);
- });
-});
-
--- a/web/test/jstests/utils.js Tue Dec 15 19:05:41 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,29 +0,0 @@
-function datetuple(d) {
- return [d.getFullYear(), d.getMonth()+1, d.getDate(),
- d.getHours(), d.getMinutes()];
-}
-
-function pprint(obj) {
- print('{');
- for(k in obj) {
- print(' ' + k + ' = ' + obj[k]);
- }
- print('}');
-}
-
-function arrayrepr(array) {
- return '[' + array.join(', ') + ']';
-}
-
-function assertArrayEquals(array1, array2) {
- if (array1.length != array2.length) {
- throw new crosscheck.AssertionFailure(array1.join(', ') + ' != ' + array2.join(', '));
- }
- for (var i=0; i<array1.length; i++) {
- if (array1[i] != array2[i]) {
-
- throw new crosscheck.AssertionFailure(arrayrepr(array1) + ' and ' + arrayrepr(array2)
- + ' differs at index ' + i);
- }
- }
-}
--- a/web/test/test_jscript.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/test_jscript.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,42 +1,38 @@
-from cubicweb.devtools.qunit import QUnitTestCase, unittest_main
+from cubicweb.devtools import qunit
from os import path as osp
-class JScript(QUnitTestCase):
+class JScript(qunit.QUnitTestCase):
all_js_tests = (
- ("jstests/test_utils.js", (
- "../../web/data/cubicweb.js",
- "../../web/data/cubicweb.compat.js",
- "../../web/data/cubicweb.python.js",
- "jstests/utils.js",
+ ("/static/jstests/test_utils.js", (
+ "/data/cubicweb.js",
+ "/data/cubicweb.compat.js",
+ "/data/cubicweb.python.js",
+ "/static/jstests/utils.js",
),
),
- ("jstests/test_htmlhelpers.js", (
- "../../web/data/cubicweb.js",
- "../../web/data/cubicweb.compat.js",
- "../../web/data/cubicweb.python.js",
- "../../web/data/cubicweb.htmlhelpers.js",
+ ("/static/jstests/test_htmlhelpers.js", (
+ "/data/cubicweb.js",
+ "/data/cubicweb.compat.js",
+ "/data/cubicweb.python.js",
+ "/data/cubicweb.htmlhelpers.js",
),
),
- ("jstests/test_ajax.js", (
- "../../web/data/cubicweb.python.js",
- "../../web/data/cubicweb.js",
- "../../web/data/cubicweb.compat.js",
- "../../web/data/cubicweb.htmlhelpers.js",
- "../../web/data/cubicweb.ajax.js",
- ), (
- "jstests/ajax_url0.html",
- "jstests/ajax_url1.html",
- "jstests/ajax_url2.html",
- "jstests/ajaxresult.json",
+ ("/static/jstests/test_ajax.js", (
+ "/data/cubicweb.python.js",
+ "/data/cubicweb.js",
+ "/data/cubicweb.compat.js",
+ "/data/cubicweb.htmlhelpers.js",
+ "/data/cubicweb.ajax.js",
),
),
)
if __name__ == '__main__':
- unittest_main()
+ from unittest import main
+ main()
--- a/web/test/test_views.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/test_views.py Wed Dec 16 11:23:48 2015 +0100
@@ -50,19 +50,19 @@
composite entity
"""
with self.admin_access.web_request() as req:
- rset = req.execute('CWUser X WHERE X login "admin"')
+ rset = req.execute(u'CWUser X WHERE X login "admin"')
self.view('copy', rset, req=req)
def test_sortable_js_added(self):
with self.admin_access.web_request() as req:
# sortable.js should not be included by default
rset = req.execute('CWUser X')
- self.assertNotIn('jquery.tablesorter.js', self.view('oneline', rset, req=req).source)
+ self.assertNotIn(b'jquery.tablesorter.js', self.view('oneline', rset, req=req).source)
with self.admin_access.web_request() as req:
# but should be included by the tableview
rset = req.execute('Any P,F,S LIMIT 1 WHERE P is CWUser, P firstname F, P surname S')
- self.assertIn('jquery.tablesorter.js', self.view('table', rset, req=req).source)
+ self.assertIn(b'jquery.tablesorter.js', self.view('table', rset, req=req).source)
def test_js_added_only_once(self):
with self.admin_access.web_request() as req:
@@ -70,14 +70,14 @@
self.vreg.register(SomeView)
rset = req.execute('CWUser X')
source = self.view('someview', rset, req=req).source
- self.assertEqual(source.count('spam.js'), 1)
+ self.assertEqual(source.count(b'spam.js'), 1)
def test_unrelateddivs(self):
with self.admin_access.client_cnx() as cnx:
group = cnx.create_entity('CWGroup', name=u'R&D')
cnx.commit()
with self.admin_access.web_request(relation='in_group_subject') as req:
- rset = req.execute('Any X WHERE X is CWUser, X login "admin"')
+ rset = req.execute(u'Any X WHERE X is CWUser, X login "admin"')
self.view('unrelateddivs', rset, req=req)
--- a/web/test/unittest_application.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_application.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,8 +17,11 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""unit tests for cubicweb.web.application"""
-import base64, Cookie
-import httplib
+import base64
+
+from six import text_type
+from six.moves import http_client
+from six.moves.http_cookies import SimpleCookie
from logilab.common.testlib import TestCase, unittest_main
from logilab.common.decorators import clear_cache, classproperty
@@ -179,8 +182,8 @@
def test_publish_validation_error(self):
with self.admin_access.web_request() as req:
- user = self.user(req)
- eid = unicode(user.eid)
+ user = req.user
+ eid = text_type(user.eid)
req.form = {
'eid': eid,
'__type:'+eid: 'CWUser', '_cw_entity_fields:'+eid: 'login-subject',
@@ -267,14 +270,14 @@
with self.admin_access.web_request(vid='test.ajax.error') as req:
req.ajax_request = True
page = app.handle_request(req, '')
- self.assertEqual(httplib.INTERNAL_SERVER_ERROR,
+ self.assertEqual(http_client.INTERNAL_SERVER_ERROR,
req.status_out)
def _test_cleaned(self, kwargs, injected, cleaned):
with self.admin_access.web_request(**kwargs) as req:
page = self.app_handle_request(req, 'view')
- self.assertNotIn(injected, page)
- self.assertIn(cleaned, page)
+ self.assertNotIn(injected.encode('ascii'), page)
+ self.assertIn(cleaned.encode('ascii'), page)
def test_nonregr_script_kiddies(self):
"""test against current script injection"""
@@ -314,8 +317,8 @@
self.app.handle_request(req, 'login')
self.assertEqual(401, req.status_out)
clear_cache(req, 'get_authorization')
- authstr = base64.encodestring('%s:%s' % (self.admlogin, self.admpassword))
- req.set_request_header('Authorization', 'basic %s' % authstr)
+ authstr = base64.encodestring(('%s:%s' % (self.admlogin, self.admpassword)).encode('ascii'))
+ req.set_request_header('Authorization', 'basic %s' % authstr.decode('ascii'))
self.assertAuthSuccess(req, origsession)
self.assertRaises(LogOut, self.app_handle_request, req, 'logout')
self.assertEqual(len(self.open_sessions), 0)
@@ -328,8 +331,8 @@
except Redirect as redir:
self.fail('anonymous user should get login form')
clear_cache(req, 'get_authorization')
- self.assertIn('__login', form)
- self.assertIn('__password', form)
+ self.assertIn(b'__login', form)
+ self.assertIn(b'__password', form)
self.assertFalse(req.cnx) # Mock cnx are False
req.form['__login'] = self.admlogin
req.form['__password'] = self.admpassword
@@ -361,7 +364,7 @@
def _reset_cookie(self, req):
# preparing the suite of the test
# set session id in cookie
- cookie = Cookie.SimpleCookie()
+ cookie = SimpleCookie()
sessioncookie = self.app.session_handler.session_cookie(req)
cookie[sessioncookie] = req.session.sessionid
req.set_request_header('Cookie', cookie[sessioncookie].OutputString(),
@@ -390,11 +393,11 @@
def test_http_auth_anon_allowed(self):
req, origsession = self.init_authentication('http', 'anon')
self._test_auth_anon(req)
- authstr = base64.encodestring('toto:pouet')
- req.set_request_header('Authorization', 'basic %s' % authstr)
+ authstr = base64.encodestring(b'toto:pouet')
+ req.set_request_header('Authorization', 'basic %s' % authstr.decode('ascii'))
self._test_anon_auth_fail(req)
- authstr = base64.encodestring('%s:%s' % (self.admlogin, self.admpassword))
- req.set_request_header('Authorization', 'basic %s' % authstr)
+ authstr = base64.encodestring(('%s:%s' % (self.admlogin, self.admpassword)).encode('ascii'))
+ req.set_request_header('Authorization', 'basic %s' % authstr.decode('ascii'))
self.assertAuthSuccess(req, origsession)
self.assertRaises(LogOut, self.app_handle_request, req, 'logout')
self.assertEqual(len(self.open_sessions), 0)
--- a/web/test/unittest_facet.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_facet.py Wed Dec 16 11:23:48 2015 +0100
@@ -70,8 +70,8 @@
def test_relation_optional_rel(self):
with self.admin_access.web_request() as req:
- rset = req.cnx.execute('Any X,GROUP_CONCAT(GN) GROUPBY X '
- 'WHERE X in_group G?, G name GN, NOT G name "users"')
+ rset = req.cnx.execute(u'Any X,GROUP_CONCAT(GN) GROUPBY X '
+ 'WHERE X in_group G?, G name GN, NOT G name "users"')
rqlst = rset.syntax_tree().copy()
select = rqlst.children[0]
filtered_variable, baserql = facet.init_facets(rset, select)
@@ -87,18 +87,18 @@
self.assertEqual(f.vocabulary(),
[(u'guests', guests), (u'managers', managers)])
# ensure rqlst is left unmodified
- self.assertEqual(rqlst.as_string(), "DISTINCT Any WHERE X in_group G?, G name GN, NOT G name 'users'")
+ self.assertEqual(rqlst.as_string(), 'DISTINCT Any WHERE X in_group G?, G name GN, NOT G name "users"')
#rqlst = rset.syntax_tree()
self.assertEqual(sorted(f.possible_values()),
[str(guests), str(managers)])
# ensure rqlst is left unmodified
- self.assertEqual(rqlst.as_string(), "DISTINCT Any WHERE X in_group G?, G name GN, NOT G name 'users'")
+ self.assertEqual(rqlst.as_string(), 'DISTINCT Any WHERE X in_group G?, G name GN, NOT G name "users"')
req.form[f.__regid__] = str(guests)
f.add_rql_restrictions()
# selection is cluttered because rqlst has been prepared for facet (it
# is not in real life)
self.assertEqual(f.select.as_string(),
- "DISTINCT Any WHERE X in_group G?, G name GN, NOT G name 'users', X in_group D, D eid %s" % guests)
+ 'DISTINCT Any WHERE X in_group G?, G name GN, NOT G name "users", X in_group D, D eid %s' % guests)
def test_relation_no_relation_1(self):
with self.admin_access.web_request() as req:
@@ -141,12 +141,12 @@
['guests', 'managers'])
# ensure rqlst is left unmodified
self.assertEqual(f.select.as_string(), 'DISTINCT Any WHERE X is CWUser')
- f._cw.form[f.__regid__] = 'guests'
+ f._cw.form[f.__regid__] = u'guests'
f.add_rql_restrictions()
# selection is cluttered because rqlst has been prepared for facet (it
# is not in real life)
self.assertEqual(f.select.as_string(),
- "DISTINCT Any WHERE X is CWUser, X in_group E, E name 'guests'")
+ 'DISTINCT Any WHERE X is CWUser, X in_group E, E name "guests"')
def test_hasrelation(self):
with self.admin_access.web_request() as req:
@@ -207,12 +207,12 @@
['admin', 'anon'])
# ensure rqlst is left unmodified
self.assertEqual(rqlst.as_string(), 'DISTINCT Any WHERE X is CWUser')
- req.form[f.__regid__] = 'admin'
+ req.form[f.__regid__] = u'admin'
f.add_rql_restrictions()
# selection is cluttered because rqlst has been prepared for facet (it
# is not in real life)
self.assertEqual(f.select.as_string(),
- "DISTINCT Any WHERE X is CWUser, X login 'admin'")
+ 'DISTINCT Any WHERE X is CWUser, X login "admin"')
def test_bitfield(self):
with self.admin_access.web_request() as req:
@@ -310,12 +310,12 @@
self.assertEqual(f.possible_values(), ['admin',])
# ensure rqlst is left unmodified
self.assertEqual(rqlst.as_string(), 'DISTINCT Any WHERE X is CWUser')
- req.form[f.__regid__] = 'admin'
+ req.form[f.__regid__] = u'admin'
f.add_rql_restrictions()
# selection is cluttered because rqlst has been prepared for facet (it
# is not in real life)
self.assertEqual(f.select.as_string(),
- "DISTINCT Any WHERE X is CWUser, X created_by G, G owned_by H, H login 'admin'")
+ 'DISTINCT Any WHERE X is CWUser, X created_by G, G owned_by H, H login "admin"')
def test_rql_path_check_filter_label_variable(self):
with self.admin_access.web_request() as req:
@@ -359,13 +359,13 @@
def prepareg_aggregat_rqlst(self, req):
return self.prepare_rqlst(req,
- 'Any 1, COUNT(X) WHERE X is CWUser, X creation_date XD, '
- 'X modification_date XM, Y creation_date YD, Y is CWGroup '
- 'HAVING DAY(XD)>=DAY(YD) AND DAY(XM)<=DAY(YD)', 'X',
- expected_baserql='Any 1,COUNT(X) WHERE X is CWUser, X creation_date XD, '
+ u'Any 1, COUNT(X) WHERE X is CWUser, X creation_date XD, '
+ 'X modification_date XM, Y creation_date YD, Y is CWGroup '
+ 'HAVING DAY(XD)>=DAY(YD) AND DAY(XM)<=DAY(YD)', 'X',
+ expected_baserql=u'Any 1,COUNT(X) WHERE X is CWUser, X creation_date XD, '
'X modification_date XM, Y creation_date YD, Y is CWGroup '
'HAVING DAY(XD) >= DAY(YD), DAY(XM) <= DAY(YD)',
- expected_preparedrql='DISTINCT Any WHERE X is CWUser, X creation_date XD, '
+ expected_preparedrql=u'DISTINCT Any WHERE X is CWUser, X creation_date XD, '
'X modification_date XM, Y creation_date YD, Y is CWGroup '
'HAVING DAY(XD) >= DAY(YD), DAY(XM) <= DAY(YD)')
@@ -390,13 +390,13 @@
filtered_variable=filtered_variable)
self.assertEqual(f.vocabulary(), [(u'admin', u'admin')])
self.assertEqual(f.possible_values(), ['admin'])
- req.form[f.__regid__] = 'admin'
+ req.form[f.__regid__] = u'admin'
f.add_rql_restrictions()
self.assertEqual(f.select.as_string(),
- "DISTINCT Any WHERE X is CWUser, X creation_date XD, "
- "X modification_date XM, Y creation_date YD, Y is CWGroup, "
- "X created_by G, G owned_by H, H login 'admin' "
- "HAVING DAY(XD) >= DAY(YD), DAY(XM) <= DAY(YD)")
+ 'DISTINCT Any WHERE X is CWUser, X creation_date XD, '
+ 'X modification_date XM, Y creation_date YD, Y is CWGroup, '
+ 'X created_by G, G owned_by H, H login "admin" '
+ 'HAVING DAY(XD) >= DAY(YD), DAY(XM) <= DAY(YD)')
def test_aggregat_query_attribute(self):
with self.admin_access.web_request() as req:
@@ -409,12 +409,12 @@
[(u'admin', u'admin'), (u'anon', u'anon')])
self.assertEqual(f.possible_values(),
['admin', 'anon'])
- req.form[f.__regid__] = 'admin'
+ req.form[f.__regid__] = u'admin'
f.add_rql_restrictions()
self.assertEqual(f.select.as_string(),
- "DISTINCT Any WHERE X is CWUser, X creation_date XD, "
- "X modification_date XM, Y creation_date YD, Y is CWGroup, X login 'admin' "
- "HAVING DAY(XD) >= DAY(YD), DAY(XM) <= DAY(YD)")
+ 'DISTINCT Any WHERE X is CWUser, X creation_date XD, '
+ 'X modification_date XM, Y creation_date YD, Y is CWGroup, X login "admin" '
+ 'HAVING DAY(XD) >= DAY(YD), DAY(XM) <= DAY(YD)')
if __name__ == '__main__':
from logilab.common.testlib import unittest_main
--- a/web/test/unittest_form.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_form.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,6 +21,8 @@
from xml.etree.ElementTree import fromstring
from lxml import html
+from six import text_type
+
from logilab.common.testlib import unittest_main
from cubicweb import Binary, ValidationError
@@ -65,19 +67,19 @@
t = req.create_entity('Tag', name=u'x')
form1 = self.vreg['forms'].select('edition', req, entity=t)
choices = [reid for rview, reid in form1.field_by_name('tags', 'subject', t.e_schema).choices(form1)]
- self.assertIn(unicode(b.eid), choices)
+ self.assertIn(text_type(b.eid), choices)
form2 = self.vreg['forms'].select('edition', req, entity=b)
choices = [reid for rview, reid in form2.field_by_name('tags', 'object', t.e_schema).choices(form2)]
- self.assertIn(unicode(t.eid), choices)
+ self.assertIn(text_type(t.eid), choices)
b.cw_clear_all_caches()
t.cw_clear_all_caches()
req.cnx.execute('SET X tags Y WHERE X is Tag, Y is BlogEntry')
choices = [reid for rview, reid in form1.field_by_name('tags', 'subject', t.e_schema).choices(form1)]
- self.assertIn(unicode(b.eid), choices)
+ self.assertIn(text_type(b.eid), choices)
choices = [reid for rview, reid in form2.field_by_name('tags', 'object', t.e_schema).choices(form2)]
- self.assertIn(unicode(t.eid), choices)
+ self.assertIn(text_type(t.eid), choices)
def test_form_field_choices_new_entity(self):
with self.admin_access.web_request() as req:
@@ -217,7 +219,7 @@
eidparam=True, role='subject')
with self.admin_access.web_request() as req:
file = req.create_entity('File', data_name=u"pouet.txt", data_encoding=u'UTF-8',
- data=Binary('new widgets system'))
+ data=Binary(b'new widgets system'))
form = FFForm(req, redirect_path='perdu.com', entity=file)
self.assertMultiLineEqual(self._render_entity_field(req, 'data', form),
'''<input id="data-subject:%(eid)s" name="data-subject:%(eid)s" tabindex="1" type="file" value="" />
@@ -241,7 +243,7 @@
eidparam=True, role='subject')
with self.admin_access.web_request() as req:
file = req.create_entity('File', data_name=u"pouet.txt", data_encoding=u'UTF-8',
- data=Binary('new widgets system'))
+ data=Binary(b'new widgets system'))
form = EFFForm(req, redirect_path='perdu.com', entity=file)
self.assertMultiLineEqual(self._render_entity_field(req, 'data', form),
'''<input id="data-subject:%(eid)s" name="data-subject:%(eid)s" tabindex="1" type="file" value="" />
--- a/web/test/unittest_formfields.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_formfields.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,6 +21,7 @@
from yams.constraints import StaticVocabularyConstraint, SizeConstraint
+import cubicweb
from cubicweb.devtools import TestServerConfiguration
from cubicweb.devtools.testlib import CubicWebTC
from cubicweb.web.formwidgets import PasswordInput, TextArea, Select, Radio
@@ -127,7 +128,7 @@
self.assertIsInstance(field, BooleanField)
self.assertEqual(field.required, False)
self.assertIsInstance(field.widget, Radio)
- self.assertEqual(field.vocabulary(mock(_cw=mock(_=unicode))),
+ self.assertEqual(field.vocabulary(mock(_cw=mock(_=cubicweb._))),
[(u'yes', '1'), (u'no', '')])
def test_bool_field_explicit_choices(self):
@@ -135,7 +136,7 @@
field = guess_field(schema['CWAttribute'], schema['indexed'],
choices=[(u'maybe', '1'), (u'no', '')], req=req)
self.assertIsInstance(field.widget, Radio)
- self.assertEqual(field.vocabulary(mock(req=mock(_=unicode))),
+ self.assertEqual(field.vocabulary(mock(req=mock(_=cubicweb._))),
[(u'maybe', '1'), (u'no', '')])
--- a/web/test/unittest_formwidgets.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_formwidgets.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,23 +17,18 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""unittests for cw.web.formwidgets"""
-from logilab.common.testlib import TestCase, unittest_main, mock_object as mock
-
-from cubicweb.devtools import TestServerConfiguration, fake
-from cubicweb.web import formwidgets, formfields
-
-from cubes.file.entities import File
+from logilab.common.testlib import unittest_main, mock_object as mock
-def setUpModule(*args):
- global schema
- config = TestServerConfiguration('data', apphome=WidgetsTC.datadir)
- config.bootstrap_cubes()
- schema = config.load_schema()
+from cubicweb.devtools import fake
+from cubicweb.devtools.testlib import CubicWebTC
+from cubicweb.web import formwidgets, formfields
+from cubicweb.web.views.forms import FieldsForm
-class WidgetsTC(TestCase):
+
+class WidgetsTC(CubicWebTC):
def test_editableurl_widget(self):
- field = formfields.guess_field(schema['Bookmark'], schema['path'])
+ field = formfields.guess_field(self.schema['Bookmark'], self.schema['path'])
widget = formwidgets.EditableURLWidget()
req = fake.FakeRequest(form={'path-subjectfqs:A': 'param=value&vid=view'})
form = mock(_cw=req, formvalues={}, edited_entity=mock(eid='A'))
@@ -41,7 +36,7 @@
'?param=value%26vid%3Dview')
def test_bitselect_widget(self):
- field = formfields.guess_field(schema['CWAttribute'], schema['ordernum'])
+ field = formfields.guess_field(self.schema['CWAttribute'], self.schema['ordernum'])
field.choices = [('un', '1',), ('deux', '2',)]
widget = formwidgets.BitSelect(settabindex=False)
req = fake.FakeRequest(form={'ordernum-subject:A': ['1', '2']})
@@ -56,5 +51,21 @@
self.assertEqual(widget.process_field_data(form, field),
3)
+ def test_xml_escape_checkbox(self):
+ class TestForm(FieldsForm):
+ bool = formfields.BooleanField(ignore_req_params=True,
+ choices=[('python >> others', '1')],
+ widget=formwidgets.CheckBox())
+ with self.admin_access.web_request() as req:
+ form = TestForm(req, None)
+ form.build_context()
+ field = form.field_by_name('bool')
+ widget = field.widget
+ self.assertMultiLineEqual(widget._render(form, field, None),
+ '<label><input id="bool" name="bool" tabindex="1" '
+ 'type="checkbox" value="1" /> '
+ 'python >> others</label>')
+
+
if __name__ == '__main__':
unittest_main()
--- a/web/test/unittest_http.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_http.py Wed Dec 16 11:23:48 2015 +0100
@@ -253,46 +253,6 @@
req = _test_cache(hin, hout, method='POST')
self.assertCache(412, req.status_out, 'not modifier HEAD verb')
- @tag('expires')
- def test_expires_added(self):
- #: Check that Expires header is added:
- #: - when the page is modified
- #: - when none was already present
- hin = [('if-none-match', 'babar'),
- ]
- hout = [('etag', 'rhino/really-not-babar'),
- ]
- req = _test_cache(hin, hout)
- self.assertCache(None, req.status_out, 'modifier HEAD verb')
- value = req.headers_out.getHeader('expires')
- self.assertIsNotNone(value)
-
- @tag('expires')
- def test_expires_not_added(self):
- #: Check that Expires header is not added if NOT-MODIFIED
- hin = [('if-none-match', 'babar'),
- ]
- hout = [('etag', 'babar'),
- ]
- req = _test_cache(hin, hout)
- self.assertCache(304, req.status_out, 'not modifier HEAD verb')
- value = req.headers_out.getHeader('expires')
- self.assertIsNone(value)
-
- @tag('expires')
- def test_expires_no_overwrite(self):
- #: Check that cache does not overwrite existing Expires header
- hin = [('if-none-match', 'babar'),
- ]
- DATE = 'Sat, 13 Apr 2012 14:39:32 GM'
- hout = [('etag', 'rhino/really-not-babar'),
- ('expires', DATE),
- ]
- req = _test_cache(hin, hout)
- self.assertCache(None, req.status_out, 'not modifier HEAD verb')
- value = req.headers_out.getRawHeaders('expires')
- self.assertEqual(value, [DATE])
-
alloworig = 'access-control-allow-origin'
allowmethods = 'access-control-allow-methods'
--- a/web/test/unittest_idownloadable.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_idownloadable.py Wed Dec 16 11:23:48 2015 +0100
@@ -42,7 +42,7 @@
return self.entity.name() + '.txt'
def download_data(self):
- return 'Babar is not dead!'
+ return b'Babar is not dead!'
class BrokenIDownloadableGroup(IDownloadableUser):
@@ -72,7 +72,7 @@
get('content-disposition'))
self.assertEqual(['text/plain;charset=ascii'],
get('content-type'))
- self.assertEqual('Babar is not dead!', data)
+ self.assertEqual(b'Babar is not dead!', data)
def test_header_with_space(self):
with self.admin_access.web_request() as req:
@@ -87,13 +87,13 @@
get('content-disposition'))
self.assertEqual(['text/plain;charset=ascii'],
get('content-type'))
- self.assertEqual('Babar is not dead!', data)
+ self.assertEqual(b'Babar is not dead!', data)
def test_header_with_space_and_comma(self):
with self.admin_access.web_request() as req:
- self.create_user(req, login=ur'c " l\ a', password='babar')
+ self.create_user(req, login=u'c " l\\ a', password='babar')
req.cnx.commit()
- with self.new_access(ur'c " l\ a').web_request() as req:
+ with self.new_access(u'c " l\\ a').web_request() as req:
req.form['vid'] = 'download'
req.form['eid'] = str(req.user.eid)
data = self.ctrl_publish(req,'view')
@@ -102,7 +102,7 @@
get('content-disposition'))
self.assertEqual(['text/plain;charset=ascii'],
get('content-type'))
- self.assertEqual('Babar is not dead!', data)
+ self.assertEqual(b'Babar is not dead!', data)
def test_header_unicode_filename(self):
with self.admin_access.web_request() as req:
--- a/web/test/unittest_magicsearch.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_magicsearch.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,6 +21,8 @@
import sys
from contextlib import contextmanager
+from six.moves import range
+
from logilab.common.testlib import TestCase, unittest_main
from rql import BadRQLQuery, RQLSyntaxError
@@ -62,19 +64,19 @@
def test_basic_translations(self):
"""tests basic translations (no ambiguities)"""
with self.proc() as proc:
- rql = "Any C WHERE C is Adresse, P adel C, C adresse 'Logilab'"
+ rql = u"Any C WHERE C is Adresse, P adel C, C adresse 'Logilab'"
rql, = proc.preprocess_query(rql)
- self.assertEqual(rql, "Any C WHERE C is EmailAddress, P use_email C, C address 'Logilab'")
+ self.assertEqual(rql, 'Any C WHERE C is EmailAddress, P use_email C, C address "Logilab"')
def test_ambiguous_translations(self):
"""tests possibly ambiguous translations"""
with self.proc() as proc:
- rql = "Any P WHERE P adel C, C is EmailAddress, C nom 'Logilab'"
+ rql = u"Any P WHERE P adel C, C is EmailAddress, C nom 'Logilab'"
rql, = proc.preprocess_query(rql)
- self.assertEqual(rql, "Any P WHERE P use_email C, C is EmailAddress, C alias 'Logilab'")
- rql = "Any P WHERE P is Utilisateur, P adel C, P nom 'Smith'"
+ self.assertEqual(rql, 'Any P WHERE P use_email C, C is EmailAddress, C alias "Logilab"')
+ rql = u"Any P WHERE P is Utilisateur, P adel C, P nom 'Smith'"
rql, = proc.preprocess_query(rql)
- self.assertEqual(rql, "Any P WHERE P is CWUser, P use_email C, P surname 'Smith'")
+ self.assertEqual(rql, 'Any P WHERE P is CWUser, P use_email C, P surname "Smith"')
class QSPreProcessorTC(CubicWebTC):
@@ -330,7 +332,7 @@
# suggestions should contain any possible value for
# a given attribute (limited to 10)
with self.admin_access.web_request() as req:
- for i in xrange(15):
+ for i in range(15):
req.create_entity('Personne', nom=u'n%s' % i, prenom=u'p%s' % i)
req.cnx.commit()
self.assertListEqual(['Any X WHERE X is Personne, X nom "n0"',
--- a/web/test/unittest_propertysheet.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_propertysheet.py Wed Dec 16 11:23:48 2015 +0100
@@ -49,19 +49,14 @@
'a {bgcolor: #FFFFFF; size: 1%;}')
self.assertEqual(ps.process_resource(DATADIR, 'pouet.css'),
self.cachedir)
- self.assertIn('pouet.css', ps._cache)
self.assertFalse(ps.need_reload())
os.utime(self.data('sheet1.py'), None)
- self.assertIn('pouet.css', ps._cache)
self.assertTrue(ps.need_reload())
- self.assertIn('pouet.css', ps._cache)
ps.reload()
- self.assertNotIn('pouet.css', ps._cache)
self.assertFalse(ps.need_reload())
ps.process_resource(DATADIR, 'pouet.css') # put in cache
os.utime(self.data('pouet.css'), None)
self.assertFalse(ps.need_reload())
- self.assertNotIn('pouet.css', ps._cache)
if __name__ == '__main__':
--- a/web/test/unittest_urlpublisher.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_urlpublisher.py Wed Dec 16 11:23:48 2015 +0100
@@ -25,7 +25,7 @@
from cubicweb.rset import ResultSet
from cubicweb.devtools.testlib import CubicWebTC
from cubicweb.devtools.fake import FakeRequest
-from cubicweb.web import NotFound, Redirect
+from cubicweb.web import NotFound, Redirect, views
from cubicweb.web.views.urlrewrite import SimpleReqRewriter
@@ -69,6 +69,7 @@
self.assertEqual("Any X,AA,AB ORDERBY AB WHERE X is_instance_of CWEType, "
"X modification_date AA, X name AB",
rset.printable_rql())
+ self.assertEqual(req.form['vid'], 'sameetypelist')
def test_rest_path_by_attr(self):
with self.admin_access.web_request() as req:
@@ -91,10 +92,11 @@
'X firstname AA, X login AB, X modification_date AC, '
'X surname AD, X login "admin"',
rset.printable_rql())
+ self.assertEqual(req.form['vid'], 'primary')
def test_rest_path_eid(self):
with self.admin_access.web_request() as req:
- ctrl, rset = self.process(req, 'cwuser/eid/%s' % self.user(req).eid)
+ ctrl, rset = self.process(req, 'cwuser/eid/%s' % req.user.eid)
self.assertEqual(ctrl, 'view')
self.assertEqual(len(rset), 1)
self.assertEqual(rset.description[0][0], 'CWUser')
@@ -125,6 +127,15 @@
'X title "hell\'o"',
rset.printable_rql())
+ def test_rest_path_use_vid_from_rset(self):
+ with self.admin_access.web_request(headers={'Accept': 'application/rdf+xml'}) as req:
+ views.VID_BY_MIMETYPE['application/rdf+xml'] = 'rdf'
+ try:
+ ctrl, rset = self.process(req, 'CWEType')
+ finally:
+ views.VID_BY_MIMETYPE.pop('application/rdf+xml')
+ self.assertEqual(req.form['vid'], 'rdf')
+
def test_rest_path_errors(self):
with self.admin_access.web_request() as req:
self.assertRaises(NotFound, self.process, req, 'CWUser/eid/30000')
@@ -141,25 +152,24 @@
self.assertRaises(NotFound, self.process, req, '1/non_action')
self.assertRaises(NotFound, self.process, req, 'CWUser/login/admin/non_action')
-
def test_regexp_path(self):
"""tests the regexp path resolution"""
with self.admin_access.web_request() as req:
ctrl, rset = self.process(req, 'add/Task')
self.assertEqual(ctrl, 'view')
self.assertEqual(rset, None)
- self.assertEqual(req.form, {'etype' : "Task", 'vid' : "creation"})
+ self.assertEqual(req.form, {'etype': "Task", 'vid': "creation"})
self.assertRaises(NotFound, self.process, req, 'add/foo/bar')
def test_nonascii_path(self):
oldrules = SimpleReqRewriter.rules
- SimpleReqRewriter.rules = [(re.compile('/\w+', re.U), dict(vid='foo')),]
+ SimpleReqRewriter.rules = [(re.compile('/\w+', re.U), dict(vid='foo'))]
with self.admin_access.web_request() as req:
try:
path = str(FakeRequest().url_quote(u'été'))
ctrl, rset = self.process(req, path)
self.assertEqual(rset, None)
- self.assertEqual(req.form, {'vid' : "foo"})
+ self.assertEqual(req.form, {'vid': "foo"})
finally:
SimpleReqRewriter.rules = oldrules
--- a/web/test/unittest_urlrewrite.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_urlrewrite.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,6 +16,8 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+from six import text_type
+
from logilab.common import tempattr
from cubicweb.devtools.testlib import CubicWebTC
@@ -137,8 +139,8 @@
rgx_action(r'Any X WHERE X surname %(sn)s, '
'X firstname %(fn)s',
argsgroups=('sn', 'fn'),
- transforms={'sn' : unicode.capitalize,
- 'fn' : unicode.lower,})),
+ transforms={'sn' : text_type.capitalize,
+ 'fn' : text_type.lower,})),
]
with self.admin_access.web_request() as req:
rewriter = TestSchemaBasedRewriter(req)
--- a/web/test/unittest_views_basecontrollers.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_views_basecontrollers.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,22 +17,18 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""cubicweb.web.views.basecontrollers unit tests"""
-from urlparse import urlsplit, urlunsplit, urljoin
-# parse_qs is deprecated in cgi and has been moved to urlparse in Python 2.6
-try:
- from urlparse import parse_qs as url_parse_query
-except ImportError:
- from cgi import parse_qs as url_parse_query
+from six import text_type
+from six.moves.urllib.parse import urlsplit, urlunsplit, urljoin, parse_qs
import lxml
from logilab.common.testlib import unittest_main
-
from logilab.common.decorators import monkeypatch
from cubicweb import Binary, NoSelectableObject, ValidationError
from cubicweb.schema import RRQLExpression
from cubicweb.devtools.testlib import CubicWebTC
+from cubicweb.devtools.webtest import CubicWebTestTC
from cubicweb.utils import json_dumps
from cubicweb.uilib import rql_for_eid
from cubicweb.web import Redirect, RemoteCallFailed
@@ -45,6 +41,19 @@
from cubicweb.server.hook import Hook, Operation
from cubicweb.predicates import is_instance
+
+class ViewControllerTC(CubicWebTestTC):
+ def test_view_ctrl_with_valid_cache_headers(self):
+ resp = self.webapp.get('/manage')
+ self.assertEqual(resp.etag, 'manage/guests')
+ self.assertEqual(resp.status_code, 200)
+ cache_headers = {'if-modified-since': resp.headers['Last-Modified'],
+ 'if-none-match': resp.etag}
+ resp = self.webapp.get('/manage', headers=cache_headers)
+ self.assertEqual(resp.status_code, 304)
+ self.assertEqual(len(resp.body), 0)
+
+
def req_form(user):
return {'eid': [str(user.eid)],
'_cw_entity_fields:%s' % user.eid: '_cw_generic_field',
@@ -82,7 +91,7 @@
}
with self.assertRaises(ValidationError) as cm:
self.ctrl_publish(req)
- cm.exception.translate(unicode)
+ cm.exception.translate(text_type)
self.assertEqual({'login-subject': 'the value "admin" is already used, use another one'},
cm.exception.errors)
@@ -136,12 +145,12 @@
user = req.user
groupeids = [eid for eid, in req.execute('CWGroup G WHERE G name '
'in ("managers", "users")')]
- groups = [unicode(eid) for eid in groupeids]
- eid = unicode(user.eid)
+ groups = [text_type(eid) for eid in groupeids]
+ eid = text_type(user.eid)
req.form = {
'eid': eid, '__type:'+eid: 'CWUser',
'_cw_entity_fields:'+eid: 'login-subject,firstname-subject,surname-subject,in_group-subject',
- 'login-subject:'+eid: unicode(user.login),
+ 'login-subject:'+eid: text_type(user.login),
'surname-subject:'+eid: u'Th\xe9nault',
'firstname-subject:'+eid: u'Sylvain',
'in_group-subject:'+eid: groups,
@@ -159,7 +168,7 @@
self.create_user(cnx, u'user')
cnx.commit()
with self.new_access(u'user').web_request() as req:
- eid = unicode(req.user.eid)
+ eid = text_type(req.user.eid)
req.form = {
'eid': eid, '__maineid' : eid,
'__type:'+eid: 'CWUser',
@@ -179,12 +188,12 @@
with self.admin_access.web_request() as req:
user = req.user
groupeids = [g.eid for g in user.in_group]
- eid = unicode(user.eid)
+ eid = text_type(user.eid)
req.form = {
'eid': eid,
'__type:'+eid: 'CWUser',
'_cw_entity_fields:'+eid: 'login-subject,firstname-subject,surname-subject',
- 'login-subject:'+eid: unicode(user.login),
+ 'login-subject:'+eid: text_type(user.login),
'firstname-subject:'+eid: u'Th\xe9nault',
'surname-subject:'+eid: u'Sylvain',
}
@@ -207,7 +216,7 @@
'login-subject:X': u'adim',
'upassword-subject:X': u'toto', 'upassword-subject-confirm:X': u'toto',
'surname-subject:X': u'Di Mascio',
- 'in_group-subject:X': unicode(gueid),
+ 'in_group-subject:X': text_type(gueid),
'__type:Y': 'EmailAddress',
'_cw_entity_fields:Y': 'address-subject,use_email-object',
@@ -231,7 +240,7 @@
'__type:Y': 'File',
'_cw_entity_fields:Y': 'data-subject,described_by_test-object',
- 'data-subject:Y': (u'coucou.txt', Binary('coucou')),
+ 'data-subject:Y': (u'coucou.txt', Binary(b'coucou')),
'described_by_test-object:Y': 'X',
}
path, _params = self.expect_redirect_handle_request(req, 'edit')
@@ -256,7 +265,7 @@
'__type:Y': 'File',
'_cw_entity_fields:Y': 'data-subject',
- 'data-subject:Y': (u'coucou.txt', Binary('coucou')),
+ 'data-subject:Y': (u'coucou.txt', Binary(b'coucou')),
}
path, _params = self.expect_redirect_handle_request(req, 'edit')
self.assertTrue(path.startswith('salesterm/'), path)
@@ -274,7 +283,7 @@
# non regression test for #3120495. Without the fix, leads to
# "unhashable type: 'list'" error
with self.admin_access.web_request() as req:
- cwrelation = unicode(req.execute('CWEType X WHERE X name "CWSource"')[0][0])
+ cwrelation = text_type(req.execute('CWEType X WHERE X name "CWSource"')[0][0])
req.form = {'eid': [cwrelation], '__maineid' : cwrelation,
'__type:'+cwrelation: 'CWEType',
@@ -287,7 +296,7 @@
def test_edit_multiple_linked(self):
with self.admin_access.web_request() as req:
- peid = unicode(self.create_user(req, u'adim').eid)
+ peid = text_type(self.create_user(req, u'adim').eid)
req.form = {'eid': [peid, 'Y'], '__maineid': peid,
'__type:'+peid: u'CWUser',
@@ -307,7 +316,7 @@
self.assertEqual(email.address, 'dima@logilab.fr')
# with self.admin_access.web_request() as req:
- emaileid = unicode(email.eid)
+ emaileid = text_type(email.eid)
req.form = {'eid': [peid, emaileid],
'__type:'+peid: u'CWUser',
@@ -329,7 +338,7 @@
with self.admin_access.web_request() as req:
user = req.user
req.form = {'eid': 'X',
- '__cloned_eid:X': unicode(user.eid), '__type:X': 'CWUser',
+ '__cloned_eid:X': text_type(user.eid), '__type:X': 'CWUser',
'_cw_entity_fields:X': 'login-subject,upassword-subject',
'login-subject:X': u'toto',
'upassword-subject:X': u'toto',
@@ -338,7 +347,7 @@
self.ctrl_publish(req)
self.assertEqual({'upassword-subject': u'password and confirmation don\'t match'},
cm.exception.errors)
- req.form = {'__cloned_eid:X': unicode(user.eid),
+ req.form = {'__cloned_eid:X': text_type(user.eid),
'eid': 'X', '__type:X': 'CWUser',
'_cw_entity_fields:X': 'login-subject,upassword-subject',
'login-subject:X': u'toto',
@@ -354,7 +363,7 @@
def test_interval_bound_constraint_success(self):
with self.admin_access.repo_cnx() as cnx:
feid = cnx.execute('INSERT File X: X data_name "toto.txt", X data %(data)s',
- {'data': Binary('yo')})[0][0]
+ {'data': Binary(b'yo')})[0][0]
cnx.commit()
with self.admin_access.web_request(rollbackfirst=True) as req:
@@ -362,11 +371,11 @@
'__type:X': 'Salesterm',
'_cw_entity_fields:X': 'amount-subject,described_by_test-subject',
'amount-subject:X': u'-10',
- 'described_by_test-subject:X': unicode(feid),
+ 'described_by_test-subject:X': text_type(feid),
}
with self.assertRaises(ValidationError) as cm:
self.ctrl_publish(req)
- cm.exception.translate(unicode)
+ cm.exception.translate(text_type)
self.assertEqual({'amount-subject': 'value -10 must be >= 0'},
cm.exception.errors)
@@ -375,11 +384,11 @@
'__type:X': 'Salesterm',
'_cw_entity_fields:X': 'amount-subject,described_by_test-subject',
'amount-subject:X': u'110',
- 'described_by_test-subject:X': unicode(feid),
+ 'described_by_test-subject:X': text_type(feid),
}
with self.assertRaises(ValidationError) as cm:
self.ctrl_publish(req)
- cm.exception.translate(unicode)
+ cm.exception.translate(text_type)
self.assertEqual(cm.exception.errors, {'amount-subject': 'value 110 must be <= 100'})
with self.admin_access.web_request(rollbackfirst=True) as req:
@@ -387,7 +396,7 @@
'__type:X': 'Salesterm',
'_cw_entity_fields:X': 'amount-subject,described_by_test-subject',
'amount-subject:X': u'10',
- 'described_by_test-subject:X': unicode(feid),
+ 'described_by_test-subject:X': text_type(feid),
}
self.expect_redirect_handle_request(req, 'edit')
# should be redirected on the created
@@ -400,31 +409,31 @@
constrained attributes"""
with self.admin_access.repo_cnx() as cnx:
feid = cnx.execute('INSERT File X: X data_name "toto.txt", X data %(data)s',
- {'data': Binary('yo')})[0][0]
+ {'data': Binary(b'yo')})[0][0]
seid = cnx.create_entity('Salesterm', amount=0, described_by_test=feid).eid
cnx.commit()
# ensure a value that violate a constraint is properly detected
with self.admin_access.web_request(rollbackfirst=True) as req:
- req.form = {'eid': [unicode(seid)],
+ req.form = {'eid': [text_type(seid)],
'__type:%s'%seid: 'Salesterm',
'_cw_entity_fields:%s'%seid: 'amount-subject',
'amount-subject:%s'%seid: u'-10',
}
self.assertMultiLineEqual('''<script type="text/javascript">
window.parent.handleFormValidationResponse('entityForm', null, null, [false, [%s, {"amount-subject": "value -10 must be >= 0"}], null], null);
-</script>'''%seid, self.ctrl_publish(req, 'validateform'))
+</script>'''%seid, self.ctrl_publish(req, 'validateform').decode('ascii'))
# ensure a value that comply a constraint is properly processed
with self.admin_access.web_request(rollbackfirst=True) as req:
- req.form = {'eid': [unicode(seid)],
+ req.form = {'eid': [text_type(seid)],
'__type:%s'%seid: 'Salesterm',
'_cw_entity_fields:%s'%seid: 'amount-subject',
'amount-subject:%s'%seid: u'20',
}
self.assertMultiLineEqual('''<script type="text/javascript">
window.parent.handleFormValidationResponse('entityForm', null, null, [true, "http://testing.fr/cubicweb/view", null], null);
-</script>''', self.ctrl_publish(req, 'validateform'))
+</script>''', self.ctrl_publish(req, 'validateform').decode('ascii'))
self.assertEqual(20, req.execute('Any V WHERE X amount V, X eid %(eid)s',
{'eid': seid})[0][0])
@@ -433,7 +442,7 @@
'__type:X': 'Salesterm',
'_cw_entity_fields:X': 'amount-subject,described_by_test-subject',
'amount-subject:X': u'0',
- 'described_by_test-subject:X': unicode(feid),
+ 'described_by_test-subject:X': text_type(feid),
}
# ensure a value that is modified in an operation on a modify
@@ -452,11 +461,11 @@
with self.temporary_appobjects(ValidationErrorInOpAfterHook):
self.assertMultiLineEqual('''<script type="text/javascript">
window.parent.handleFormValidationResponse('entityForm', null, null, [false, ["X", {"amount-subject": "value -10 must be >= 0"}], null], null);
-</script>''', self.ctrl_publish(req, 'validateform'))
+</script>''', self.ctrl_publish(req, 'validateform').decode('ascii'))
self.assertMultiLineEqual('''<script type="text/javascript">
window.parent.handleFormValidationResponse('entityForm', null, null, [true, "http://testing.fr/cubicweb/view", null], null);
-</script>''', self.ctrl_publish(req, 'validateform'))
+</script>''', self.ctrl_publish(req, 'validateform').decode('ascii'))
def test_req_pending_insert(self):
"""make sure req's pending insertions are taken into account"""
@@ -541,7 +550,7 @@
def test_redirect_delete_button(self):
with self.admin_access.web_request() as req:
eid = req.create_entity('BlogEntry', title=u'hop', content=u'hop').eid
- req.form = {'eid': unicode(eid), '__type:%s'%eid: 'BlogEntry',
+ req.form = {'eid': text_type(eid), '__type:%s'%eid: 'BlogEntry',
'__action_delete': ''}
path, params = self.expect_redirect_handle_request(req, 'edit')
self.assertEqual(path, 'blogentry')
@@ -550,14 +559,14 @@
req.execute('SET X use_email E WHERE E eid %(e)s, X eid %(x)s',
{'x': req.user.eid, 'e': eid})
req.cnx.commit()
- req.form = {'eid': unicode(eid), '__type:%s'%eid: 'EmailAddress',
+ req.form = {'eid': text_type(eid), '__type:%s'%eid: 'EmailAddress',
'__action_delete': ''}
path, params = self.expect_redirect_handle_request(req, 'edit')
self.assertEqual(path, 'cwuser/admin')
self.assertIn('_cwmsgid', params)
eid1 = req.create_entity('BlogEntry', title=u'hop', content=u'hop').eid
eid2 = req.create_entity('EmailAddress', address=u'hop@logilab.fr').eid
- req.form = {'eid': [unicode(eid1), unicode(eid2)],
+ req.form = {'eid': [text_type(eid1), text_type(eid2)],
'__type:%s'%eid1: 'BlogEntry',
'__type:%s'%eid2: 'EmailAddress',
'__action_delete': ''}
@@ -607,13 +616,13 @@
groupeids = sorted(eid
for eid, in req.execute('CWGroup G '
'WHERE G name in ("managers", "users")'))
- groups = [unicode(eid) for eid in groupeids]
+ groups = [text_type(eid) for eid in groupeids]
cwetypeeid = req.execute('CWEType X WHERE X name "CWEType"')[0][0]
- basegroups = [unicode(eid)
+ basegroups = [text_type(eid)
for eid, in req.execute('CWGroup G '
'WHERE X read_permission G, X eid %(x)s',
{'x': cwetypeeid})]
- cwetypeeid = unicode(cwetypeeid)
+ cwetypeeid = text_type(cwetypeeid)
req.form = {
'eid': cwetypeeid,
'__type:'+cwetypeeid: 'CWEType',
@@ -662,7 +671,7 @@
'_cw_entity_fields:X': 'login-subject,upassword-subject,in_group-subject',
'login-subject:X': u'adim',
'upassword-subject:X': u'toto', 'upassword-subject-confirm:X': u'toto',
- 'in_group-subject:X': `gueid`,
+ 'in_group-subject:X': repr(gueid),
'__type:Y': 'EmailAddress',
'_cw_entity_fields:Y': 'address-subject,alias-subject,use_email-object',
@@ -737,7 +746,7 @@
'__type:Y': 'File',
'_cw_entity_fields:Y': 'data-subject',
- 'data-subject:Y': (u'coucou.txt', Binary('coucou')),
+ 'data-subject:Y': (u'coucou.txt', Binary(b'coucou')),
}
values_by_eid = dict((eid, req.extract_entity_params(eid, minparams=2))
for eid in req.edited_eids())
@@ -783,7 +792,7 @@
rset = self.john.as_rset()
rset.req = req
source = ctrl.publish()
- self.assertTrue(source.startswith('<div>'))
+ self.assertTrue(source.startswith(b'<div>'))
# def test_json_exec(self):
# rql = 'Any T,N WHERE T is Tag, T name N'
@@ -824,7 +833,7 @@
rset.req = req
source = ctrl.publish()
# maydel jscall
- self.assertIn('ajaxBoxRemoveLinkedEntity', source)
+ self.assertIn(b'ajaxBoxRemoveLinkedEntity', source)
def test_pending_insertion(self):
with self.remote_calling('add_pending_inserts', [['12', 'tags', '13']]) as (_, req):
@@ -887,16 +896,16 @@
# silly tests
def test_external_resource(self):
with self.remote_calling('external_resource', 'RSS_LOGO') as (res, _):
- self.assertEqual(json_dumps(self.config.uiprops['RSS_LOGO']),
+ self.assertEqual(json_dumps(self.config.uiprops['RSS_LOGO']).encode('ascii'),
res)
def test_i18n(self):
with self.remote_calling('i18n', ['bimboom']) as (res, _):
- self.assertEqual(json_dumps(['bimboom']), res)
+ self.assertEqual(json_dumps(['bimboom']).encode('ascii'), res)
def test_format_date(self):
with self.remote_calling('format_date', '2007-01-01 12:00:00') as (res, _):
- self.assertEqual(json_dumps('2007/01/01'), res)
+ self.assertEqual(json_dumps('2007/01/01').encode('ascii'), res)
def test_ajaxfunc_noparameter(self):
@ajaxfunc
@@ -968,7 +977,7 @@
def js_foo(self):
return u'hello'
with self.remote_calling('foo') as (res, _):
- self.assertEqual(res, u'hello')
+ self.assertEqual(res, b'hello')
def test_monkeypatch_jsoncontroller_xhtmlize(self):
with self.assertRaises(RemoteCallFailed):
@@ -979,7 +988,7 @@
def js_foo(self):
return u'hello'
with self.remote_calling('foo') as (res, _):
- self.assertEqual(u'<div>hello</div>', res)
+ self.assertEqual(b'<div>hello</div>', res)
def test_monkeypatch_jsoncontroller_jsonize(self):
with self.assertRaises(RemoteCallFailed):
@@ -990,7 +999,7 @@
def js_foo(self):
return 12
with self.remote_calling('foo') as (res, _):
- self.assertEqual(res, '12')
+ self.assertEqual(res, b'12')
def test_monkeypatch_jsoncontroller_stdfunc(self):
@monkeypatch(JSonController)
@@ -998,7 +1007,7 @@
def js_reledit_form(self):
return 12
with self.remote_calling('reledit_form') as (res, _):
- self.assertEqual(res, '12')
+ self.assertEqual(res, b'12')
class UndoControllerTC(CubicWebTC):
@@ -1042,7 +1051,7 @@
"""
with self.admin_access.web_request() as req:
scheme, netloc, path, query, fragment = urlsplit(url)
- query_dict = url_parse_query(query)
+ query_dict = parse_qs(query)
expected_url = urljoin(req.base_url(), expected_path)
self.assertEqual( urlunsplit((scheme, netloc, path, None, None)), expected_url)
@@ -1058,17 +1067,6 @@
result = controller.publish(rset=None)
self.assertURLPath(cm.exception.location, rpath)
- def test_redirect_default(self):
- with self.admin_access.web_request() as req:
- txuuid = self.txuuid_toto_email
- req.form['txuuid'] = txuuid
- req.session.data['breadcrumbs'] = [ urljoin(req.base_url(), path)
- for path in ('tata', 'toto',)]
- controller = self.vreg['controllers'].select('undo', req)
- with self.assertRaises(Redirect) as cm:
- result = controller.publish(rset=None)
- self.assertURLPath(cm.exception.location, 'toto')
-
class LoginControllerTC(CubicWebTC):
--- a/web/test/unittest_views_baseviews.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_views_baseviews.py Wed Dec 16 11:23:48 2015 +0100
@@ -129,8 +129,8 @@
source_lines = [line.strip()
for line in html_source.splitlines(False)
if line.strip()]
- self.assertListEqual(['<!DOCTYPE html>',
- '<html xmlns:cubicweb="http://www.cubicweb.org" lang="en">'],
+ self.assertListEqual([b'<!DOCTYPE html>',
+ b'<html xmlns:cubicweb="http://www.cubicweb.org" lang="en">'],
source_lines[:2])
def test_set_doctype_no_reset_xmldecl(self):
@@ -151,9 +151,9 @@
source_lines = [line.strip()
for line in html_source.splitlines(False)
if line.strip()]
- self.assertListEqual([html_doctype,
- '<html xmlns:cubicweb="http://www.cubicweb.org" lang="cz">',
- '<head>'],
+ self.assertListEqual([html_doctype.encode('ascii'),
+ b'<html xmlns:cubicweb="http://www.cubicweb.org" lang="cz">',
+ b'<head>'],
source_lines[:3])
if __name__ == '__main__':
--- a/web/test/unittest_views_csv.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_views_csv.py Wed Dec 16 11:23:48 2015 +0100
@@ -30,19 +30,19 @@
self.assertEqual(req.headers_out.getRawHeaders('content-type'),
['text/comma-separated-values;charset=UTF-8'])
expected_data = "String;COUNT(CWUser)\nguests;1\nmanagers;1"
- self.assertMultiLineEqual(expected_data, data)
+ self.assertMultiLineEqual(expected_data, data.decode('utf-8'))
def test_csvexport_on_empty_rset(self):
"""Should return the CSV header.
"""
with self.admin_access.web_request() as req:
- rset = req.execute('Any GN,COUNT(X) GROUPBY GN ORDERBY GN '
- 'WHERE X in_group G, G name GN, X login "Miles"')
+ rset = req.execute(u'Any GN,COUNT(X) GROUPBY GN ORDERBY GN '
+ 'WHERE X in_group G, G name GN, X login "Miles"')
data = self.view('csvexport', rset, req=req)
self.assertEqual(req.headers_out.getRawHeaders('content-type'),
['text/comma-separated-values;charset=UTF-8'])
expected_data = "String;COUNT(CWUser)"
- self.assertMultiLineEqual(expected_data, data)
+ self.assertMultiLineEqual(expected_data, data.decode('utf-8'))
if __name__ == '__main__':
--- a/web/test/unittest_views_editforms.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_views_editforms.py Wed Dec 16 11:23:48 2015 +0100
@@ -255,4 +255,3 @@
if __name__ == '__main__':
unittest_main()
-
--- a/web/test/unittest_views_errorform.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_views_errorform.py Wed Dec 16 11:23:48 2015 +0100
@@ -50,8 +50,8 @@
req.data['excinfo'] = sys.exc_info()
req.data['ex'] = e
html = self.view('error', req=req)
- self.failUnless(re.search(r'^<input name="__signature" type="hidden" '
- 'value="[0-9a-f]{32}" />$',
+ self.assertTrue(re.search(b'^<input name="__signature" type="hidden" '
+ b'value="[0-9a-f]{32}" />$',
html.source, re.M))
--- a/web/test/unittest_views_json.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_views_json.py Wed Dec 16 11:23:48 2015 +0100
@@ -16,12 +16,14 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+from six import binary_type
+
from cubicweb.devtools.testlib import CubicWebTC
class JsonViewsTC(CubicWebTC):
anonymize = True
- res_jsonp_data = '[["guests", 1]]'
+ res_jsonp_data = b'[["guests", 1]]'
def setUp(self):
super(JsonViewsTC, self).setUp()
@@ -29,14 +31,15 @@
def test_json_rsetexport(self):
with self.admin_access.web_request() as req:
- rset = req.execute('Any GN,COUNT(X) GROUPBY GN ORDERBY GN WHERE X in_group G, G name GN')
+ rset = req.execute(
+ 'Any GN,COUNT(X) GROUPBY GN ORDERBY GN WHERE X in_group G, G name GN')
data = self.view('jsonexport', rset, req=req)
self.assertEqual(req.headers_out.getRawHeaders('content-type'), ['application/json'])
self.assertListEqual(data, [["guests", 1], ["managers", 1]])
def test_json_rsetexport_empty_rset(self):
with self.admin_access.web_request() as req:
- rset = req.execute('Any X WHERE X is CWUser, X login "foobarbaz"')
+ rset = req.execute(u'Any X WHERE X is CWUser, X login "foobarbaz"')
data = self.view('jsonexport', rset, req=req)
self.assertEqual(req.headers_out.getRawHeaders('content-type'), ['application/json'])
self.assertListEqual(data, [])
@@ -47,21 +50,24 @@
'rql': u'Any GN,COUNT(X) GROUPBY GN ORDERBY GN '
'WHERE X in_group G, G name GN'})
data = self.ctrl_publish(req, ctrl='jsonp')
- self.assertIsInstance(data, str)
- self.assertEqual(req.headers_out.getRawHeaders('content-type'), ['application/javascript'])
+ self.assertIsInstance(data, binary_type)
+ self.assertEqual(req.headers_out.getRawHeaders('content-type'),
+ ['application/javascript'])
# because jsonp anonymizes data, only 'guests' group should be found
- self.assertEqual(data, 'foo(%s)' % self.res_jsonp_data)
+ self.assertEqual(data, b'foo(' + self.res_jsonp_data + b')')
def test_json_rsetexport_with_jsonp_and_bad_vid(self):
with self.admin_access.web_request() as req:
req.form.update({'callback': 'foo',
- 'vid': 'table', # <-- this parameter should be ignored by jsonp controller
+ # "vid" parameter should be ignored by jsonp controller
+ 'vid': 'table',
'rql': 'Any GN,COUNT(X) GROUPBY GN ORDERBY GN '
'WHERE X in_group G, G name GN'})
data = self.ctrl_publish(req, ctrl='jsonp')
- self.assertEqual(req.headers_out.getRawHeaders('content-type'), ['application/javascript'])
+ self.assertEqual(req.headers_out.getRawHeaders('content-type'),
+ ['application/javascript'])
# result should be plain json, not the table view
- self.assertEqual(data, 'foo(%s)' % self.res_jsonp_data)
+ self.assertEqual(data, b'foo(' + self.res_jsonp_data + b')')
def test_json_ersetexport(self):
with self.admin_access.web_request() as req:
@@ -71,7 +77,7 @@
self.assertEqual(data[0]['name'], 'guests')
self.assertEqual(data[1]['name'], 'managers')
- rset = req.execute('Any G WHERE G is CWGroup, G name "foo"')
+ rset = req.execute(u'Any G WHERE G is CWGroup, G name "foo"')
data = self.view('ejsonexport', rset, req=req)
self.assertEqual(req.headers_out.getRawHeaders('content-type'), ['application/json'])
self.assertEqual(data, [])
@@ -79,7 +85,8 @@
class NotAnonymousJsonViewsTC(JsonViewsTC):
anonymize = False
- res_jsonp_data = '[["guests", 1], ["managers", 1]]'
+ res_jsonp_data = b'[["guests", 1], ["managers", 1]]'
+
if __name__ == '__main__':
from logilab.common.testlib import unittest_main
--- a/web/test/unittest_views_searchrestriction.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_views_searchrestriction.py Wed Dec 16 11:23:48 2015 +0100
@@ -37,62 +37,62 @@
@property
def select(self):
- return self.parse('Any B,(NOW - CD),S,V,U,GROUP_CONCAT(TN),VN,P,CD,BMD '
- 'GROUPBY B,CD,S,V,U,VN,P,BMD '
- 'WHERE B in_state S, B creation_date CD, '
- 'B modification_date BMD, T? tags B, T name TN, '
- 'V? bookmarked_by B, V title VN, B created_by U?, '
- 'B in_group P, P name "managers"')
+ return self.parse(u'Any B,(NOW - CD),S,V,U,GROUP_CONCAT(TN),VN,P,CD,BMD '
+ 'GROUPBY B,CD,S,V,U,VN,P,BMD '
+ 'WHERE B in_state S, B creation_date CD, '
+ 'B modification_date BMD, T? tags B, T name TN, '
+ 'V? bookmarked_by B, V title VN, B created_by U?, '
+ 'B in_group P, P name "managers"')
def test_1(self):
self.assertEqual(self._generate(self.select, 'in_state', 'subject', 'name'),
- "DISTINCT Any A,C ORDERBY C WHERE B in_group P, P name 'managers', "
- "B in_state A, B is CWUser, A name C")
+ 'DISTINCT Any A,C ORDERBY C WHERE B in_group P, P name "managers", '
+ 'B in_state A, B is CWUser, A name C')
def test_2(self):
self.assertEqual(self._generate(self.select, 'tags', 'object', 'name'),
- "DISTINCT Any A,C ORDERBY C WHERE B in_group P, P name 'managers', "
- "A tags B, B is CWUser, A name C")
+ 'DISTINCT Any A,C ORDERBY C WHERE B in_group P, P name "managers", '
+ 'A tags B, B is CWUser, A name C')
def test_3(self):
self.assertEqual(self._generate(self.select, 'created_by', 'subject', 'login'),
- "DISTINCT Any A,C ORDERBY C WHERE B in_group P, P name 'managers', "
- "B created_by A, B is CWUser, A login C")
+ 'DISTINCT Any A,C ORDERBY C WHERE B in_group P, P name "managers", '
+ 'B created_by A, B is CWUser, A login C')
def test_4(self):
- self.assertEqual(self._generate(self.parse('Any X WHERE X is CWUser'), 'created_by', 'subject', 'login'),
+ self.assertEqual(self._generate(self.parse(u'Any X WHERE X is CWUser'), 'created_by', 'subject', 'login'),
"DISTINCT Any A,B ORDERBY B WHERE X is CWUser, X created_by A, A login B")
def test_5(self):
- self.assertEqual(self._generate(self.parse('Any X,L WHERE X is CWUser, X login L'), 'created_by', 'subject', 'login'),
+ self.assertEqual(self._generate(self.parse(u'Any X,L WHERE X is CWUser, X login L'), 'created_by', 'subject', 'login'),
"DISTINCT Any A,B ORDERBY B WHERE X is CWUser, X created_by A, A login B")
def test_nonregr1(self):
- select = self.parse('Any T,V WHERE T bookmarked_by V?, '
- 'V in_state VS, VS name "published", T created_by U')
+ select = self.parse(u'Any T,V WHERE T bookmarked_by V?, '
+ 'V in_state VS, VS name "published", T created_by U')
self.assertEqual(self._generate(select, 'created_by', 'subject', 'login'),
"DISTINCT Any A,B ORDERBY B WHERE T created_by U, "
"T created_by A, T is Bookmark, A login B")
def test_nonregr2(self):
#'DISTINCT Any X,TMP,N WHERE P name TMP, X version_of P, P is Project, X is Version, not X in_state S,S name "published", X num N ORDERBY TMP,N'
- select = self.parse('DISTINCT Any V,TN,L ORDERBY TN,L WHERE T nom TN, V connait T, T is Personne, V is CWUser,'
- 'NOT V in_state VS, VS name "published", V login L')
+ select = self.parse(u'DISTINCT Any V,TN,L ORDERBY TN,L WHERE T nom TN, V connait T, T is Personne, V is CWUser,'
+ 'NOT V in_state VS, VS name "published", V login L')
rschema = self.schema['connait']
- for rdefs in rschema.rdefs.itervalues():
+ for rdefs in rschema.rdefs.values():
rdefs.cardinality = '++'
try:
self.assertEqual(self._generate(select, 'in_state', 'subject', 'name'),
- "DISTINCT Any A,B ORDERBY B WHERE V is CWUser, "
- "NOT EXISTS(V in_state VS), VS name 'published', "
- "V in_state A, A name B")
+ 'DISTINCT Any A,B ORDERBY B WHERE V is CWUser, '
+ 'NOT EXISTS(V in_state VS), VS name "published", '
+ 'V in_state A, A name B')
finally:
- for rdefs in rschema.rdefs.itervalues():
+ for rdefs in rschema.rdefs.values():
rdefs.cardinality = '**'
def test_nonregr3(self):
#'DISTINCT Any X,TMP,N WHERE P name TMP, X version_of P, P is Project, X is Version, not X in_state S,S name "published", X num N ORDERBY TMP,N'
- select = self.parse('DISTINCT Any X, MAX(Y) GROUPBY X WHERE X is CWUser, Y is Bookmark, X in_group A')
+ select = self.parse(u'DISTINCT Any X, MAX(Y) GROUPBY X WHERE X is CWUser, Y is Bookmark, X in_group A')
self.assertEqual(self._generate(select, 'in_group', 'subject', 'name'),
"DISTINCT Any B,C ORDERBY C WHERE X is CWUser, X in_group B, B name C")
--- a/web/test/unittest_views_staticcontrollers.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_views_staticcontrollers.py Wed Dec 16 11:23:48 2015 +0100
@@ -70,12 +70,21 @@
with self._publish_static_files(fname) as req:
self.assertEqual(200, req.status_out)
self.assertIn('last-modified', req.headers_out)
+ self.assertIn('expires', req.headers_out)
+ self.assertEqual(req.get_response_header('cache-control'),
+ {'max-age': 604800})
next_headers = {
'if-modified-since': req.get_response_header('last-modified', raw=True),
}
with self._publish_static_files(fname, next_headers) as req:
self.assertEqual(304, req.status_out)
+ def _check_datafile_redirect(self, fname, expected):
+ with self._publish_static_files(fname) as req:
+ self.assertEqual(302, req.status_out)
+ self.assertEqual(req.get_response_header('location'),
+ req.base_url() + expected)
+
def _check_no_datafile(self, fname):
with self._publish_static_files(fname) as req:
self.assertEqual(404, req.status_out)
@@ -90,10 +99,12 @@
self._check_no_datafile('data/%s/cubicweb.css' % ('0'*len(hash)))
with tempattr(self.vreg.config, 'mode', 'notest'):
- self._check_datafile_ok('data/cubicweb.css')
+ self.config._init_base_url() # reset config.datadir_url
+ self._check_datafile_redirect('data/cubicweb.css', 'data/%s/cubicweb.css' % hash)
self._check_datafile_ok('data/%s/cubicweb.css' % hash)
- self._check_no_datafile('data/does/not/exist')
- self._check_no_datafile('data/%s/cubicweb.css' % ('0'*len(hash)))
+ self._check_no_datafile('data/%s/does/not/exist' % hash)
+ self._check_datafile_redirect('data/%s/does/not/exist' % ('0'*len(hash)),
+ 'data/%s/%s/does/not/exist' % (hash, '0'*len(hash)))
class ConcatFilesTC(CubicWebTC):
@@ -120,12 +131,12 @@
yield res, req
def expected_content(self, js_files):
- content = u''
+ content = b''
for js_file in js_files:
dirpath, rid = self.config.locate_resource(js_file)
if dirpath is not None: # ignore resources not found
- with open(osp.join(dirpath, rid)) as f:
- content += f.read() + '\n'
+ with open(osp.join(dirpath, rid), 'rb') as f:
+ content += f.read() + b'\n'
return content
def test_cache(self):
@@ -162,4 +173,3 @@
if __name__ == '__main__':
from logilab.common.testlib import unittest_main
unittest_main()
-
--- a/web/test/unittest_viewselector.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_viewselector.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,6 +17,7 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""XXX rename, split, reorganize this"""
+from __future__ import print_function
from logilab.common.testlib import unittest_main
@@ -76,9 +77,9 @@
try:
self.assertSetEqual(list(content), expected)
except Exception:
- print registry, sorted(expected), sorted(content)
- print 'no more', [v for v in expected if not v in content]
- print 'missing', [v for v in content if not v in expected]
+ print(registry, sorted(expected), sorted(content))
+ print('no more', [v for v in expected if not v in content])
+ print('missing', [v for v in content if not v in expected])
raise
def setUp(self):
@@ -421,7 +422,7 @@
def test_interface_selector(self):
with self.admin_access.web_request() as req:
- req.create_entity('File', data_name=u'bim.png', data=Binary('bim'))
+ req.create_entity('File', data_name=u'bim.png', data=Binary(b'bim'))
# image primary view priority
rset = req.execute('File X WHERE X data_name "bim.png"')
self.assertIsInstance(self.vreg['views'].select('primary', req, rset=rset),
@@ -430,21 +431,21 @@
def test_score_entity_selector(self):
with self.admin_access.web_request() as req:
- req.create_entity('File', data_name=u'bim.png', data=Binary('bim'))
+ req.create_entity('File', data_name=u'bim.png', data=Binary(b'bim'))
# image/ehtml primary view priority
rset = req.execute('File X WHERE X data_name "bim.png"')
self.assertIsInstance(self.vreg['views'].select('image', req, rset=rset),
idownloadable.ImageView)
self.assertRaises(NoSelectableObject, self.vreg['views'].select, 'ehtml', req, rset=rset)
- fileobj = req.create_entity('File', data_name=u'bim.html', data=Binary('<html>bam</html'))
+ fileobj = req.create_entity('File', data_name=u'bim.html', data=Binary(b'<html>bam</html'))
# image/ehtml primary view priority
rset = req.execute('File X WHERE X data_name "bim.html"')
self.assertIsInstance(self.vreg['views'].select('ehtml', req, rset=rset),
idownloadable.EHTMLView)
self.assertRaises(NoSelectableObject, self.vreg['views'].select, 'image', req, rset=rset)
- fileobj = req.create_entity('File', data_name=u'bim.txt', data=Binary('boum'))
+ fileobj = req.create_entity('File', data_name=u'bim.txt', data=Binary(b'boum'))
# image/ehtml primary view priority
rset = req.execute('File X WHERE X data_name "bim.txt"')
self.assertRaises(NoSelectableObject, self.vreg['views'].select, 'image', req, rset=rset)
@@ -461,7 +462,7 @@
obj = self.vreg['views'].select(vid, req, rset=rset, **args)
return obj.render(**args)
except Exception:
- print vid, rset, args
+ print(vid, rset, args)
raise
def test_form(self):
@@ -476,12 +477,12 @@
def test_properties(self):
- self.assertEqual(sorted(k for k in self.vreg['propertydefs'].iterkeys()
+ self.assertEqual(sorted(k for k in self.vreg['propertydefs']
if k.startswith('ctxcomponents.edit_box')),
['ctxcomponents.edit_box.context',
'ctxcomponents.edit_box.order',
'ctxcomponents.edit_box.visible'])
- self.assertEqual([k for k in self.vreg['propertyvalues'].iterkeys()
+ self.assertEqual([k for k in self.vreg['propertyvalues']
if not k.startswith('system.version')],
[])
self.assertEqual(self.vreg.property_value('ctxcomponents.edit_box.visible'), True)
--- a/web/test/unittest_webconfig.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/test/unittest_webconfig.py Wed Dec 16 11:23:48 2015 +0100
@@ -56,5 +56,3 @@
if __name__ == '__main__':
unittest_main()
-
-
--- a/web/uicfg.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/uicfg.py Wed Dec 16 11:23:48 2015 +0100
@@ -26,4 +26,3 @@
warn('[3.16] moved to cubicweb.web.views.uicfg',
DeprecationWarning, stacklevel=2)
-
--- a/web/uihelper.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/uihelper.py Wed Dec 16 11:23:48 2015 +0100
@@ -45,6 +45,7 @@
"""
__docformat__ = "restructuredtext en"
+from six import add_metaclass
from logilab.common.deprecation import deprecated
from cubicweb.web.views import uicfg
@@ -93,6 +94,7 @@
super(meta_formconfig, cls).__init__(name, bases, classdict)
+@add_metaclass(meta_formconfig)
class FormConfig:
"""helper base class to define uicfg rules on a given entity type.
@@ -162,7 +164,6 @@
inlined = ('use_email',)
"""
- __metaclass__ = meta_formconfig
formtype = 'main'
etype = None # must be defined in concrete subclasses
hidden = ()
--- a/web/views/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -23,6 +23,8 @@
import sys
import tempfile
+from six import add_metaclass
+
from rql import nodes
from logilab.mtconverter import xml_escape
from logilab.common.deprecation import class_deprecated
@@ -77,7 +79,7 @@
#'text/xml': 'xml',
# XXX rss, owl...
}
-def vid_from_rset(req, rset, schema):
+def vid_from_rset(req, rset, schema, check_table=True):
"""given a result set, return a view id"""
if rset is None:
return 'index'
@@ -90,7 +92,7 @@
return 'noresult'
# entity result set
if not schema.eschema(rset.description[0][0]).final:
- if need_table_view(rset, schema):
+ if check_table and need_table_view(rset, schema):
return 'table'
if nb_rows == 1:
if req.search_state[0] == 'normal':
@@ -127,8 +129,8 @@
+@add_metaclass(class_deprecated)
class TmpFileViewMixin(object):
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.18] %(cls)s is deprecated'
binary = True
content_type = 'application/octet-stream'
--- a/web/views/actions.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/actions.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""Set of HTML base actions"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
@@ -76,7 +76,7 @@
return 0
select = rqlst.children[0]
if len(select.defined_vars) == 1 and len(select.solutions) == 1:
- rset._searched_etype = select.solutions[0].itervalues().next()
+ rset._searched_etype = next(iter(select.solutions[0].values()))
eschema = req.vreg.schema.eschema(rset._searched_etype)
if not (eschema.final or eschema.is_subobject(strict=True)) \
and eschema.has_perm(req, 'add'):
--- a/web/views/ajaxcontroller.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/ajaxcontroller.py Wed Dec 16 11:23:48 2015 +0100
@@ -66,6 +66,8 @@
from warnings import warn
from functools import partial
+from six import PY2, text_type
+
from logilab.common.date import strptime
from logilab.common.registry import yes
from logilab.common.deprecation import deprecated
@@ -84,7 +86,7 @@
if extraargs is None:
return {}
# we receive unicode keys which is not supported by the **syntax
- return dict((str(key), value) for key, value in extraargs.iteritems())
+ return dict((str(key), value) for key, value in extraargs.items())
class AjaxController(Controller):
@@ -117,7 +119,9 @@
raise RemoteCallFailed('no method specified')
# 1/ check first for old-style (JSonController) ajax func for bw compat
try:
- func = getattr(basecontrollers.JSonController, 'js_%s' % fname).im_func
+ func = getattr(basecontrollers.JSonController, 'js_%s' % fname)
+ if PY2:
+ func = func.__func__
func = partial(func, self)
except AttributeError:
# 2/ check for new-style (AjaxController) ajax func
@@ -150,7 +154,7 @@
if result is None:
return ''
# get unicode on @htmlize methods, encoded string on @jsonize methods
- elif isinstance(result, unicode):
+ elif isinstance(result, text_type):
return result.encode(self._cw.encoding)
return result
--- a/web/views/authentication.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/authentication.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,12 +19,9 @@
__docformat__ = "restructuredtext en"
-from threading import Lock
-
-from logilab.common.decorators import clear_cache
from logilab.common.deprecation import class_renamed
-from cubicweb import AuthenticationError, BadConnectionId
+from cubicweb import AuthenticationError
from cubicweb.view import Component
from cubicweb.web import InvalidSession
@@ -101,41 +98,11 @@
'("ie" instead of "ei")')
-class AbstractAuthenticationManager(Component):
- """authenticate user associated to a request and check session validity"""
- __abstract__ = True
- __regid__ = 'authmanager'
- def __init__(self, repo):
- self.vreg = repo.vreg
-
- def validate_session(self, req, session):
- """check session validity, reconnecting it to the repository if the
- associated connection expired in the repository side (hence the
- necessity for this method).
-
- raise :exc:`InvalidSession` if session is corrupted for a reason or
- another and should be closed
- """
- raise NotImplementedError()
-
- def authenticate(self, req):
- """authenticate user using connection information found in the request,
- and return corresponding a :class:`~cubicweb.dbapi.Connection` instance,
- as well as login and authentication information dictionary used to open
- the connection.
-
- raise :exc:`cubicweb.AuthenticationError` if authentication failed
- (no authentication info found or wrong user/password)
- """
- raise NotImplementedError()
-
-
-class RepositoryAuthenticationManager(AbstractAuthenticationManager):
+class RepositoryAuthenticationManager(object):
"""authenticate user associated to a request and check session validity"""
def __init__(self, repo):
- super(RepositoryAuthenticationManager, self).__init__(repo)
self.repo = repo
vreg = repo.vreg
self.log_queries = vreg.config['query-log-file']
@@ -205,4 +172,3 @@
def _authenticate(self, login, authinfo):
sessionid = self.repo.connect(login, **authinfo)
return self.repo._sessions[sessionid]
-
--- a/web/views/autoform.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/autoform.py Wed Dec 16 11:23:48 2015 +0100
@@ -119,10 +119,12 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
+from six.moves import range
+
from logilab.mtconverter import xml_escape
from logilab.common.decorators import iclassmethod, cached
from logilab.common.deprecation import deprecated
@@ -355,7 +357,7 @@
self.w(self._cw._('no such entity type %s') % self.etype)
return
entity = cls(self._cw)
- entity.eid = self._cw.varmaker.next()
+ entity.eid = next(self._cw.varmaker)
return entity
def call(self, i18nctx, **kwargs):
@@ -491,7 +493,8 @@
pendings.remove( (int(eidfrom), rel, int(eidto)) )
@ajaxfunc(output_type='json')
-def remove_pending_insert(self, (eidfrom, rel, eidto)):
+def remove_pending_insert(self, args):
+ eidfrom, rel, eidto = args
_remove_pending(self._cw, eidfrom, rel, eidto, 'insert')
@ajaxfunc(output_type='json')
@@ -500,11 +503,13 @@
_add_pending(self._cw, eidfrom, rel, eidto, 'insert')
@ajaxfunc(output_type='json')
-def remove_pending_delete(self, (eidfrom, rel, eidto)):
+def remove_pending_delete(self, args):
+ eidfrom, rel, eidto = args
_remove_pending(self._cw, eidfrom, rel, eidto, 'delete')
@ajaxfunc(output_type='json')
-def add_pending_delete(self, (eidfrom, rel, eidto)):
+def add_pending_delete(self, args):
+ eidfrom, rel, eidto = args
_add_pending(self._cw, eidfrom, rel, eidto, 'delete')
@@ -608,7 +613,7 @@
toggleable_rel_link_func = toggleable_relation_link
else:
toggleable_rel_link_func = lambda x, y, z: u''
- for row in xrange(rset.rowcount):
+ for row in range(rset.rowcount):
nodeid = relation_id(entity.eid, rschema, role,
rset[row][0])
if nodeid in pending_deletes:
@@ -737,7 +742,8 @@
copy_nav_params = True
form_buttons = [fw.SubmitButton(),
fw.Button(stdmsgs.BUTTON_APPLY, cwaction='apply'),
- fw.Button(stdmsgs.BUTTON_CANCEL, cwaction='cancel')]
+ fw.Button(stdmsgs.BUTTON_CANCEL,
+ {'class': fw.Button.css_class + ' cwjs-edition-cancel'})]
# for attributes selection when searching in uicfg.autoform_section
formtype = 'main'
# set this to a list of [(relation, role)] if you want to explictily tell
@@ -1048,4 +1054,4 @@
AutomaticEntityForm.error('field for %s %s may not be found in schema' % (rtype, role))
return None
- vreg.register_all(globals().itervalues(), __name__)
+ vreg.register_all(globals().values(), __name__)
--- a/web/views/basecomponents.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/basecomponents.py Wed Dec 16 11:23:48 2015 +0100
@@ -21,7 +21,7 @@
* the logged user link
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from logilab.mtconverter import xml_escape
from logilab.common.registry import yes
--- a/web/views/basecontrollers.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/basecontrollers.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,10 +20,12 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
+from six import text_type
+
from logilab.common.deprecation import deprecated
from cubicweb import (NoSelectableObject, ObjectNotFound, ValidationError,
@@ -124,10 +126,9 @@
def publish(self, rset=None):
"""publish a request, returning an encoded string"""
view, rset = self._select_view_and_rset(rset)
- self.add_to_breadcrumbs(view)
view.set_http_cache_headers()
if self._cw.is_client_cache_valid():
- return ''
+ return b''
template = self.appli.main_template_id(self._cw)
return self._cw.vreg['views'].main_template(self._cw, template,
rset=rset, view=view)
@@ -158,13 +159,6 @@
view = req.vreg['views'].select(vid, req, rset=rset)
return view, rset
- def add_to_breadcrumbs(self, view):
- # update breadcrumbs **before** validating cache, unless the view
- # specifies explicitly it should not be added to breadcrumb or the
- # view is a binary view
- if view.add_to_breadcrumbs and not view.binary:
- self._cw.update_breadcrumbs()
-
def execute_linkto(self, eid=None):
"""XXX __linkto parameter may cause security issue
@@ -233,7 +227,7 @@
except Exception as ex:
req.cnx.rollback()
req.exception('unexpected error while validating form')
- return (False, str(ex).decode('utf-8'), ctrl._edited_entity)
+ return (False, text_type(ex), ctrl._edited_entity)
return (False, '???', None)
@@ -255,9 +249,8 @@
# XXX unclear why we have a separated controller here vs
# js_validate_form on the json controller
status, args, entity = _validate_form(self._cw, self._cw.vreg)
- domid = self._cw.form.get('__domid', 'entityForm').encode(
- self._cw.encoding)
- return self.response(domid, status, args, entity)
+ domid = self._cw.form.get('__domid', 'entityForm')
+ return self.response(domid, status, args, entity).encode(self._cw.encoding)
class JSonController(Controller):
@@ -306,5 +299,4 @@
def redirect(self, msg=None):
req = self._cw
msg = msg or req._("transaction undone")
- self._return_to_lastpage( dict(_cwmsgid= req.set_redirect_message(msg)) )
-
+ self._redirect({'_cwmsgid': req.set_redirect_message(msg)})
--- a/web/views/basetemplates.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/basetemplates.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""default templates for CubicWeb web client"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from logilab.mtconverter import xml_escape
from logilab.common.deprecation import class_renamed
--- a/web/views/baseviews.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/baseviews.py Wed Dec 16 11:23:48 2015 +0100
@@ -76,11 +76,13 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from datetime import timedelta
from warnings import warn
+from six.moves import range
+
from rql import nodes
from logilab.mtconverter import TransformError, xml_escape
@@ -231,8 +233,8 @@
"""
rset = self.cw_rset
if rset is None:
- raise NotImplementedError, self
- for i in xrange(len(rset)):
+ raise NotImplementedError(self)
+ for i in range(len(rset)):
self.wview(self.__regid__, rset, row=i, **kwargs)
if len(rset) > 1:
self.w(u"\n")
@@ -314,7 +316,7 @@
self.w(u'<ul>\n')
else:
self.w(u'<ul%s class="%s">\n' % (listid, klass or 'section'))
- for i in xrange(self.cw_rset.rowcount):
+ for i in range(self.cw_rset.rowcount):
self.cell_call(row=i, col=0, vid=subvid, klass=klass, **kwargs)
self.w(u'</ul>\n')
if title:
@@ -393,7 +395,7 @@
@property
def title(self):
- etype = iter(self.cw_rset.column_types(0)).next()
+ etype = next(iter(self.cw_rset.column_types(0)))
return display_name(self._cw, etype, form='plural')
def call(self, **kwargs):
@@ -427,7 +429,7 @@
def call(self, subvid=None, **kwargs):
kwargs['vid'] = subvid
rset = self.cw_rset
- for i in xrange(len(rset)):
+ for i in range(len(rset)):
self.cell_call(i, 0, **kwargs)
if i < rset.rowcount-1:
self.w(self.separator)
--- a/web/views/bookmark.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/bookmark.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""Primary view for bookmarks + user's bookmarks box"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from logilab.mtconverter import xml_escape
--- a/web/views/boxes.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/boxes.py Wed Dec 16 11:23:48 2015 +0100
@@ -26,10 +26,12 @@
* startup views box
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
+from six import text_type, add_metaclass
+
from logilab.mtconverter import xml_escape
from logilab.common.deprecation import class_deprecated
@@ -93,7 +95,7 @@
etypes = self.cw_rset.column_types(0)
if len(etypes) == 1:
plural = self.cw_rset.rowcount > 1 and 'plural' or ''
- etypelabel = display_name(self._cw, iter(etypes).next(), plural)
+ etypelabel = display_name(self._cw, next(iter(etypes)), plural)
title = u'%s - %s' % (title, etypelabel.lower())
w(title)
@@ -216,7 +218,7 @@
@property
def domid(self):
- return super(RsetBox, self).domid + unicode(abs(id(self))) + unicode(abs(id(self.cw_rset)))
+ return super(RsetBox, self).domid + text_type(abs(id(self))) + text_type(abs(id(self.cw_rset)))
def render_title(self, w):
w(self.cw_extra_kwargs['title'])
@@ -231,9 +233,9 @@
# helper classes ##############################################################
+@add_metaclass(class_deprecated)
class SideBoxView(EntityView):
"""helper view class to display some entities in a sidebox"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.10] SideBoxView is deprecated, use RsetBox instead (%(cls)s)'
__regid__ = 'sidebox'
--- a/web/views/calendar.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/calendar.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""html calendar views"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
import copy
from datetime import timedelta
--- a/web/views/csvexport.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/csvexport.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,10 @@
"""csv export views"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six import PY2
+from six.moves import range
from cubicweb.schema import display_name
from cubicweb.predicates import any_rset, empty_rset
@@ -29,7 +32,7 @@
"""mixin class for CSV views"""
templatable = False
content_type = "text/comma-separated-values"
- binary = True # avoid unicode assertion
+ binary = PY2 # python csv module is unicode aware in py3k
csv_params = {'dialect': 'excel',
'quotechar': '"',
'delimiter': ';',
@@ -88,7 +91,7 @@
rows_by_type = {}
writer = self.csvwriter()
rowdef_by_type = {}
- for index in xrange(len(self.cw_rset)):
+ for index in range(len(self.cw_rset)):
entity = self.cw_rset.complete_entity(index)
if entity.e_schema not in rows_by_type:
rowdef_by_type[entity.e_schema] = [rs for rs, at in entity.e_schema.attribute_definitions()
@@ -98,8 +101,7 @@
rows = rows_by_type[entity.e_schema]
rows.append([entity.printable_value(rs.type, format='text/plain')
for rs in rowdef_by_type[entity.e_schema]])
- for rows in rows_by_type.itervalues():
+ for rows in rows_by_type.values():
writer.writerows(rows)
# use two empty lines as separator
writer.writerows([[], []])
-
--- a/web/views/cwproperties.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/cwproperties.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""Specific views for CWProperty (eg site/user preferences"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from logilab.mtconverter import xml_escape
@@ -119,10 +119,10 @@
_ = self._cw._
self.w(u'<h1>%s</h1>\n' % _(self.title))
for label, group, form in sorted((_(g), g, f)
- for g, f in mainforms.iteritems()):
+ for g, f in mainforms.items()):
self.wrap_main_form(group, label, form)
for label, group, objects in sorted((_(g), g, o)
- for g, o in groupedforms.iteritems()):
+ for g, o in groupedforms.items()):
self.wrap_grouped_form(group, label, objects)
@property
@@ -171,7 +171,7 @@
entity = self.cwprops_rset.get_entity(values[key], 0)
else:
entity = self._cw.vreg['etypes'].etype_class('CWProperty')(self._cw)
- entity.eid = self._cw.varmaker.next()
+ entity.eid = next(self._cw.varmaker)
entity.cw_attr_cache['pkey'] = key
entity.cw_attr_cache['value'] = self._cw.vreg.property_value(key)
return entity
@@ -224,7 +224,7 @@
(make_togglable_link('fieldset_' + group, label)))
self.w(u'<div id="fieldset_%s" %s>' % (group, status))
sorted_objects = sorted((self._cw.__('%s_%s' % (group, o)), o, f)
- for o, f in objects.iteritems())
+ for o, f in objects.items())
for label, oid, form in sorted_objects:
self.wrap_object_form(group, oid, label, form)
self.w(u'</div>')
--- a/web/views/cwsources.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/cwsources.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,20 +20,23 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
import logging
from itertools import repeat
+
+from six.moves import range
+
from logilab.mtconverter import xml_escape
from logilab.common.decorators import cachedproperty
from cubicweb import Unauthorized, tags
from cubicweb.utils import make_uid
from cubicweb.predicates import (is_instance, score_entity, has_related_entities,
- match_user_groups, match_kwargs, match_view)
+ match_user_groups, match_kwargs, match_view, one_line_rset)
from cubicweb.view import EntityView, StartupView
from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES, display_name
-from cubicweb.web import formwidgets as wdgs, facet
+from cubicweb.web import Redirect, formwidgets as wdgs, facet, action
from cubicweb.web.views import add_etype_button
from cubicweb.web.views import (uicfg, tabs, actions, ibreadcrumbs, navigation,
tableview, pyviews)
@@ -95,7 +98,7 @@
if hostconfig:
self.w(u'<h3>%s</h3>' % self._cw._('CWSourceHostConfig_plural'))
self._cw.view('table', hostconfig, w=self.w,
- displaycols=range(2),
+ displaycols=list(range(2)),
cellvids={1: 'editable-final'})
@@ -186,7 +189,7 @@
warning(_('relation %(rtype)s with %(etype)s as %(role)s is '
'supported but no target type supported') %
{'rtype': rschema, 'role': role, 'etype': etype})
- for rtype, rdefs in self.srelations.iteritems():
+ for rtype, rdefs in self.srelations.items():
if rdefs is None:
rschema = self.schema[rtype]
for subj, obj in rschema.rdefs:
@@ -223,6 +226,36 @@
layout_args = {'display_filter': 'top'}
+class CWSourceSyncAction(action.Action):
+ __regid__ = 'cw.source-sync'
+ __select__ = (action.Action.__select__ & match_user_groups('managers')
+ & one_line_rset() & is_instance('CWSource')
+ & score_entity(lambda x: x.name != 'system'))
+
+ title = _('synchronize')
+ category = 'mainactions'
+ order = 20
+
+ def url(self):
+ entity = self.cw_rset.get_entity(self.cw_row or 0, self.cw_col or 0)
+ return entity.absolute_url(vid=self.__regid__)
+
+
+class CWSourceSyncView(EntityView):
+ __regid__ = 'cw.source-sync'
+ __select__ = (match_user_groups('managers')
+ & one_line_rset() & is_instance('CWSource')
+ & score_entity(lambda x: x.name != 'system'))
+
+ title = _('synchronize')
+
+ def entity_call(self, entity):
+ self._cw.call_service('source-sync', source_eid=entity.eid)
+ msg = self._cw._('Source has been synchronized')
+ url = entity.absolute_url(tab='cwsource-imports', __message=msg)
+ raise Redirect(url)
+
+
# sources management view ######################################################
--- a/web/views/cwuser.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/cwuser.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,10 +18,13 @@
"""Specific views for users and groups"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from hashlib import sha1 # pylint: disable=E0611
+from six import text_type
+from six.moves import range
+
from logilab.mtconverter import xml_escape
from cubicweb import tags
@@ -64,7 +67,7 @@
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3org/2000/01/rdf-schema#"
xmlns:foaf="http://xmlns.com/foaf/0.1/"> '''% self._cw.encoding)
- for i in xrange(self.cw_rset.rowcount):
+ for i in range(self.cw_rset.rowcount):
self.cell_call(i, 0)
self.w(u'</rdf:RDF>\n')
@@ -250,6 +253,6 @@
'group': tableview.MainEntityColRenderer(),
'nb_users': tableview.EntityTableColRenderer(
header=_('num. users'),
- renderfunc=lambda w,x: w(unicode(x.num_users())),
+ renderfunc=lambda w,x: w(text_type(x.num_users())),
sortfunc=lambda x: x.num_users()),
}
--- a/web/views/debug.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/debug.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,10 +18,12 @@
"""management and error screens"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from time import strftime, localtime
+from six import text_type
+
from logilab.mtconverter import xml_escape
from cubicweb.predicates import none_rset, match_user_groups
@@ -33,7 +35,7 @@
if dict:
w(u'<ul>')
for key in sorted(dict):
- w(u'<li><span class="label">%s</span>: <span>%s</span></li>' % (
+ w(u'<li><span>%s</span>: <span>%s</span></li>' % (
xml_escape(str(key)), xml_escape(repr(dict[key]))))
w(u'</ul>')
@@ -71,31 +73,23 @@
dtformat = req.property_value('ui.datetime-format')
_ = req._
w = self.w
+ repo = req.cnx.repo
# generic instance information
w(u'<h2>%s</h2>' % _('Instance'))
- w(u'<table>')
- w(u'<tr><th align="left">%s</th><td>%s</td></tr>' % (
- _('config type'), self._cw.vreg.config.name))
- w(u'<tr><th align="left">%s</th><td>%s</td></tr>' % (
- _('config mode'), self._cw.vreg.config.mode))
- w(u'<tr><th align="left">%s</th><td>%s</td></tr>' % (
- _('instance home'), self._cw.vreg.config.apphome))
- w(u'</table>')
- vcconf = req.vreg.config.vc_config()
+ pyvalue = ((_('config type'), self._cw.vreg.config.name),
+ (_('config mode'), self._cw.vreg.config.mode),
+ (_('instance home'), self._cw.vreg.config.apphome))
+ self.wview('pyvaltable', pyvalue=pyvalue, header_column_idx=0)
+ vcconf = repo.get_versions()
w(u'<h3>%s</h3>' % _('versions configuration'))
- w(u'<table>')
- w(u'<tr><th align="left">%s</th><td>%s</td></tr>' % (
- 'CubicWeb', vcconf.get('cubicweb', _('no version information'))))
- for cube in sorted(self._cw.vreg.config.cubes()):
- cubeversion = vcconf.get(cube, _('no version information'))
- w(u'<tr><th align="left">%s</th><td>%s</td></tr>' % (
- cube, cubeversion))
- w(u'</table>')
+ missing = _('no version information')
+ pyvalue = [('CubicWeb', vcconf.get('cubicweb', missing))]
+ pyvalue += [(cube, vcconf.get(cube, missing))
+ for cube in sorted(self._cw.vreg.config.cubes())]
+ self.wview('pyvaltable', pyvalue=pyvalue, header_column_idx=0)
# repository information
- repo = req.vreg.config.repository(None)
w(u'<h2>%s</h2>' % _('Repository'))
w(u'<h3>%s</h3>' % _('resources usage'))
- w(u'<table>')
stats = self._cw.call_service('repo_stats')
stats['looping_tasks'] = ', '.join('%s (%s seconds)' % (n, i) for n, i in stats['looping_tasks'])
stats['threads'] = ', '.join(sorted(stats['threads']))
@@ -104,11 +98,13 @@
continue
if k.endswith('_cache_size'):
stats[k] = '%s / %s' % (stats[k]['size'], stats[k]['maxsize'])
- for element in sorted(stats):
- w(u'<tr><th align="left">%s</th><td>%s %s</td></tr>'
- % (element, xml_escape(unicode(stats[element])),
- element.endswith('percent') and '%' or '' ))
- w(u'</table>')
+ def format_stat(sname, sval):
+ return '%s %s' % (xml_escape(text_type(sval)),
+ sname.endswith('percent') and '%' or '')
+ pyvalue = [(sname, format_stat(sname, sval))
+ for sname, sval in sorted(stats.items())]
+ self.wview('pyvaltable', pyvalue=pyvalue, header_column_idx=0)
+ # open repo sessions
if req.cnx.is_repo_in_memory and req.user.is_in_group('managers'):
w(u'<h3>%s</h3>' % _('opened sessions'))
sessions = repo._sessions.values()
@@ -116,7 +112,7 @@
w(u'<ul>')
for session in sessions:
w(u'<li>%s (%s: %s)<br/>' % (
- xml_escape(unicode(session)),
+ xml_escape(text_type(session)),
_('last usage'),
strftime(dtformat, localtime(session.timestamp))))
dict_to_html(w, session.data)
@@ -126,12 +122,9 @@
w(u'<p>%s</p>' % _('no repository sessions found'))
# web server information
w(u'<h2>%s</h2>' % _('Web server'))
- w(u'<table>')
- w(u'<tr><th align="left">%s</th><td>%s</td></tr>' % (
- _('base url'), req.base_url()))
- w(u'<tr><th align="left">%s</th><td>%s</td></tr>' % (
- _('data directory url'), req.datadir_url))
- w(u'</table>')
+ pyvalue = ((_('base url'), req.base_url()),
+ (_('data directory url'), req.datadir_url))
+ self.wview('pyvaltable', pyvalue=pyvalue, header_column_idx=0)
from cubicweb.web.application import SESSION_MANAGER
if SESSION_MANAGER is not None and req.user.is_in_group('managers'):
sessions = SESSION_MANAGER.current_sessions()
@@ -170,7 +163,7 @@
continue
self.w(u'<h3 id="%s">%s</h3>' % (key, key))
if self._cw.vreg[key]:
- values = sorted(self._cw.vreg[key].iteritems())
+ values = sorted(self._cw.vreg[key].items())
self.wview('pyvaltable', pyvalue=[(key, xml_escape(repr(val)))
for key, val in values])
else:
--- a/web/views/dotgraphview.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/dotgraphview.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""some basic stuff to build dot generated graph images"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
import tempfile
import os
--- a/web/views/editcontroller.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/editcontroller.py Wed Dec 16 11:23:48 2015 +0100
@@ -24,12 +24,14 @@
from datetime import datetime
+from six import text_type
+
from logilab.common.deprecation import deprecated
from logilab.common.graph import ordered_nodes
from rql.utils import rqlvar_maker
-from cubicweb import Binary, ValidationError
+from cubicweb import _, Binary, ValidationError
from cubicweb.view import EntityAdapter
from cubicweb.predicates import is_instance
from cubicweb.web import (INTERNAL_FIELD_VALUE, RequestError, NothingToEdit,
@@ -93,9 +95,9 @@
def update_query(self, eid):
varmaker = rqlvar_maker()
- var = varmaker.next()
+ var = next(varmaker)
while var in self.kwargs:
- var = varmaker.next()
+ var = next(varmaker)
rql = 'SET %s WHERE X eid %%(%s)s' % (','.join(self.edited), var)
if self.restrictions:
rql += ', %s' % ','.join(self.restrictions)
@@ -143,7 +145,7 @@
values_by_eid = dict((eid, req.extract_entity_params(eid, minparams=2))
for eid in req.edited_eids())
# iterate over all the edited entities
- for eid, values in values_by_eid.iteritems():
+ for eid, values in values_by_eid.items():
# add eid to the dependency graph
graph.setdefault(eid, set())
# search entity's edited fields for mandatory inlined relation
@@ -197,7 +199,7 @@
if '__linkto' in req.form and 'eid' in req.form:
self.execute_linkto()
elif not ('__delete' in req.form or '__insert' in req.form):
- raise ValidationError(None, {None: unicode(ex)})
+ raise ValidationError(None, {None: text_type(ex)})
# all pending inlined relations to newly created entities have been
# treated now (pop to ensure there are no attempt to add new ones)
pending_inlined = req.data.pop('pending_inlined')
@@ -215,7 +217,7 @@
autoform.delete_relations(self._cw, todelete)
self._cw.remove_pending_operations()
if self.errors:
- errors = dict((f.name, unicode(ex)) for f, ex in self.errors)
+ errors = dict((f.name, text_type(ex)) for f, ex in self.errors)
raise ValidationError(valerror_eid(form.get('__maineid')), errors)
def _insert_entity(self, etype, eid, rqlquery):
@@ -265,7 +267,7 @@
for form_, field in req.data['pending_inlined'].pop(entity.eid, ()):
rqlquery.set_inlined(field.name, form_.edited_entity.eid)
if self.errors:
- errors = dict((f.role_name(), unicode(ex)) for f, ex in self.errors)
+ errors = dict((f.role_name(), text_type(ex)) for f, ex in self.errors)
raise ValidationError(valerror_eid(entity.eid), errors)
if eid is None: # creation or copy
entity.eid = eid = self._insert_entity(etype, formparams['eid'], rqlquery)
@@ -316,7 +318,7 @@
"""handle edition for the (rschema, x) relation of the given entity
"""
if values:
- rqlquery.set_inlined(field.name, iter(values).next())
+ rqlquery.set_inlined(field.name, next(iter(values)))
elif form.edited_entity.has_eid():
self.handle_relation(form, field, values, origvalues)
@@ -355,13 +357,13 @@
for eid, etype in eidtypes:
entity = self._cw.entity_from_eid(eid, etype)
path, params = entity.cw_adapt_to('IEditControl').after_deletion_path()
- redirect_info.add( (path, tuple(params.iteritems())) )
+ redirect_info.add( (path, tuple(params.items())) )
entity.cw_delete()
if len(redirect_info) > 1:
# In the face of ambiguity, refuse the temptation to guess.
self._after_deletion_path = 'view', ()
else:
- self._after_deletion_path = iter(redirect_info).next()
+ self._after_deletion_path = next(iter(redirect_info))
if len(eidtypes) > 1:
self._cw.set_message(self._cw._('entities deleted'))
else:
@@ -388,13 +390,6 @@
self._default_publish()
self.reset()
- def _action_cancel(self):
- errorurl = self._cw.form.get('__errorurl')
- if errorurl:
- self._cw.cancel_edition(errorurl)
- self._cw.set_message(self._cw._('edit canceled'))
- return self.reset()
-
def _action_delete(self):
self.delete_entities(self._cw.edited_eids(withtype=True))
return self.reset()
--- a/web/views/editforms.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/editforms.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,10 +20,12 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from copy import copy
+from six.moves import range
+
from logilab.mtconverter import xml_escape
from logilab.common.decorators import cached
from logilab.common.registry import yes
@@ -145,7 +147,7 @@
# selector
etype = kwargs.pop('etype', self._cw.form.get('etype'))
entity = self._cw.vreg['etypes'].etype_class(etype)(self._cw)
- entity.eid = self._cw.varmaker.next()
+ entity.eid = next(self._cw.varmaker)
self.render_form(entity)
def form_title(self, entity):
@@ -197,7 +199,7 @@
entity.complete()
self.newentity = copy(entity)
self.copying = entity
- self.newentity.eid = self._cw.varmaker.next()
+ self.newentity.eid = next(self._cw.varmaker)
self.w(u'<script type="text/javascript">updateMessage("%s");</script>\n'
% self._cw._(self.warning_message))
super(CopyFormView, self).render_form(self.newentity)
@@ -230,7 +232,7 @@
def __init__(self, req, rset, **kwargs):
kwargs.setdefault('__redirectrql', rset.printable_rql())
super(TableEditForm, self).__init__(req, rset=rset, **kwargs)
- for row in xrange(len(self.cw_rset)):
+ for row in range(len(self.cw_rset)):
form = self._cw.vreg['forms'].select('edition', self._cw,
rset=self.cw_rset, row=row,
formtype='muledit',
--- a/web/views/editviews.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/editviews.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""Some views used to help to the edition process"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from logilab.common.decorators import cached
from logilab.mtconverter import xml_escape
--- a/web/views/facets.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/facets.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""the facets box and some basic facets"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
@@ -168,7 +168,7 @@
DeprecationWarning, stacklevel=2)
else:
vidargs = {}
- vidargs = dict((k, v) for k, v in vidargs.iteritems() if v)
+ vidargs = dict((k, v) for k, v in vidargs.items() if v)
facetargs = xml_escape(json_dumps([divid, vid, paginate, vidargs]))
w(u'<form id="%sForm" class="%s" method="post" action="" '
'cubicweb:facetargs="%s" >' % (divid, cssclass, facetargs))
--- a/web/views/formrenderers.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/formrenderers.py Wed Dec 16 11:23:48 2015 +0100
@@ -33,10 +33,12 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
+from six import text_type
+
from logilab.mtconverter import xml_escape
from logilab.common.registry import yes
@@ -119,7 +121,7 @@
data.insert(0, errormsg)
# NOTE: we call unicode because `tag` objects may be found within data
# e.g. from the cwtags library
- w(''.join(unicode(x) for x in data))
+ w(''.join(text_type(x) for x in data))
def render_content(self, w, form, values):
if self.display_progress_div:
@@ -241,7 +243,7 @@
if form.fieldsets_in_order:
fieldsets = form.fieldsets_in_order
else:
- fieldsets = byfieldset.iterkeys()
+ fieldsets = byfieldset
for fieldset in list(fieldsets):
try:
fields = byfieldset.pop(fieldset)
@@ -542,4 +544,3 @@
self._render_fields(fields, w, form)
self.render_child_forms(w, form, values)
w(u'</fieldset>')
-
--- a/web/views/forms.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/forms.py Wed Dec 16 11:23:48 2015 +0100
@@ -48,6 +48,9 @@
from warnings import warn
import time
+import inspect
+
+from six import text_type
from logilab.common import dictattr, tempattr
from logilab.common.decorators import iclassmethod, cached
@@ -257,7 +260,7 @@
editedfields = self._cw.form['_cw_fields']
except KeyError:
raise RequestError(self._cw._('no edited fields specified'))
- entityform = entity and self.field_by_name.im_func.func_code.co_argcount == 4 # XXX
+ entityform = entity and len(inspect.getargspec(self.field_by_name)) == 4 # XXX
for editedfield in splitstrip(editedfields):
try:
name, role = editedfield.split('-')
@@ -286,7 +289,7 @@
except ProcessFormError as exc:
errors.append((field, exc))
if errors:
- errors = dict((f.role_name(), unicode(ex)) for f, ex in errors)
+ errors = dict((f.role_name(), text_type(ex)) for f, ex in errors)
raise ValidationError(None, errors)
return processed
@@ -377,7 +380,7 @@
Warning: this method must be called only when all form fields are setup
"""
- for (rtype, role), eids in self.linked_to.iteritems():
+ for (rtype, role), eids in self.linked_to.items():
# if the relation is already setup by a form field, do not add it
# in a __linkto hidden to avoid setting it twice in the controller
try:
--- a/web/views/ibreadcrumbs.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/ibreadcrumbs.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,10 +18,12 @@
"""breadcrumbs components definition for CubicWeb web client"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
+from six import text_type
+
from logilab.mtconverter import xml_escape
from cubicweb import tags, uilib
@@ -141,7 +143,7 @@
xml_escape(url), xml_escape(uilib.cut(title, textsize))))
else:
textsize = self._cw.property_value('navigation.short-line-size')
- w(xml_escape(uilib.cut(unicode(part), textsize)))
+ w(xml_escape(uilib.cut(text_type(part), textsize)))
class BreadCrumbETypeVComponent(BreadCrumbEntityVComponent):
--- a/web/views/idownloadable.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/idownloadable.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,7 +20,9 @@
=====================================================
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six.moves import range
from logilab.mtconverter import BINARY_ENCODINGS, TransformError, xml_escape
from logilab.common.deprecation import class_renamed, deprecated
@@ -166,7 +168,7 @@
def call(self, **kwargs):
rset = self.cw_rset
- for i in xrange(len(rset)):
+ for i in range(len(rset)):
self.w(u'<div class="efile">')
self.wview(self.__regid__, rset, row=i, col=0, **kwargs)
self.w(u'</div>')
@@ -199,6 +201,3 @@
title = _('embedded html')
_embedding_tag = tags.iframe
-
-
-
--- a/web/views/json.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/json.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -18,7 +18,7 @@
"""json export views"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from cubicweb.uilib import rest_traceback
@@ -28,6 +28,7 @@
from cubicweb.web.application import anonymized_request
from cubicweb.web.views import basecontrollers, management
+
class JsonpController(basecontrollers.ViewController):
"""The jsonp controller is the same as a ViewController but :
@@ -49,7 +50,7 @@
self.warning("vid %s can't be used with jsonp controller, "
"falling back to jsonexport", vid)
self._cw.form['vid'] = 'jsonexport'
- else: # if no vid is specified, use jsonexport
+ else: # if no vid is specified, use jsonexport
self._cw.form['vid'] = 'jsonexport'
if self._cw.vreg.config['anonymize-jsonp-queries']:
with anonymized_request(self._cw):
@@ -59,12 +60,12 @@
def _get_json_data(self, rset):
json_data = super(JsonpController, self).publish(rset)
- if 'callback' in self._cw.form: # jsonp
+ if 'callback' in self._cw.form: # jsonp
json_padding = self._cw.form['callback'].encode('ascii')
# use ``application/javascript`` if ``callback`` parameter is
# provided, keep ``application/json`` otherwise
self._cw.set_content_type('application/javascript')
- json_data = b'%s(%s)' % (json_padding, json_data)
+ json_data = json_padding + b'(' + json_data + b')'
return json_data
@@ -85,13 +86,14 @@
indent = int(self._cw.form['_indent'])
else:
indent = None
- self.w(json_dumps(data, indent=indent))
+ # python's json.dumps escapes non-ascii characters
+ self.w(json_dumps(data, indent=indent).encode('ascii'))
class JsonRsetView(JsonMixIn, AnyRsetView):
"""dumps raw result set in JSON format"""
__regid__ = 'jsonexport'
- __select__ = any_rset() # means rset might be empty or have any shape
+ __select__ = any_rset() # means rset might be empty or have any shape
title = _('json-export-view')
def call(self):
@@ -105,7 +107,8 @@
The following additional metadata is added to each row :
- - ``__cwetype__`` : entity type
+ - ``cw_etype`` : entity type
+ - ``cw_source`` : source url
"""
__regid__ = 'ejsonexport'
__select__ = EntityView.__select__ | empty_rset()
@@ -114,12 +117,8 @@
def call(self):
entities = []
for entity in self.cw_rset.entities():
- entity.complete() # fetch all attributes
- # hack to add extra metadata
- entity.cw_attr_cache.update({
- '__cwetype__': entity.cw_etype,
- })
- entities.append(entity)
+ serializer = entity.cw_adapt_to('ISerializable')
+ entities.append(serializer.serialize())
self.wdata(entities)
@@ -148,4 +147,4 @@
'errmsg': errmsg,
'exclass': exclass,
'traceback': rest_traceback(excinfo, errmsg),
- })
+ })
--- a/web/views/magicsearch.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/magicsearch.py Wed Dec 16 11:23:48 2015 +0100
@@ -23,6 +23,8 @@
import re
from logging import getLogger
+from six import text_type
+
from yams.interfaces import IVocabularyConstraint
from rql import RQLSyntaxError, BadRQLQuery, parse
@@ -86,7 +88,7 @@
else:
# Only one possible translation, no ambiguity
if len(translation_set) == 1:
- relation.r_type = iter(translations[rtype]).next()
+ relation.r_type = next(iter(translations[rtype]))
# More than 1 possible translation => resolve it later
else:
ambiguous_nodes[relation] = (lhs.name, translation_set)
@@ -386,7 +388,7 @@
self.processors = sorted(processors, key=lambda x: x.priority)
def process_query(self, uquery):
- assert isinstance(uquery, unicode)
+ assert isinstance(uquery, text_type)
try:
procname, query = uquery.split(':', 1)
proc = self.by_name[procname.strip().lower()]
@@ -589,7 +591,7 @@
"""
schema = self._cw.vreg.schema
relations = set()
- untyped_dest_var = rqlvar_maker(defined=select.defined_vars).next()
+ untyped_dest_var = next(rqlvar_maker(defined=select.defined_vars))
# for each solution
# 1. find each possible relation
# 2. for each relation:
@@ -643,7 +645,7 @@
vocab_kwargs = {}
if rtype_incomplete_value:
vocab_rql += ', X %s LIKE %%(value)s' % user_rtype
- vocab_kwargs['value'] = '%s%%' % rtype_incomplete_value
+ vocab_kwargs['value'] = u'%s%%' % rtype_incomplete_value
vocab += [value for value, in
self._cw.execute(vocab_rql, vocab_kwargs)]
return sorted(set(vocab))
--- a/web/views/management.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/management.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""security management and error screens"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from logilab.mtconverter import xml_escape
@@ -137,7 +137,7 @@
# if excinfo is not None, it's probably not a bug
if excinfo is None:
return
- vcconf = self._cw.vreg.config.vc_config()
+ vcconf = self._cw.cnx.repo.get_versions()
w(u"<div>")
eversion = vcconf.get('cubicweb', self._cw._('no version information'))
# NOTE: tuple wrapping needed since eversion is itself a tuple
@@ -169,7 +169,7 @@
binfo += u'\n\n:URL: %s\n' % req.url()
if not '__bugreporting' in req.form:
binfo += u'\n:form params:\n'
- binfo += u'\n'.join(u' * %s = %s' % (k, v) for k, v in req.form.iteritems())
+ binfo += u'\n'.join(u' * %s = %s' % (k, v) for k, v in req.form.items())
binfo += u'\n\n:CubicWeb version: %s\n' % (eversion,)
for pkg, pkgversion in cubes:
binfo += u":Cube %s version: %s\n" % (pkg, pkgversion)
--- a/web/views/navigation.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/navigation.py Wed Dec 16 11:23:48 2015 +0100
@@ -46,10 +46,12 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from datetime import datetime
+from six import text_type
+
from rql.nodes import VariableRef, Constant
from logilab.mtconverter import xml_escape
@@ -192,10 +194,10 @@
return entity.printable_value(attrname, format='text/plain')
elif col is None: # smart links disabled.
def index_display(row):
- return unicode(row)
+ return text_type(row)
elif self._cw.vreg.schema.eschema(rset.description[0][col]).final:
def index_display(row):
- return unicode(rset[row][col])
+ return text_type(rset[row][col])
else:
def index_display(row):
return rset.get_entity(row, col).view('text')
--- a/web/views/owl.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/owl.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,7 +19,9 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six.moves import range
from logilab.mtconverter import TransformError, xml_escape
@@ -166,7 +168,7 @@
def call(self):
self.w(OWL_OPENING_ROOT % {'appid': self._cw.vreg.schema.name})
- for i in xrange(self.cw_rset.rowcount):
+ for i in range(self.cw_rset.rowcount):
self.cell_call(i, 0)
self.w(OWL_CLOSING_ROOT)
--- a/web/views/plots.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/plots.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,10 @@
"""basic plot views"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six import add_metaclass
+from six.moves import range
from logilab.common.date import datetime2ticks
from logilab.common.deprecation import class_deprecated
@@ -83,9 +86,10 @@
def _render(self, *args, **kwargs):
raise NotImplementedError
+
+@add_metaclass(class_deprecated)
class FlotPlotWidget(PlotWidget):
"""PlotRenderer widget using Flot"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.14] cubicweb.web.views.plots module is deprecated, use the jqplot cube instead'
onload = u"""
var fig = jQuery('#%(figid)s');
@@ -117,7 +121,7 @@
if req.ie_browser():
req.add_js('excanvas.js')
req.add_js(('jquery.flot.js', 'cubicweb.flot.js'))
- figid = u'figure%s' % req.varmaker.next()
+ figid = u'figure%s' % next(req.varmaker)
plotdefs = []
plotdata = []
self.w(u'<div id="%s" style="width: %spx; height: %spx;"></div>' %
@@ -137,8 +141,8 @@
'dateformat': '"%s"' % fmt})
+@add_metaclass(class_deprecated)
class PlotView(baseviews.AnyRsetView):
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.14] cubicweb.web.views.plots module is deprecated, use the jqplot cube instead'
__regid__ = 'plot'
title = _('generic plot')
@@ -154,7 +158,7 @@
abscissa = [row[0] for row in self.cw_rset]
plots = []
nbcols = len(self.cw_rset.rows[0])
- for col in xrange(1, nbcols):
+ for col in range(1, nbcols):
data = [row[col] for row in self.cw_rset]
plots.append(filterout_nulls(abscissa, data))
plotwidget = FlotPlotWidget(varnames, plots, timemode=self.timemode)
--- a/web/views/primary.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/primary.py Wed Dec 16 11:23:48 2015 +0100
@@ -38,7 +38,7 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
--- a/web/views/pyviews.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/pyviews.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,6 +19,9 @@
"""
__docformat__ = "restructuredtext en"
+from six import text_type
+from six.moves import range
+
from cubicweb.view import View
from cubicweb.predicates import match_kwargs
from cubicweb.web.views import tableview
@@ -38,7 +41,7 @@
w(self.empty_cell_content)
def render_cell(self, w, rownum):
- w(unicode(self.data[rownum][self.colid]))
+ w(text_type(self.data[rownum][self.colid]))
class PyValTableView(tableview.TableMixIn, View):
@@ -100,7 +103,7 @@
def build_column_renderers(self):
return [self.column_renderer(colid)
- for colid in xrange(len(self.pyvalue[0]))]
+ for colid in range(len(self.pyvalue[0]))]
def facets_form(self, mainvar=None):
return None # not supported
--- a/web/views/rdf.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/rdf.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,9 @@
"""base xml and rss views"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six.moves import range
from yams import xy
@@ -56,7 +58,7 @@
graph.bind('cw', CW)
for prefix, xmlns in xy.XY.prefixes.items():
graph.bind(prefix, rdflib.Namespace(xmlns))
- for i in xrange(self.cw_rset.rowcount):
+ for i in range(self.cw_rset.rowcount):
entity = self.cw_rset.complete_entity(i, 0)
self.entity2graph(graph, entity)
self.w(graph.serialize(format=self.format))
--- a/web/views/reledit.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/reledit.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,7 +20,7 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
import copy
from warnings import warn
@@ -259,7 +259,7 @@
elif action == 'add':
add_etype = self._compute_ttypes(rschema, role)[0]
_new_entity = self._cw.vreg['etypes'].etype_class(add_etype)(self._cw)
- _new_entity.eid = self._cw.varmaker.next()
+ _new_entity.eid = next(self._cw.varmaker)
edit_entity = _new_entity
# XXX see forms.py ~ 276 and entities.linked_to method
# is there another way?
@@ -292,7 +292,7 @@
cwtarget='eformframe', cssclass='releditForm',
**formargs)
# pass reledit arguments
- for pname, pvalue in event_args.iteritems():
+ for pname, pvalue in event_args.items():
form.add_hidden('__reledit|' + pname, pvalue)
# handle buttons
if form.form_buttons: # edition, delete
@@ -402,4 +402,3 @@
assert args['reload'].startswith('http')
view = req.vreg['views'].select('reledit', req, rset=rset, rtype=args['rtype'])
return self._call_view(view, **args)
-
--- a/web/views/schema.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/schema.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""Specific views for schema related entities"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from itertools import cycle
@@ -26,6 +26,8 @@
import os, os.path as osp
import codecs
+from six import text_type
+
from logilab.common.graph import GraphGenerator, DotBackend
from logilab.common.ureports import Section, Table
from logilab.common.registry import yes
@@ -114,7 +116,7 @@
def grouped_permissions_table(self, rschema):
# group relation definitions with identical permissions
perms = {}
- for rdef in rschema.rdefs.itervalues():
+ for rdef in rschema.rdefs.values():
rdef_perms = []
for action in rdef.ACTIONS:
groups = sorted(rdef.get_groups(action))
@@ -131,7 +133,7 @@
_ = self._cw._
w(u'<div style="margin: 0px 1.5em">')
tmpl = u'<strong>%s</strong> %s <strong>%s</strong>'
- for perm, rdefs in perms.iteritems():
+ for perm, rdefs in perms.items():
w(u'<div>%s</div>' % u', '.join(
tmpl % (_(s.type), _(rschema.type), _(o.type)) for s, o in rdefs))
# accessing rdef from previous loop by design: only used to get
@@ -279,7 +281,7 @@
def cell_call(self, row, col):
defaultval = self.cw_rset.rows[row][col]
if defaultval is not None:
- self.w(unicode(self.cw_rset.rows[row][col].unzpickle()))
+ self.w(text_type(self.cw_rset.rows[row][col].unzpickle()))
class CWETypeRelationCardinalityCell(baseviews.FinalView):
__regid__ = 'etype-rel-cardinality-cell'
@@ -487,7 +489,7 @@
entity = self.cw_rset.get_entity(row, col)
rschema = self._cw.vreg.schema.rschema(entity.rtype.name)
rdef = rschema.rdefs[(entity.stype.name, entity.otype.name)]
- constraints = [xml_escape(unicode(c)) for c in getattr(rdef, 'constraints')]
+ constraints = [xml_escape(text_type(c)) for c in getattr(rdef, 'constraints')]
self.w(u'<br/>'.join(constraints))
class CWAttributeOptionsCell(EntityView):
@@ -557,8 +559,9 @@
def __init__(self, visitor, cw):
self.visitor = visitor
self.cw = cw
- self.nextcolor = cycle( ('#ff7700', '#000000',
- '#ebbc69', '#888888') ).next
+ self._cycle = iter(cycle(('#ff7700', '#000000', '#ebbc69', '#888888')))
+ self.nextcolor = lambda: next(self._cycle)
+
self.colors = {}
def node_properties(self, eschema):
--- a/web/views/sessions.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/sessions.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,20 +19,28 @@
__docformat__ = "restructuredtext en"
from time import time
+from logging import getLogger
-from cubicweb import RepositoryError, Unauthorized, BadConnectionId
-from cubicweb.web import InvalidSession, component
+from logilab.common.registry import RegistrableObject
+
+from cubicweb import RepositoryError, Unauthorized, BadConnectionId, set_log_methods
+from cubicweb.predicates import yes
+from cubicweb.web import InvalidSession
+
+from cubicweb.web.views import authentication
-class AbstractSessionManager(component.Component):
+class AbstractSessionManager(RegistrableObject):
"""manage session data associated to a session identifier"""
__abstract__ = True
+ __select__ = yes()
+ __registry__ = 'sessions'
__regid__ = 'sessionmanager'
def __init__(self, repo):
vreg = repo.vreg
self.session_time = vreg.config['http-session-time'] or None
- self.authmanager = vreg['components'].select('authmanager', repo=repo)
+ self.authmanager = authentication.RepositoryAuthenticationManager(repo)
interval = (self.session_time or 0) / 2.
if vreg.config.anonymous_user()[0] is not None:
self.cleanup_anon_session_time = vreg.config['cleanup-anonymous-session-time'] or 5 * 60
@@ -53,15 +61,7 @@
closed, total = 0, 0
for session in self.current_sessions():
total += 1
- try:
- last_usage_time = session.cnx.check()
- except AttributeError:
- last_usage_time = session.mtime
- except BadConnectionId:
- self.close_session(session)
- closed += 1
- continue
-
+ last_usage_time = session.mtime
no_use_time = (time() - last_usage_time)
if session.anonymous_session:
if no_use_time >= self.cleanup_anon_session_time:
@@ -95,11 +95,14 @@
raise NotImplementedError()
+set_log_methods(AbstractSessionManager, getLogger('cubicweb.sessionmanager'))
+
+
class InMemoryRepositorySessionManager(AbstractSessionManager):
"""manage session data associated to a session identifier"""
def __init__(self, *args, **kwargs):
- AbstractSessionManager.__init__(self, *args, **kwargs)
+ super(InMemoryRepositorySessionManager, self).__init__(*args, **kwargs)
# XXX require a RepositoryAuthenticationManager which violates
# authenticate interface by returning a session instead of a user
#assert isinstance(self.authmanager, RepositoryAuthenticationManager)
--- a/web/views/sparql.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/sparql.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,9 @@
"""SPARQL integration"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six.moves import range
from yams import xy
from rql import TypeResolverException
@@ -111,7 +113,7 @@
rqlst = self.cw_rset.syntax_tree().children[0]
varnames = [var.name for var in rqlst.selection]
results = E.results()
- for rowidx in xrange(len(self.cw_rset)):
+ for rowidx in range(len(self.cw_rset)):
result = E.result()
for colidx, varname in enumerate(varnames):
result.append(self.cell_binding(rowidx, colidx, varname))
@@ -140,4 +142,4 @@
def registration_callback(vreg):
if Sparql2rqlTranslator is not None:
- vreg.register_all(globals().itervalues(), __name__)
+ vreg.register_all(globals().values(), __name__)
--- a/web/views/startup.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/startup.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,7 +22,7 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from logilab.common.textutils import unormalize
from logilab.common.deprecation import deprecated
@@ -106,7 +106,7 @@
def entity_types_table(self, eschemas):
infos = sorted(self.entity_types(eschemas),
- key=lambda (l,a,e): unormalize(l))
+ key=lambda t: unormalize(t[0]))
q, r = divmod(len(infos), 2)
if r:
infos.append( (None, ' ', ' ') )
@@ -172,4 +172,3 @@
@deprecated('[3.11] display_folders method is deprecated, backport it if needed')
def display_folders(self):
return 'Folder' in self._cw.vreg.schema and self._cw.execute('Any COUNT(X) WHERE X is Folder')[0][0]
-
--- a/web/views/staticcontrollers.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/staticcontrollers.py Wed Dec 16 11:23:48 2015 +0100
@@ -33,7 +33,7 @@
from logging import getLogger
from cubicweb import Forbidden
-from cubicweb.web import NotFound
+from cubicweb.web import NotFound, Redirect
from cubicweb.web.http_headers import generateDateTime
from cubicweb.web.controller import Controller
from cubicweb.web.views.urlrewrite import URLRewriter
@@ -66,9 +66,10 @@
if not debugmode:
# XXX: Don't provide additional resource information to error responses
#
- # the HTTP RFC recommands not going further than 1 year ahead
- expires = datetime.now() + timedelta(days=6*30)
+ # the HTTP RFC recommends not going further than 1 year ahead
+ expires = datetime.now() + timedelta(seconds=self.max_age(path))
self._cw.set_header('Expires', generateDateTime(mktime(expires.timetuple())))
+ self._cw.set_header('Cache-Control', 'max-age=%s' % self.max_age(path))
# XXX system call to os.stats could be cached once and for all in
# production mode (where static files are not expected to change)
@@ -140,7 +141,7 @@
"""return the filepath that will be used to cache concatenation of `paths`
"""
_, ext = osp.splitext(paths[0])
- fname = 'cache_concat_' + hashlib.md5(';'.join(paths)).hexdigest() + ext
+ fname = 'cache_concat_' + hashlib.md5((';'.join(paths)).encode('ascii')).hexdigest() + ext
return osp.join(self.config.appdatahome, 'uicache', fname)
def concat_cached_filepath(self, paths):
@@ -167,7 +168,7 @@
with open(osp.join(dirpath, rid), 'rb') as source:
for line in source:
f.write(line)
- f.write('\n')
+ f.write(b'\n')
f.close()
except:
os.remove(tmpfile)
@@ -200,11 +201,13 @@
paths = relpath[len(self.data_modconcat_basepath):].split(',')
filepath = self.concat_files_registry.concat_cached_filepath(paths)
else:
- # skip leading '/data/' and url params
- if relpath.startswith(self.base_datapath):
- prefix = self.base_datapath
- else:
+ if not relpath.startswith(self.base_datapath):
+ # /data/foo, redirect to /data/{hash}/foo
prefix = 'data/'
+ relpath = relpath[len(prefix):]
+ raise Redirect(self._cw.data_url(relpath), 302)
+ # skip leading '/data/{hash}/' and url params
+ prefix = self.base_datapath
relpath = relpath[len(prefix):]
relpath = relpath.split('?', 1)[0]
dirpath, rid = config.locate_resource(relpath)
--- a/web/views/tableview.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/tableview.py Wed Dec 16 11:23:48 2015 +0100
@@ -42,7 +42,7 @@
.. autoclass:: cubicweb.web.views.tableview.TableLayout
:members:
-There is by default only on table layout, using the 'table_layout' identifier,
+There is by default only one table layout, using the 'table_layout' identifier,
that is referenced by table views
:attr:`cubicweb.web.views.tableview.TableMixIn.layout_id`. If you want to
customize the look and feel of your table, you can either replace the default
@@ -52,21 +52,24 @@
Notice you can gives options to the layout using a `layout_args` dictionary on
your class.
-If you can still find a view that suit your needs, you should take a look at the
+If you still can't find a view that suit your needs, you should take a look at the
class below that is the common abstract base class for the three views defined
-above and implements you own class.
+above and implement your own class.
.. autoclass:: cubicweb.web.views.tableview.TableMixIn
:members:
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
from copy import copy
from types import MethodType
+from six import string_types, add_metaclass, create_bound_method
+from six.moves import range
+
from logilab.mtconverter import xml_escape
from logilab.common.decorators import cachedproperty
from logilab.common.deprecation import class_deprecated
@@ -162,7 +165,7 @@
def __init__(self, req, view, **kwargs):
super(TableLayout, self).__init__(req, **kwargs)
- for key, val in self.cw_extra_kwargs.items():
+ for key, val in list(self.cw_extra_kwargs.items()):
if hasattr(self.__class__, key) and not key[0] == '_':
setattr(self, key, val)
self.cw_extra_kwargs.pop(key)
@@ -225,7 +228,7 @@
def render_table_body(self, w, colrenderers):
w(u'<tbody>')
- for rownum in xrange(self.view.table_size):
+ for rownum in range(self.view.table_size):
self.render_row(w, rownum, colrenderers)
w(u'</tbody>')
@@ -284,7 +287,7 @@
attrs = renderer.attributes.copy()
if renderer.sortable:
sortvalue = renderer.sortvalue(rownum)
- if isinstance(sortvalue, basestring):
+ if isinstance(sortvalue, string_types):
sortvalue = sortvalue[:self.sortvalue_limit]
if sortvalue is not None:
attrs[u'cubicweb:sortvalue'] = js_dumps(sortvalue)
@@ -646,10 +649,10 @@
# compute displayed columns
if self.displaycols is None:
if headers is not None:
- displaycols = range(len(headers))
+ displaycols = list(range(len(headers)))
else:
rqlst = self.cw_rset.syntax_tree()
- displaycols = range(len(rqlst.children[0].selection))
+ displaycols = list(range(len(rqlst.children[0].selection)))
else:
displaycols = self.displaycols
# compute table headers
@@ -723,7 +726,7 @@
for aname, member in[('renderfunc', renderfunc),
('sortfunc', sortfunc)]:
if isinstance(member, MethodType):
- member = MethodType(member.im_func, acopy, acopy.__class__)
+ member = create_bound_method(member.__func__, acopy)
setattr(acopy, aname, member)
return acopy
finally:
@@ -918,13 +921,13 @@
################################################################################
+@add_metaclass(class_deprecated)
class TableView(AnyRsetView):
"""The table view accepts any non-empty rset. It uses introspection on the
result set to compute column names and the proper way to display the cells.
It is however highly configurable and accepts a wealth of options.
"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.14] %(cls)s is deprecated'
__regid__ = 'table'
title = _('table')
@@ -977,9 +980,9 @@
if 'displaycols' in self._cw.form:
displaycols = [int(idx) for idx in self._cw.form['displaycols']]
elif headers is not None:
- displaycols = range(len(headers))
+ displaycols = list(range(len(headers)))
else:
- displaycols = range(len(self.cw_rset.syntax_tree().children[0].selection))
+ displaycols = list(range(len(self.cw_rset.syntax_tree().children[0].selection)))
return displaycols
def _setup_tablesorter(self, divid):
@@ -1143,7 +1146,7 @@
else:
column.append_renderer(subvid or 'incontext', colindex)
if cellattrs and colindex in cellattrs:
- for name, value in cellattrs[colindex].iteritems():
+ for name, value in cellattrs[colindex].items():
column.add_attr(name, value)
# add column
columns.append(column)
@@ -1184,8 +1187,8 @@
title = _('editable-table')
+@add_metaclass(class_deprecated)
class CellView(EntityView):
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.14] %(cls)s is deprecated'
__regid__ = 'cell'
__select__ = nonempty_rset()
@@ -1271,6 +1274,7 @@
finalview = 'editable-final'
+@add_metaclass(class_deprecated)
class EntityAttributesTableView(EntityView):
"""This table displays entity attributes in a table and allow to set a
specific method to help building cell content for each attribute as well as
@@ -1282,7 +1286,6 @@
Table will render column header using the method header_for_COLNAME if
defined otherwise COLNAME will be used.
"""
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.14] %(cls)s is deprecated'
__abstract__ = True
columns = ()
@@ -1298,7 +1301,7 @@
self.w(u'<table class="%s">' % self.table_css)
self.table_header(sample)
self.w(u'<tbody>')
- for row in xrange(self.cw_rset.rowcount):
+ for row in range(self.cw_rset.rowcount):
self.cell_call(row=row, col=0)
self.w(u'</tbody>')
self.w(u'</table>')
@@ -1333,4 +1336,3 @@
colname = self._cw._(column)
self.w(u'<th>%s</th>' % xml_escape(colname))
self.w(u'</tr></thead>\n')
-
--- a/web/views/tabs.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/tabs.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,9 @@
"""base classes to handle tabbed views"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six import string_types
from logilab.common.deprecation import class_renamed
from logilab.mtconverter import xml_escape
@@ -114,7 +116,7 @@
active_tab = uilib.domid(default_tab)
viewsvreg = self._cw.vreg['views']
for tab in tabs:
- if isinstance(tab, basestring):
+ if isinstance(tab, string_types):
tabid, tabkwargs = tab, {}
else:
tabid, tabkwargs = tab
--- a/web/views/timetable.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/timetable.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,9 @@
"""html timetable views"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six.moves import range
from logilab.mtconverter import xml_escape
from logilab.common.date import ONEDAY, date_range, todatetime
@@ -51,7 +53,7 @@
users = []
users_max = {}
# XXX: try refactoring with calendar.py:OneMonthCal
- for row in xrange(self.cw_rset.rowcount):
+ for row in range(self.cw_rset.rowcount):
task = self.cw_rset.get_entity(row, 0)
icalendarable = task.cw_adapt_to('ICalendarable')
if len(self.cw_rset[row]) > 1 and self.cw_rset.description[row][1] == 'CWUser':
@@ -88,7 +90,7 @@
rows = []
# colors here are class names defined in cubicweb.css
- colors = ["col%x" % i for i in xrange(12)]
+ colors = ["col%x" % i for i in range(12)]
next_color_index = 0
visited_tasks = {} # holds a description of a task for a user
--- a/web/views/treeview.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/treeview.py Wed Dec 16 11:23:48 2015 +0100
@@ -20,7 +20,7 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from warnings import warn
@@ -140,7 +140,7 @@
ajaxargs = json.loads(form.pop('morekwargs'))
# got unicode & python keywords must be strings
morekwargs.update(dict((str(k), v)
- for k, v in ajaxargs.iteritems()))
+ for k, v in ajaxargs.items()))
toplevel_thru_ajax = form.pop('treeview_top', False) or initial_thru_ajax
toplevel = toplevel_thru_ajax or (initial_load and not form.get('fname'))
return subvid, treeid, toplevel_thru_ajax, toplevel
--- a/web/views/uicfg.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/uicfg.py Wed Dec 16 11:23:48 2015 +0100
@@ -57,6 +57,8 @@
from warnings import warn
+from six import string_types
+
from cubicweb import neg_role
from cubicweb.rtags import (RelationTags, RelationTagsBool, RelationTagsSet,
RelationTagsDict, NoTargetRelationTagsDict,
@@ -267,7 +269,7 @@
if not 'inlined' in sectdict:
sectdict['inlined'] = sectdict['main']
# recompute formsections and set it to avoid recomputing
- for formtype, section in sectdict.iteritems():
+ for formtype, section in sectdict.items():
formsections.add('%s_%s' % (formtype, section))
def tag_relation(self, key, formtype, section):
@@ -302,7 +304,7 @@
rtags[section] = value
cls = self.tag_container_cls
rtags = cls('_'.join([section,value])
- for section,value in rtags.iteritems())
+ for section,value in rtags.items())
return rtags
def get(self, *key):
@@ -650,7 +652,7 @@
self.tag_relation((sschema, rschema, oschema, role), True)
def _tag_etype_attr(self, etype, attr, desttype='*', *args, **kwargs):
- if isinstance(attr, basestring):
+ if isinstance(attr, string_types):
attr, role = attr, 'subject'
else:
attr, role = attr
@@ -687,5 +689,5 @@
def registration_callback(vreg):
- vreg.register_all(globals().itervalues(), __name__)
+ vreg.register_all(globals().values(), __name__)
indexview_etype_section.init(vreg.schema)
--- a/web/views/undohistory.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/undohistory.py Wed Dec 16 11:23:48 2015 +0100
@@ -17,7 +17,7 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from logilab.common.registry import Predicate
@@ -46,7 +46,7 @@
def __str__(self):
return '%s(%s)' % (self.__class__.__name__, ', '.join(
- "%s=%v" % (str(k), str(v)) for k, v in kwargs.iteritems() ))
+ "%s=%v" % (str(k), str(v)) for k, v in kwargs.items() ))
def __call__(self, cls, req, tx_action=None, **kwargs):
# tx_action is expected to be a transaction.AbstractAction
--- a/web/views/urlpublishing.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/urlpublishing.py Wed Dec 16 11:23:48 2015 +0100
@@ -60,7 +60,7 @@
from rql import TypeResolverException
from cubicweb import RegistryException
-from cubicweb.web import NotFound, Redirect, component
+from cubicweb.web import NotFound, Redirect, component, views
class PathDontMatch(Exception):
@@ -201,18 +201,14 @@
return self.handle_etype_attr(req, cls, attrname, value)
return self.handle_etype(req, cls)
- def set_vid_for_rset(self, req, cls, rset):# cls is there to ease overriding
+ def set_vid_for_rset(self, req, cls, rset): # cls is there to ease overriding
if rset.rowcount == 0:
raise NotFound()
- # we've to set a default vid here, since vid_from_rset may try to use a
- # table view if fetch_rql include some non final relation
- if rset.rowcount == 1:
- req.form.setdefault('vid', 'primary')
- else: # rset.rowcount >= 1
- if len(rset.column_types(0)) > 1:
- req.form.setdefault('vid', 'list')
- else:
- req.form.setdefault('vid', 'sameetypelist')
+ if 'vid' not in req.form:
+ # check_table=False tells vid_from_rset not to try to use a table view if fetch_rql
+ # include some non final relation
+ req.form['vid'] = views.vid_from_rset(req, rset, req.vreg.schema,
+ check_table=False)
def handle_etype(self, req, cls):
rset = req.execute(cls.fetch_rql(req.user))
--- a/web/views/urlrewrite.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/urlrewrite.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,6 +19,8 @@
import re
+from six import string_types, add_metaclass
+
from cubicweb.uilib import domid
from cubicweb.appobject import AppObject
@@ -51,6 +53,7 @@
return super(metarewriter, mcs).__new__(mcs, name, bases, classdict)
+@add_metaclass(metarewriter)
class URLRewriter(AppObject):
"""Base class for URL rewriters.
@@ -64,7 +67,6 @@
should be tried first. The higher the priority is, the earlier the
rewriter will be tried.
"""
- __metaclass__ = metarewriter
__registry__ = 'urlrewriting'
__abstract__ = True
priority = 1
@@ -122,14 +124,14 @@
required_groups = None
if required_groups and not req.user.matching_groups(required_groups):
continue
- if isinstance(inputurl, basestring):
+ if isinstance(inputurl, string_types):
if inputurl == uri:
req.form.update(infos)
break
elif inputurl.match(uri): # it's a regexp
# XXX what about i18n? (vtitle for instance)
for param, value in infos.items():
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
req.form[param] = inputurl.sub(value, uri)
else:
req.form[param] = value
@@ -222,7 +224,7 @@
required_groups = None
if required_groups and not req.user.matching_groups(required_groups):
continue
- if isinstance(inputurl, basestring):
+ if isinstance(inputurl, string_types):
if inputurl == uri:
return callback(inputurl, uri, req, self._cw.vreg.schema)
elif inputurl.match(uri): # it's a regexp
--- a/web/views/vcard.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/vcard.py Wed Dec 16 11:23:48 2015 +0100
@@ -23,7 +23,7 @@
from cubicweb.predicates import is_instance
from cubicweb.view import EntityView
-_ = unicode
+from cubicweb import _
VCARD_PHONE_TYPES = {'home': 'HOME', 'office': 'WORK', 'mobile': 'CELL', 'fax': 'FAX'}
--- a/web/views/wdoc.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/wdoc.py Wed Dec 16 11:23:48 2015 +0100
@@ -35,7 +35,7 @@
from cubicweb.view import StartupView
from cubicweb.uilib import rest_publish
from cubicweb.web import NotFound, action
-_ = unicode
+from cubicweb import _
# table of content management #################################################
@@ -73,7 +73,7 @@
def build_toc(config):
alltocfiles = reversed(tuple(config.locate_all_files('toc.xml')))
- maintoc = parse(alltocfiles.next()).getroot()
+ maintoc = parse(next(alltocfiles)).getroot()
maintoc.parent = None
index = {}
build_toc_index(maintoc, index)
@@ -229,4 +229,3 @@
def url(self):
return self._cw.build_url('doc/about')
-
--- a/web/views/workflow.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/workflow.py Wed Dec 16 11:23:48 2015 +0100
@@ -22,11 +22,13 @@
"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
import os
from warnings import warn
+from six import add_metaclass
+
from logilab.mtconverter import xml_escape
from logilab.common.graph import escape
from logilab.common.deprecation import class_deprecated
@@ -116,7 +118,7 @@
'changestate', self._cw, entity=entity, transition=transition,
redirect_path=self.redirectpath(entity), **kwargs)
trinfo = self._cw.vreg['etypes'].etype_class('TrInfo')(self._cw)
- trinfo.eid = self._cw.varmaker.next()
+ trinfo.eid = next(self._cw.varmaker)
subform = self._cw.vreg['forms'].select('edition', self._cw, entity=trinfo,
mainform=False)
subform.field_by_name('wf_info_for', 'subject').value = entity.eid
@@ -429,8 +431,8 @@
return WorkflowDotPropsHandler(self._cw)
+@add_metaclass(class_deprecated)
class TmpPngView(TmpFileViewMixin, EntityView):
- __metaclass__ = class_deprecated
__deprecation_warning__ = '[3.18] %(cls)s is deprecated'
__regid__ = 'tmppng'
__select__ = match_form_params('tmpfile')
--- a/web/views/xbel.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/xbel.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,9 @@
"""xbel views"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
+
+from six.moves import range
from logilab.mtconverter import xml_escape
@@ -42,7 +44,7 @@
self.w(u'<!DOCTYPE xbel PUBLIC "+//IDN python.org//DTD XML Bookmark Exchange Language 1.0//EN//XML" "http://www.python.org/topics/xml/dtds/xbel-1.0.dtd">')
self.w(u'<xbel version="1.0">')
self.w(u'<title>%s</title>' % self._cw._('bookmarks'))
- for i in xrange(self.cw_rset.rowcount):
+ for i in range(self.cw_rset.rowcount):
self.cell_call(i, 0)
self.w(u"</xbel>")
@@ -65,4 +67,3 @@
def url(self, entity):
return entity.actual_url()
-
--- a/web/views/xmlrss.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/views/xmlrss.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,11 +18,13 @@
"""base xml and rss views"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
from base64 import b64encode
from time import timezone
+from six.moves import range
+
from logilab.mtconverter import xml_escape
from cubicweb.predicates import (is_instance, non_final_entity, one_line_rset,
@@ -64,7 +66,7 @@
"""display a list of entities by calling their <item_vid> view"""
self.w(u'<?xml version="1.0" encoding="%s"?>\n' % self._cw.encoding)
self.w(u'<%s size="%s">\n' % (self.xml_root, len(self.cw_rset)))
- for i in xrange(self.cw_rset.rowcount):
+ for i in range(self.cw_rset.rowcount):
self.cell_call(i, 0)
self.w(u'</%s>\n' % self.xml_root)
@@ -256,7 +258,7 @@
def call(self):
"""display a list of entities by calling their <item_vid> view"""
self._open()
- for i in xrange(self.cw_rset.rowcount):
+ for i in range(self.cw_rset.rowcount):
self.cell_call(i, 0)
self._close()
--- a/web/webconfig.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/webconfig.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,7 +18,7 @@
"""web ui configuration for cubicweb instances"""
__docformat__ = "restructuredtext en"
-_ = unicode
+from cubicweb import _
import os
import hmac
@@ -26,6 +26,8 @@
from os.path import join, exists, split, isdir
from warnings import warn
+from six import text_type
+
from logilab.common.decorators import cached, cachedproperty
from logilab.common.deprecation import deprecated
from logilab.common.configuration import merge_options
@@ -280,18 +282,7 @@
continue
yield key, pdef
- # don't use @cached: we want to be able to disable it while this must still
- # be cached
- def repository(self, vreg=None):
- """return the instance's repository object"""
- try:
- return self.__repo
- except AttributeError:
- from cubicweb.repoapi import get_repository
- repo = get_repository(config=self, vreg=vreg)
- self.__repo = repo
- return repo
-
+ @deprecated('[3.22] call req.cnx.repo.get_versions() directly')
def vc_config(self):
return self.repository().get_versions()
@@ -305,7 +296,7 @@
user = self['anonymous-user'] or None
passwd = self['anonymous-password']
if user:
- user = unicode(user)
+ user = text_type(user)
except KeyError:
user, passwd = None, None
except UnicodeDecodeError:
@@ -317,17 +308,17 @@
"""This random key/salt is used to sign content to be sent back by
browsers, eg. in the error report form.
"""
- return str(uuid4())
+ return str(uuid4()).encode('ascii')
def sign_text(self, text):
"""sign some text for later checking"""
# hmac.new expect bytes
- if isinstance(text, unicode):
+ if isinstance(text, text_type):
text = text.encode('utf-8')
# replace \r\n so we do not depend on whether a browser "reencode"
# original message using \r\n or not
return hmac.new(self._instance_salt,
- text.strip().replace('\r\n', '\n')).hexdigest()
+ text.strip().replace(b'\r\n', b'\n')).hexdigest()
def check_text_sign(self, text, signature):
"""check the text signature is equal to the given signature"""
@@ -472,7 +463,7 @@
staticdir = join(staticdir, rdir)
if not isdir(staticdir) and 'w' in mode:
os.makedirs(staticdir)
- return file(join(staticdir, filename), mode)
+ return open(join(staticdir, filename), mode)
def static_file_add(self, rpath, data):
stream = self.static_file_open(rpath)
--- a/web/webctl.py Tue Dec 15 19:05:41 2015 +0100
+++ b/web/webctl.py Wed Dec 16 11:23:48 2015 +0100
@@ -18,6 +18,7 @@
"""cubicweb-ctl commands and command handlers common to twisted/modpython
web configuration
"""
+from __future__ import print_function
__docformat__ = "restructuredtext en"
@@ -44,7 +45,7 @@
def bootstrap(self, cubes, automatic=False, inputlevel=0):
"""bootstrap this configuration"""
if not automatic:
- print '\n' + underline_title('Generic web configuration')
+ print('\n' + underline_title('Generic web configuration'))
config = self.config
config.input_config('web', inputlevel)
if ASK.confirm('Allow anonymous access ?', False):
@@ -87,8 +88,8 @@
copy(osp.join(resource_dir, resource_path), dest_resource)
# handle md5 version subdirectory
linkdir(dest, osp.join(dest, config.instance_md5_version()))
- print ('You can use apache rewrite rule below :\n'
- 'RewriteRule ^/data/(.*) %s/$1 [L]' % dest)
+ print('You can use apache rewrite rule below :\n'
+ 'RewriteRule ^/data/(.*) %s/$1 [L]' % dest)
def _datadirs(self, config, repo=None):
if repo is None:
--- a/wsgi/__init__.py Tue Dec 15 19:05:41 2015 +0100
+++ b/wsgi/__init__.py Wed Dec 16 11:23:48 2015 +0100
@@ -27,11 +27,9 @@
__docformat__ = "restructuredtext en"
from email import message, message_from_string
-from Cookie import SimpleCookie
-from StringIO import StringIO
-from cgi import parse_header
from pprint import pformat as _pformat
+from six.moves.http_cookies import SimpleCookie
def pformat(obj):
"""pretty prints `obj` if possible"""
--- a/wsgi/handler.py Tue Dec 15 19:05:41 2015 +0100
+++ b/wsgi/handler.py Wed Dec 16 11:23:48 2015 +0100
@@ -19,7 +19,9 @@
__docformat__ = "restructuredtext en"
-from itertools import chain, repeat, izip
+from itertools import chain, repeat
+
+from six.moves import zip
from cubicweb import AuthenticationError
from cubicweb.web import DirectResponse
@@ -78,7 +80,7 @@
def __init__(self, code, req, body=None):
text = STATUS_CODE_TEXT.get(code, 'UNKNOWN STATUS CODE')
self.status = '%s %s' % (code, text)
- self.headers = list(chain(*[izip(repeat(k), v)
+ self.headers = list(chain(*[zip(repeat(k), v)
for k, v in req.headers_out.getAllRawHeaders()]))
self.headers = [(str(k), str(v)) for k, v in self.headers]
if body:
--- a/wsgi/request.py Tue Dec 15 19:05:41 2015 +0100
+++ b/wsgi/request.py Wed Dec 16 11:23:48 2015 +0100
@@ -27,13 +27,12 @@
import tempfile
-from StringIO import StringIO
-from urllib import quote
-from urlparse import parse_qs
-from warnings import warn
+from io import BytesIO
+
+from six.moves.urllib.parse import parse_qs
from cubicweb.multipart import (
- copy_file, parse_form_data, MultipartError, parse_options_header)
+ copy_file, parse_form_data, parse_options_header)
from cubicweb.web import RequestError
from cubicweb.web.request import CubicWebRequestBase
from cubicweb.wsgi import pformat, normalize_header
@@ -59,7 +58,7 @@
length = 0
# wsgi.input is not seekable, so copy the request contents to a temporary file
if length < 100000:
- self.content = StringIO()
+ self.content = BytesIO()
else:
self.content = tempfile.TemporaryFile()
copy_file(environ['wsgi.input'], self.content, maxread=length)
@@ -82,7 +81,7 @@
headers= headers_in)
self.content = environ['wsgi.input']
if files is not None:
- for key, part in files.iteritems():
+ for key, part in files.items():
self.form[key] = (part.filename, part.file)
def __repr__(self):
@@ -149,15 +148,10 @@
if params is None:
return
encoding = self.encoding
- for param, val in params.iteritems():
+ for param, val in params.items():
if isinstance(val, (tuple, list)):
- val = [
- unicode(x, encoding) if isinstance(x, str) else x
- for x in val]
if len(val) == 1:
val = val[0]
- elif isinstance(val, str):
- val = unicode(val, encoding)
if param in self.no_script_form_params and val:
val = self.no_script_form_param(param, val)
if param == '_cwmsgid':
--- a/wsgi/test/unittest_wsgi.py Tue Dec 15 19:05:41 2015 +0100
+++ b/wsgi/test/unittest_wsgi.py Wed Dec 16 11:23:48 2015 +0100
@@ -1,7 +1,7 @@
# encoding=utf-8
import webtest.app
-from StringIO import StringIO
+from io import BytesIO
from cubicweb.devtools.webtest import CubicWebTestTC
@@ -21,11 +21,11 @@
r = webtest.app.TestRequest.blank('/', {
'CONTENT_LENGTH': 12,
'CONTENT_TYPE': 'text/plain',
- 'wsgi.input': StringIO('some content')})
+ 'wsgi.input': BytesIO(b'some content')})
req = CubicWebWsgiRequest(r.environ, self.vreg)
- self.assertEqual('some content', req.content.read())
+ self.assertEqual(b'some content', req.content.read())
def test_http_scheme(self):
r = webtest.app.TestRequest.blank('/', {
@@ -52,11 +52,11 @@
self.assertTrue(req.https)
def test_big_content(self):
- content = 'x'*100001
+ content = b'x'*100001
r = webtest.app.TestRequest.blank('/', {
'CONTENT_LENGTH': len(content),
'CONTENT_TYPE': 'text/plain',
- 'wsgi.input': StringIO(content)})
+ 'wsgi.input': BytesIO(content)})
req = CubicWebWsgiRequest(r.environ, self.vreg)
@@ -94,14 +94,14 @@
def test_post_files(self):
content_type, params = self.webapp.encode_multipart(
- (), (('filefield', 'aname', 'acontent'),))
+ (), (('filefield', 'aname', b'acontent'),))
r = webtest.app.TestRequest.blank(
'/', POST=params, content_type=content_type)
req = CubicWebWsgiRequest(r.environ, self.vreg)
self.assertIn('filefield', req.form)
fieldvalue = req.form['filefield']
self.assertEqual(u'aname', fieldvalue[0])
- self.assertEqual('acontent', fieldvalue[1].read())
+ self.assertEqual(b'acontent', fieldvalue[1].read())
def test_post_unicode_urlencoded(self):
params = 'arg=%C3%A9'
@@ -115,3 +115,8 @@
super(WSGIAppTC, cls).init_config(config)
config.https_uiprops = None
config.https_datadir_url = None
+
+
+if __name__ == '__main__':
+ import unittest
+ unittest.main()