# HG changeset patch # User Alexandre Fayolle # Date 1275121082 -7200 # Node ID 61b28589d33ff994ffd9a3996f959b7d2b0eb0ab # Parent be94157bd75418b6579b6e3ac83b524ebc676d45# Parent 2604545d7dd9bc4cacdc2b4e086a4e7b7ccc058d merge back to stable some changes made on site for a customer. diff -r 2604545d7dd9 -r 61b28589d33f .hgtags --- a/.hgtags Sat May 29 10:06:07 2010 +0000 +++ b/.hgtags Sat May 29 10:18:02 2010 +0200 @@ -125,3 +125,5 @@ 24cc65ab2eca05729d66cef3de6f69bb7f9dfa35 cubicweb-debian-version-3.8.0-1 1e074c6150fe00844160986852db364cc5992848 cubicweb-version-3.8.1 eb972d125eefd0de2d0743e95c6e1f4e3e93e4c1 cubicweb-debian-version-3.8.1-1 +ef2e37d34013488a2018e73338fbbfbde5901c5c cubicweb-version-3.8.2 +2b962bb9eee8ee7156a12cf137428c292f8e3b35 cubicweb-debian-version-3.8.2-1 diff -r 2604545d7dd9 -r 61b28589d33f __pkginfo__.py --- a/__pkginfo__.py Sat May 29 10:06:07 2010 +0000 +++ b/__pkginfo__.py Sat May 29 10:18:02 2010 +0200 @@ -22,7 +22,7 @@ modname = distname = "cubicweb" -numversion = (3, 8, 1) +numversion = (3, 8, 3) version = '.'.join(str(num) for num in numversion) description = "a repository of entities / relations for knowledge management" @@ -40,7 +40,7 @@ ] __depends__ = { - 'logilab-common': '>= 0.50.0', + 'logilab-common': '>= 0.50.2', 'logilab-mtconverter': '>= 0.6.0', 'rql': '>= 0.26.0', 'yams': '>= 0.28.1', diff -r 2604545d7dd9 -r 61b28589d33f cwconfig.py --- a/cwconfig.py Sat May 29 10:06:07 2010 +0000 +++ b/cwconfig.py Sat May 29 10:18:02 2010 +0200 @@ -957,7 +957,10 @@ def load_site_cubicweb(self): """load instance's specific site_cubicweb file""" - for path in reversed([self.apphome] + self.cubes_path()): + paths = self.cubes_path() + if self.apphome is not None: + paths = [self.apphome] + paths + for path in reversed(paths): sitefile = join(path, 'site_cubicweb.py') if exists(sitefile) and not sitefile in self._site_loaded: self._load_site_cubicweb(sitefile) diff -r 2604545d7dd9 -r 61b28589d33f debian/changelog --- a/debian/changelog Sat May 29 10:06:07 2010 +0000 +++ b/debian/changelog Sat May 29 10:18:02 2010 +0200 @@ -1,3 +1,9 @@ +cubicweb (3.8.2-1) unstable; urgency=low + + * new upstream release + + -- Sylvain Thénault Tue, 18 May 2010 14:59:07 +0200 + cubicweb (3.8.1-1) unstable; urgency=low * new upstream release diff -r 2604545d7dd9 -r 61b28589d33f debian/control --- a/debian/control Sat May 29 10:06:07 2010 +0000 +++ b/debian/control Sat May 29 10:18:02 2010 +0200 @@ -97,7 +97,7 @@ Package: cubicweb-common Architecture: all XB-Python-Version: ${python:Versions} -Depends: ${python:Depends}, graphviz, gettext, python-logilab-mtconverter (>= 0.6.0), python-logilab-common (>= 0.50.0), python-yams (>= 0.29.0), python-rql (>= 0.26.0), python-lxml +Depends: ${python:Depends}, graphviz, gettext, python-logilab-mtconverter (>= 0.6.0), python-logilab-common (>= 0.50.2), python-yams (>= 0.29.0), python-rql (>= 0.26.1), python-lxml Recommends: python-simpletal (>= 4.0), python-crypto Conflicts: cubicweb-core Replaces: cubicweb-core diff -r 2604545d7dd9 -r 61b28589d33f devtools/__init__.py --- a/devtools/__init__.py Sat May 29 10:06:07 2010 +0000 +++ b/devtools/__init__.py Sat May 29 10:18:02 2010 +0200 @@ -278,7 +278,6 @@ def init_test_database_sqlite(config): """initialize a fresh sqlite databse used for testing purpose""" # remove database file if it exists - dbfile = config.sources()['system']['db-name'] if not reset_test_database_sqlite(config): # initialize the database import shutil diff -r 2604545d7dd9 -r 61b28589d33f devtools/devctl.py --- a/devtools/devctl.py Sat May 29 10:06:07 2010 +0000 +++ b/devtools/devctl.py Sat May 29 10:18:02 2010 +0200 @@ -58,6 +58,7 @@ if cubes: self._cubes = self.reorder_cubes( self.expand_cubes(cubes, with_recommends=True)) + self.load_site_cubicweb() else: self._cubes = () @@ -351,23 +352,23 @@ def update_cubes_catalogs(cubes): for cubedir in cubes: - toedit = [] if not isdir(cubedir): print '-> ignoring %s that is not a directory.' % cubedir continue try: - toedit += update_cube_catalogs(cubedir) + toedit = update_cube_catalogs(cubedir) except Exception: import traceback traceback.print_exc() print '-> error while updating catalogs for cube', cubedir else: # instructions pour la suite - print '-> regenerated .po catalogs for cube %s.' % cubedir - print '\nYou can now edit the following files:' - print '* ' + '\n* '.join(toedit) - print ('When you are done, run "cubicweb-ctl i18ninstance ' - '" to see changes in your instances.') + if toedit: + print '-> regenerated .po catalogs for cube %s.' % cubedir + print '\nYou can now edit the following files:' + print '* ' + '\n* '.join(toedit) + print ('When you are done, run "cubicweb-ctl i18ninstance ' + '" to see changes in your instances.') def update_cube_catalogs(cubedir): import shutil @@ -375,7 +376,6 @@ from logilab.common.fileutils import ensure_fs_mode from logilab.common.shellutils import find, rm from cubicweb.i18n import extract_from_tal, execute - toedit = [] cube = basename(normpath(cubedir)) tempdir = tempfile.mkdtemp() print underline_title('Updating i18n catalogs for cube %s' % cube) @@ -420,8 +420,14 @@ print '-> merging %i .pot files:' % len(potfiles) execute('msgcat -o %s %s' % (potfile, ' '.join('"%s"' % f for f in potfiles))) + if not exists(potfile): + print 'no message catalog for cube', cube, 'nothing to translate' + # cleanup + rm(tempdir) + return () print '-> merging main pot file with existing translations:' chdir('i18n') + toedit = [] for lang in LANGS: print '-> language', lang cubepo = '%s.po' % lang @@ -520,6 +526,7 @@ # You should have received a copy of the GNU Lesser General Public License along # with this program. If not, see . ''', + 'GPL': '''\ # This program is free software: you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software diff -r 2604545d7dd9 -r 61b28589d33f entities/test/unittest_wfobjs.py --- a/entities/test/unittest_wfobjs.py Sat May 29 10:06:07 2010 +0000 +++ b/entities/test/unittest_wfobjs.py Sat May 29 10:18:02 2010 +0200 @@ -56,7 +56,7 @@ self.commit() wf.add_state(u'foo') ex = self.assertRaises(ValidationError, self.commit) - self.assertEquals(ex.errors, {'state_of-subject': 'workflow already have a state of that name'}) + self.assertEquals(ex.errors, {'name-subject': 'workflow already have a state of that name'}) # no pb if not in the same workflow wf2 = add_wf(self, 'Company') foo = wf2.add_state(u'foo', initial=True) diff -r 2604545d7dd9 -r 61b28589d33f etwist/server.py --- a/etwist/server.py Sat May 29 10:06:07 2010 +0000 +++ b/etwist/server.py Sat May 29 10:18:02 2010 +0200 @@ -41,6 +41,7 @@ from cubicweb.web import dumps from logilab.common.decorators import monkeypatch +from logilab.common.daemon import daemonize from cubicweb import AuthenticationError, ConfigurationError, CW_EVENT_MANAGER from cubicweb.web import Redirect, DirectResponse, StatusResponse, LogOut @@ -49,30 +50,6 @@ from cubicweb.etwist.request import CubicWebTwistedRequestAdapter from cubicweb.etwist.http import HTTPResponse -def daemonize(): - # XXX unix specific - # XXX factorize w/ code in cw.server.server and cw.server.serverctl - # (start-repository command) - # See http://www.erlenstar.demon.co.uk/unix/faq_toc.html#TOC16 - if os.fork(): # launch child and... - return 1 - os.setsid() - if os.fork(): # launch child again. - return 1 - # move to the root to avoit mount pb - os.chdir('/') - # set paranoid umask - os.umask(077) - null = os.open('/dev/null', os.O_RDWR) - for i in range(3): - try: - os.dup2(null, i) - except OSError, e: - if e.errno != errno.EBADF: - raise - os.close(null) - return None - def start_task(interval, func): lc = task.LoopingCall(func) # wait until interval has expired to actually start the task, else we have @@ -418,15 +395,8 @@ raise ConfigurationError("Under windows, you must use the service management " "commands (e.g : 'net start my_instance)'") print 'instance starting in the background' - if daemonize(): + if daemonize(config['pid-file']): return # child process - if config['pid-file']: - # ensure the directory where the pid-file should be set exists (for - # instance /var/run/cubicweb may be deleted on computer restart) - piddir = os.path.dirname(config['pid-file']) - if not os.path.exists(piddir): - os.makedirs(piddir) - file(config['pid-file'], 'w').write(str(os.getpid())) root_resource.init_publisher() # before changing uid if config['uid'] is not None: try: diff -r 2604545d7dd9 -r 61b28589d33f etwist/twconfig.py --- a/etwist/twconfig.py Sat May 29 10:06:07 2010 +0000 +++ b/etwist/twconfig.py Sat May 29 10:18:02 2010 +0200 @@ -39,18 +39,30 @@ options = merge_options(( # ctl configuration + ('port', + {'type' : 'int', + 'default': None, + 'help': 'http server port number (default to 8080)', + 'group': 'web', 'level': 0, + }), + ('max-post-length', + {'type' : 'bytes', + 'default': '100MB', + 'help': 'maximum length of HTTP request. Default to 100 MB.', + 'group': 'web', 'level': 1, + }), + ('profile', + {'type' : 'string', + 'default': None, + 'help': 'profile code and use the specified file to store stats if this option is set', + 'group': 'web', 'level': 3, + }), ('host', {'type' : 'string', 'default': None, 'help': 'host name if not correctly detectable through gethostname', 'group': 'main', 'level': 1, }), - ('port', - {'type' : 'int', - 'default': None, - 'help': 'http server port number (default to 8080)', - 'group': 'main', 'level': 0, - }), ('pid-file', {'type' : 'string', 'default': Method('default_pid_file'), @@ -64,24 +76,12 @@ the repository rather than the user running the command', 'group': 'main', 'level': WebConfiguration.mode == 'system' }), - ('max-post-length', - {'type' : 'bytes', - 'default': '100MB', - 'help': 'maximum length of HTTP request. Default to 100 MB.', - 'group': 'main', 'level': 1, - }), ('session-time', {'type' : 'time', 'default': '30min', 'help': 'session expiration time, default to 30 minutes', 'group': 'main', 'level': 1, }), - ('profile', - {'type' : 'string', - 'default': None, - 'help': 'profile code and use the specified file to store stats if this option is set', - 'group': 'main', 'level': 3, - }), ('pyro-server', {'type' : 'yn', # pyro is only a recommends by default, so don't activate it here diff -r 2604545d7dd9 -r 61b28589d33f hooks/integrity.py --- a/hooks/integrity.py Sat May 29 10:06:07 2010 +0000 +++ b/hooks/integrity.py Sat May 29 10:18:02 2010 +0200 @@ -136,10 +136,10 @@ if rdef.role_cardinality(role) in '1+': if role == 'subject': set_operation(self._cw, '_cwisrel', (eid, rschema.type), - _CheckSRelationOp) + _CheckSRelationOp, list) else: set_operation(self._cw, '_cwiorel', (eid, rschema.type), - _CheckORelationOp) + _CheckORelationOp, list) def before_delete_relation(self): rtype = self.rtype @@ -153,10 +153,10 @@ card = session.schema_rproperty(rtype, eidfrom, eidto, 'cardinality') if card[0] in '1+' and not session.deleted_in_transaction(eidfrom): set_operation(self._cw, '_cwisrel', (eidfrom, rtype), - _CheckSRelationOp) + _CheckSRelationOp, list) if card[1] in '1+' and not session.deleted_in_transaction(eidto): set_operation(self._cw, '_cwiorel', (eidto, rtype), - _CheckORelationOp) + _CheckORelationOp, list) class _CheckConstraintsOp(hook.LateOperation): @@ -205,7 +205,7 @@ if constraints: hook.set_operation(self._cw, 'check_constraints_op', (self.eidfrom, self.rtype, self.eidto, tuple(constraints)), - _CheckConstraintsOp) + _CheckConstraintsOp, list) class CheckAttributeConstraintHook(IntegrityHook): @@ -226,7 +226,7 @@ if constraints: hook.set_operation(self._cw, 'check_constraints_op', (self.entity.eid, attr, None, tuple(constraints)), - _CheckConstraintsOp) + _CheckConstraintsOp, list) class CheckUniqueHook(IntegrityHook): diff -r 2604545d7dd9 -r 61b28589d33f i18n/en.po --- a/i18n/en.po Sat May 29 10:06:07 2010 +0000 +++ b/i18n/en.po Sat May 29 10:18:02 2010 +0200 @@ -5,7 +5,7 @@ msgstr "" "Project-Id-Version: 2.0\n" "POT-Creation-Date: 2006-01-12 17:35+CET\n" -"PO-Revision-Date: 2009-09-17 11:53+0200\n" +"PO-Revision-Date: 2010-05-16 18:58+0200\n" "Last-Translator: Sylvain Thenault \n" "Language-Team: English \n" "MIME-Version: 1.0\n" @@ -3158,14 +3158,14 @@ msgid "schema's permissions definitions" msgstr "" +msgid "schema-diagram" +msgstr "diagram" + msgid "schema-entity-types" -msgstr "" - -msgid "schema-image" -msgstr "image" +msgstr "entities" msgid "schema-relation-types" -msgstr "" +msgstr "relations" msgid "schema-security" msgstr "permissions" @@ -3947,3 +3947,6 @@ msgid "you should probably delete that property" msgstr "" + +#~ msgid "schema-image" +#~ msgstr "image" diff -r 2604545d7dd9 -r 61b28589d33f i18n/es.po --- a/i18n/es.po Sat May 29 10:06:07 2010 +0000 +++ b/i18n/es.po Sat May 29 10:18:02 2010 +0200 @@ -3235,12 +3235,12 @@ msgid "schema's permissions definitions" msgstr "definiciones de permisos del esquema" +msgid "schema-diagram" +msgstr "" + msgid "schema-entity-types" msgstr "" -msgid "schema-image" -msgstr "esquema imagen" - msgid "schema-relation-types" msgstr "" @@ -4034,3 +4034,6 @@ msgid "you should probably delete that property" msgstr "deberia probablamente suprimir esta propriedad" + +#~ msgid "schema-image" +#~ msgstr "esquema imagen" diff -r 2604545d7dd9 -r 61b28589d33f i18n/fr.po --- a/i18n/fr.po Sat May 29 10:06:07 2010 +0000 +++ b/i18n/fr.po Sat May 29 10:18:02 2010 +0200 @@ -4,7 +4,7 @@ msgid "" msgstr "" "Project-Id-Version: cubicweb 2.46.0\n" -"PO-Revision-Date: 2010-01-15 09:35+0100\n" +"PO-Revision-Date: 2010-05-16 18:59+0200\n" "Last-Translator: Logilab Team \n" "Language-Team: fr \n" "MIME-Version: 1.0\n" @@ -3272,12 +3272,12 @@ msgid "schema's permissions definitions" msgstr "permissions définies dans le schéma" +msgid "schema-diagram" +msgstr "diagramme" + msgid "schema-entity-types" msgstr "types d'entités" -msgid "schema-image" -msgstr "image" - msgid "schema-relation-types" msgstr "types de relations" @@ -4081,3 +4081,6 @@ msgid "you should probably delete that property" msgstr "vous devriez probablement supprimer cette propriété" + +#~ msgid "schema-image" +#~ msgstr "image" diff -r 2604545d7dd9 -r 61b28589d33f mail.py --- a/mail.py Sat May 29 10:06:07 2010 +0000 +++ b/mail.py Sat May 29 10:18:02 2010 +0200 @@ -85,8 +85,11 @@ assert type(content) is unicode, repr(content) msg = MIMEText(content.encode('UTF-8'), 'plain', 'UTF-8') # safety: keep only the first newline - subject = subject.splitlines()[0] - msg['Subject'] = header(subject) + try: + subject = subject.splitlines()[0] + msg['Subject'] = header(subject) + except IndexError: + pass # no subject if uinfo.get('email'): email = uinfo['email'] elif config and config['sender-addr']: diff -r 2604545d7dd9 -r 61b28589d33f migration.py --- a/migration.py Sat May 29 10:06:07 2010 +0000 +++ b/migration.py Sat May 29 10:18:02 2010 +0200 @@ -320,7 +320,7 @@ """a configuration option has been renamed""" self._option_changes.append(('renamed', oldname, newname)) - def cmd_option_group_change(self, option, oldgroup, newgroup): + def cmd_option_group_changed(self, option, oldgroup, newgroup): """a configuration option has been moved in another group""" self._option_changes.append(('moved', option, oldgroup, newgroup)) diff -r 2604545d7dd9 -r 61b28589d33f misc/migration/3.8.3_Any.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/misc/migration/3.8.3_Any.py Sat May 29 10:18:02 2010 +0200 @@ -0,0 +1,3 @@ +if 'same_as' in schema: + sync_schema_props_perms('same_as', syncperms=False) +sync_schema_props_perms('Bookmark', syncperms=False) diff -r 2604545d7dd9 -r 61b28589d33f misc/migration/3.8.3_common.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/misc/migration/3.8.3_common.py Sat May 29 10:18:02 2010 +0200 @@ -0,0 +1,4 @@ +option_group_changed('port', 'main', 'web') +option_group_changed('query-log-file', 'main', 'web') +option_group_changed('profile', 'main', 'web') +option_group_changed('max-post-length', 'main', 'web') diff -r 2604545d7dd9 -r 61b28589d33f rqlrewrite.py --- a/rqlrewrite.py Sat May 29 10:06:07 2010 +0000 +++ b/rqlrewrite.py Sat May 29 10:18:02 2010 +0200 @@ -155,7 +155,7 @@ snippets: (varmap, list of rql expression) with varmap a *tuple* (select var, snippet var) """ - self.select = self.insert_scope = select + self.select = select self.solutions = solutions self.kwargs = kwargs self.u_varname = None @@ -163,6 +163,7 @@ self.exists_snippet = {} self.pending_keys = [] self.existingvars = existingvars + self._insert_scope = None # we have to annotate the rqlst before inserting snippets, even though # we'll have to redo it latter self.annotate(select) @@ -249,15 +250,19 @@ def _insert_snippet(self, varmap, parent, new): if new is not None: + if self._insert_scope is None: + insert_scope = self.varinfo.get('stinfo', {}).get('scope', self.select) + else: + insert_scope = self._insert_scope if self.varinfo.get('stinfo', {}).get('optrelations'): assert parent is None - self.insert_scope = self.snippet_subquery(varmap, new) + self._insert_scope = self.snippet_subquery(varmap, new) self.insert_pending() - self.insert_scope = self.select + self._insert_scope = None return new = n.Exists(new) if parent is None: - self.insert_scope.add_restriction(new) + insert_scope.add_restriction(new) else: grandpa = parent.parent or_ = n.Or(parent, new) @@ -274,9 +279,9 @@ self._cleanup_inserted(new) raise else: - self.insert_scope = new + self._insert_scope = new self.insert_pending() - self.insert_scope = self.select + self._insert_scope = None return new self.insert_pending() diff -r 2604545d7dd9 -r 61b28589d33f rset.py --- a/rset.py Sat May 29 10:06:07 2010 +0000 +++ b/rset.py Sat May 29 10:18:02 2010 +0200 @@ -475,7 +475,10 @@ if role == 'subject': rschema = eschema.subjrels[attr] if rschema.final: - entity[attr] = rowvalues[outerselidx] + if attr == 'eid': + entity.eid = rowvalues[outerselidx] + else: + entity[attr] = rowvalues[outerselidx] continue else: rschema = eschema.objrels[attr] diff -r 2604545d7dd9 -r 61b28589d33f schema.py --- a/schema.py Sat May 29 10:06:07 2010 +0000 +++ b/schema.py Sat May 29 10:18:02 2010 +0200 @@ -866,6 +866,11 @@ if self.eid is not None: session.local_perm_cache[key] = False return False + except Unauthorized, ex: + self.debug('unauthorized %s: %s', rql, str(ex)) + if self.eid is not None: + session.local_perm_cache[key] = False + return False else: rset = session.eid_rset(kwargs[keyarg]) # if no special has_*_permission relation in the rql expression, just diff -r 2604545d7dd9 -r 61b28589d33f schemas/Bookmark.py --- a/schemas/Bookmark.py Sat May 29 10:06:07 2010 +0000 +++ b/schemas/Bookmark.py Sat May 29 10:18:02 2010 +0200 @@ -34,7 +34,7 @@ } title = String(required=True, maxsize=128, internationalizable=True) - path = String(maxsize=512, required=True, + path = String(maxsize=2048, required=True, description=_("relative url of the bookmarked page")) bookmarked_by = SubjectRelation('CWUser', diff -r 2604545d7dd9 -r 61b28589d33f schemas/base.py --- a/schemas/base.py Sat May 29 10:06:07 2010 +0000 +++ b/schemas/base.py Sat May 29 10:18:02 2010 +0200 @@ -176,7 +176,8 @@ name = String(required=True, indexed=True, internationalizable=True, maxsize=100, description=_('name or identifier of the permission')) label = String(required=True, internationalizable=True, maxsize=100, - description=_('distinct label to distinguate between other permission entity of the same name')) + description=_('distinct label to distinguate between other ' + 'permission entity of the same name')) require_group = SubjectRelation('CWGroup', description=_('groups to which the permission is granted')) @@ -210,7 +211,7 @@ 'add': ('managers', 'users'), 'delete': ('managers', 'owners'), } - cardinality = '*1' + cardinality = '**' symmetric = True # NOTE: the 'object = ExternalUri' declaration will still be mandatory # in the cube's schema. diff -r 2604545d7dd9 -r 61b28589d33f server/hook.py --- a/server/hook.py Sat May 29 10:06:07 2010 +0000 +++ b/server/hook.py Sat May 29 10:18:02 2010 +0200 @@ -473,23 +473,27 @@ set_log_methods(Operation, getLogger('cubicweb.session')) +def _container_add(container, value): + {set: set.add, list: list.append}[container.__class__](container, value) -def set_operation(session, datakey, value, opcls, **opkwargs): +def set_operation(session, datakey, value, opcls, containercls=set, **opkwargs): """Search for session.transaction_data[`datakey`] (expected to be a set): * if found, simply append `value` - * else, initialize it to set([`value`]) and instantiate the given `opcls` - operation class with additional keyword arguments. + * else, initialize it to containercls([`value`]) and instantiate the given + `opcls` operation class with additional keyword arguments. `containercls` + is a set by default. Give `list` if you want to keep arrival ordering. You should use this instead of creating on operation for each `value`, since handling operations becomes coslty on massive data import. """ try: - session.transaction_data[datakey].add(value) + _container_add(session.transaction_data[datakey], value) except KeyError: opcls(session, **opkwargs) - session.transaction_data[datakey] = set((value,)) + session.transaction_data[datakey] = containercls() + _container_add(session.transaction_data[datakey], value) class LateOperation(Operation): diff -r 2604545d7dd9 -r 61b28589d33f server/migractions.py --- a/server/migractions.py Sat May 29 10:06:07 2010 +0000 +++ b/server/migractions.py Sat May 29 10:18:02 2010 +0200 @@ -1200,11 +1200,15 @@ source = self.repo.system_source storage = source.storage(etype, attribute) source.unset_storage(etype, attribute) - rset = self.rqlexec('Any X,A WHERE X is %s, X %s A' - % (etype, attribute), ask_confirm=False) + rset = self.rqlexec('Any X WHERE X is %s' % etype, ask_confirm=False) pb = ProgressBar(len(rset)) for entity in rset.entities(): + # fill cache. Do not fetch that attribute using the global rql query + # since we may exhaust memory doing that.... + getattr(entity, attribute) storage.migrate_entity(entity, attribute) + # remove from entity cache to avoid memory exhaustion + del entity[attribute] pb.update() print source.set_storage(etype, attribute, storage) diff -r 2604545d7dd9 -r 61b28589d33f server/msplanner.py --- a/server/msplanner.py Sat May 29 10:06:07 2010 +0000 +++ b/server/msplanner.py Sat May 29 10:18:02 2010 +0200 @@ -95,7 +95,8 @@ from logilab.common.decorators import cached from rql.stmts import Union, Select -from rql.nodes import VariableRef, Comparison, Relation, Constant, Variable +from rql.nodes import (VariableRef, Comparison, Relation, Constant, Variable, + Not, Exists) from cubicweb import server from cubicweb.utils import make_uid @@ -109,6 +110,40 @@ # str() Constant.value to ensure generated table name won't be unicode Constant._ms_table_key = lambda x: str(x.value) +def ms_scope(term): + rel = None + scope = term.scope + if isinstance(term, Variable) and len(term.stinfo['relations']) == 1: + rel = iter(term.stinfo['relations']).next().relation() + elif isinstance(term, Constant): + rel = term.relation() + elif isinstance(term, Relation): + rel = term + if rel is not None and ( + rel.r_type != 'identity' and rel.scope is scope + and isinstance(rel.parent, Exists) and rel.parent.neged(strict=True)): + return scope.parent.scope + return scope + +def need_intersect(select, getrschema): + for rel in select.iget_nodes(Relation): + if isinstance(rel.parent, Exists) and rel.parent.neged(strict=True) and not rel.is_types_restriction(): + rschema = getrschema(rel.r_type) + if not rschema.final: + # if one of the relation's variable is ambiguous but not + # invariant, an intersection will be necessary + for vref in rel.get_nodes(VariableRef): + var = vref.variable + if (var.valuable_references() == 1 + and len(var.stinfo['possibletypes']) > 1): + return True + return False + +def neged_relation(rel): + parent = rel.parent + return isinstance(parent, Not) or (isinstance(parent, Exists) and + isinstance(parent.parent, Not)) + def need_source_access_relation(vargraph): if not vargraph: return False @@ -195,7 +230,7 @@ """return true if the variable is used in an outer scope of the given scope """ for rel in var.stinfo['relations']: - rscope = rel.scope + rscope = ms_scope(rel) if not rscope is scope and is_ancestor(scope, rscope): return True return False @@ -378,9 +413,9 @@ elif not self._sourcesterms: self._set_source_for_term(source, const) elif source in self._sourcesterms: - source_scopes = frozenset(t.scope for t in self._sourcesterms[source]) + source_scopes = frozenset(ms_scope(t) for t in self._sourcesterms[source]) for const in vconsts: - if const.scope in source_scopes: + if ms_scope(const) in source_scopes: self._set_source_for_term(source, const) # if system source is used, add every rewritten constant # to its supported terms even when associated entity @@ -505,12 +540,15 @@ def _remove_sources_until_stable(self, term, termssources): sourcesterms = self._sourcesterms for oterm, rel in self._linkedterms.get(term, ()): - if not term.scope is oterm.scope and rel.scope.neged(strict=True): + tscope = ms_scope(term) + otscope = ms_scope(oterm) + rscope = ms_scope(rel) + if not tscope is otscope and rscope.neged(strict=True): # can't get information from relation inside a NOT exists # where terms don't belong to the same scope continue need_ancestor_scope = False - if not (term.scope is rel.scope and oterm.scope is rel.scope): + if not (tscope is rscope and otscope is rscope): if rel.ored(): continue if rel.ored(traverse_scope=True): @@ -518,7 +556,7 @@ # propagate from parent scope to child scope, nothing else need_ancestor_scope = True relsources = self._repo.rel_type_sources(rel.r_type) - if rel.neged(strict=True) and ( + if neged_relation(rel) and ( len(relsources) < 2 or not isinstance(oterm, Variable) or oterm.valuable_references() != 1 @@ -532,9 +570,9 @@ # Y) continue # compute invalid sources for terms and remove them - if not need_ancestor_scope or is_ancestor(term.scope, oterm.scope): + if not need_ancestor_scope or is_ancestor(tscope, otscope): self._remove_term_sources(term, rel, oterm, termssources) - if not need_ancestor_scope or is_ancestor(oterm.scope, term.scope): + if not need_ancestor_scope or is_ancestor(otscope, tscope): self._remove_term_sources(oterm, rel, term, termssources) def _remove_term_sources(self, term, rel, oterm, termssources): @@ -693,7 +731,7 @@ sourceterms.clear() sources = [source] else: - scope = term.scope + scope = ms_scope(term) # find which sources support the same term and solutions sources = self._expand_sources(source, term, solindices) # no try to get as much terms as possible @@ -779,7 +817,7 @@ # `terms`, eg cross relations) for c in vconsts: rel = c.relation() - if rel is None or not (rel in terms or rel.neged(strict=True)): + if rel is None or not (rel in terms or neged_relation(rel)): final = False break break @@ -802,13 +840,13 @@ # variable is refed by an outer scope and should be substituted # using an 'identity' relation (else we'll get a conflict of # temporary tables) - if rhsvar in terms and not lhsvar in terms and lhsvar.scope is lhsvar.stmt: + if rhsvar in terms and not lhsvar in terms and ms_scope(lhsvar) is lhsvar.stmt: self._identity_substitute(rel, lhsvar, terms, needsel) - elif lhsvar in terms and not rhsvar in terms and rhsvar.scope is rhsvar.stmt: + elif lhsvar in terms and not rhsvar in terms and ms_scope(rhsvar) is rhsvar.stmt: self._identity_substitute(rel, rhsvar, terms, needsel) def _identity_substitute(self, relation, var, terms, needsel): - newvar = self._insert_identity_variable(relation.scope, var) + newvar = self._insert_identity_variable(ms_scope(relation), var) # ensure relation is using '=' operator, else we rely on a # sqlgenerator side effect (it won't insert an inequality operator # in this case) @@ -824,14 +862,14 @@ if len(self._sourcesterms) > 1: # priority to variable from subscopes for term in sourceterms: - if not term.scope is self.rqlst: + if not ms_scope(term) is self.rqlst: if isinstance(term, Variable): return term, sourceterms.pop(term) secondchoice = term else: # priority to variable from outer scope for term in sourceterms: - if term.scope is self.rqlst: + if ms_scope(term) is self.rqlst: if isinstance(term, Variable): return term, sourceterms.pop(term) secondchoice = term @@ -881,7 +919,7 @@ # term has to belong to the same scope if there is more # than the system source remaining if len(sourcesterms) > 1 and not scope is self.rqlst: - candidates = (t for t in sourceterms.keys() if scope is t.scope) + candidates = (t for t in sourceterms.keys() if scope is ms_scope(t)) else: candidates = sourceterms #.iterkeys() # we only want one unlinked term in each generated query @@ -1200,9 +1238,10 @@ step = AggrStep(plan, selection, select, atemptable, temptable) step.children = steps elif len(steps) > 1: - if select.need_intersect or any(select.need_intersect - for step in steps - for select in step.union.children): + getrschema = self.schema.rschema + if need_intersect(select, getrschema) or any(need_intersect(select, getrschema) + for step in steps + for select in step.union.children): if temptable: step = IntersectFetchStep(plan) # XXX not implemented else: diff -r 2604545d7dd9 -r 61b28589d33f server/querier.py --- a/server/querier.py Sat May 29 10:06:07 2010 +0000 +++ b/server/querier.py Sat May 29 10:18:02 2010 +0200 @@ -269,6 +269,7 @@ # transform in subquery when len(localchecks)>1 and groups if nbtrees > 1 and (select.orderby or select.groupby or select.having or select.has_aggregat or + select.distinct or select.limit or select.offset): newselect = Select() # only select variables in subqueries @@ -303,6 +304,7 @@ select.offset = 0 myunion = Union() newselect.set_with([SubQuery(aliases, myunion)], check=False) + newselect.distinct = select.distinct solutions = [sol.copy() for sol in select.solutions] cleanup_solutions(newselect, solutions) newselect.set_possible_types(solutions) diff -r 2604545d7dd9 -r 61b28589d33f server/repository.py --- a/server/repository.py Sat May 29 10:06:07 2010 +0000 +++ b/server/repository.py Sat May 29 10:18:02 2010 +0200 @@ -25,14 +25,14 @@ point to a cubicweb instance. * handles session management * provides method for pyro registration, to call if pyro is enabled - +""" -""" from __future__ import with_statement __docformat__ = "restructuredtext en" import sys +import threading import Queue from os.path import join from datetime import datetime @@ -315,7 +315,6 @@ def pinfo(self): # XXX: session.pool is accessed from a local storage, would be interesting # to see if there is a pool set in any thread specific data) - import threading return '%s: %s (%s)' % (self._available_pools.qsize(), ','.join(session.user.login for session in self._sessions.values() if session.pool), @@ -362,29 +361,6 @@ except ZeroDivisionError: pass - def stats(self): # XXX restrict to managers session? - import threading - results = {} - querier = self.querier - source = self.system_source - for size, maxsize, hits, misses, title in ( - (len(querier._rql_cache), self.config['rql-cache-size'], - querier.cache_hit, querier.cache_miss, 'rqlt_st'), - (len(source._cache), self.config['rql-cache-size'], - source.cache_hit, source.cache_miss, 'sql'), - ): - results['%s_cache_size' % title] = '%s / %s' % (size, maxsize) - results['%s_cache_hit' % title] = hits - results['%s_cache_miss' % title] = misses - results['%s_cache_hit_percent' % title] = (hits * 100) / (hits + misses) - results['sql_no_cache'] = self.system_source.no_cache - results['nb_open_sessions'] = len(self._sessions) - results['nb_active_threads'] = threading.activeCount() - results['looping_tasks'] = ', '.join(str(t) for t in self._looping_tasks) - results['available_pools'] = self._available_pools.qsize() - results['threads'] = ', '.join(sorted(str(t) for t in threading.enumerate())) - return results - def _login_from_email(self, login): session = self.internal_session() try: @@ -434,6 +410,28 @@ # public (dbapi) interface ################################################ + def stats(self): # XXX restrict to managers session? + results = {} + querier = self.querier + source = self.system_source + for size, maxsize, hits, misses, title in ( + (len(querier._rql_cache), self.config['rql-cache-size'], + querier.cache_hit, querier.cache_miss, 'rqlt_st'), + (len(source._cache), self.config['rql-cache-size'], + source.cache_hit, source.cache_miss, 'sql'), + ): + results['%s_cache_size' % title] = '%s / %s' % (size, maxsize) + results['%s_cache_hit' % title] = hits + results['%s_cache_miss' % title] = misses + results['%s_cache_hit_percent' % title] = (hits * 100) / (hits + misses) + results['sql_no_cache'] = self.system_source.no_cache + results['nb_open_sessions'] = len(self._sessions) + results['nb_active_threads'] = threading.activeCount() + results['looping_tasks'] = ', '.join(str(t) for t in self._looping_tasks) + results['available_pools'] = self._available_pools.qsize() + results['threads'] = ', '.join(sorted(str(t) for t in threading.enumerate())) + return results + def get_schema(self): """return the instance schema. This is a public method, not requiring a session id @@ -877,7 +875,6 @@ recreate=False): """get eid from a local id. An eid is attributed if no record is found""" cachekey = (extid, source.uri) - self.debug('repo extid2eid %s %s %s %s', source, extid, etype, insert) try: return self._extid_cache[cachekey] except KeyError: diff -r 2604545d7dd9 -r 61b28589d33f server/rqlannotation.py --- a/server/rqlannotation.py Sat May 29 10:06:07 2010 +0000 +++ b/server/rqlannotation.py Sat May 29 10:18:02 2010 +0200 @@ -24,7 +24,7 @@ from logilab.common.compat import any from rql import BadRQLQuery -from rql.nodes import Relation, VariableRef, Constant, Variable, Or +from rql.nodes import Relation, VariableRef, Constant, Variable, Or, Exists from rql.utils import common_parent def _annotate_select(annotator, rqlst): @@ -36,7 +36,7 @@ has_text_query = False need_distinct = rqlst.distinct for rel in rqlst.iget_nodes(Relation): - if getrschema(rel.r_type).symmetric and not rel.neged(strict=True): + if getrschema(rel.r_type).symmetric and not isinstance(rel.parent, Exists): for vref in rel.iget_nodes(VariableRef): stinfo = vref.variable.stinfo if not stinfo['constnode'] and stinfo['selected']: @@ -135,7 +135,7 @@ # priority should be given to relation which are not in inner queries # (eg exists) try: - stinfo['principal'] = _select_principal(var.sqlscope, joins) + stinfo['principal'] = _select_principal(var.scope, joins) except CantSelectPrincipal: stinfo['invariant'] = False rqlst.need_distinct = need_distinct @@ -146,7 +146,7 @@ class CantSelectPrincipal(Exception): """raised when no 'principal' variable can be found""" -def _select_principal(sqlscope, relations, _sort=lambda x:x): +def _select_principal(scope, relations, _sort=lambda x:x): """given a list of rqlst relations, select one which will be used to represent an invariant variable (e.g. using on extremity of the relation instead of the variable's type table @@ -161,7 +161,7 @@ continue if rel.ored(traverse_scope=True): ored_rels.add(rel) - elif rel.sqlscope is sqlscope: + elif rel.scope is scope: return rel elif not rel.neged(traverse_scope=True): diffscope_rels.add(rel) @@ -175,12 +175,12 @@ ored_rels.discard(rel1) ored_rels.discard(rel2) for rel in _sort(ored_rels): - if rel.sqlscope is sqlscope: + if rel.scope is scope: return rel diffscope_rels.add(rel) # if DISTINCT query, can use variable from a different scope as principal # since introduced duplicates will be removed - if sqlscope.stmt.distinct and diffscope_rels: + if scope.stmt.distinct and diffscope_rels: return iter(_sort(diffscope_rels)).next() # XXX could use a relation for a different scope if it can't generate # duplicates, so we would have to check cardinality @@ -197,7 +197,7 @@ if rel.operator() not in ('=', 'IS') \ or not isinstance(rel.children[1].children[0], VariableRef): continue - if rel.sqlscope is rel.stmt: + if rel.scope is rel.stmt: return rel principal = rel if principal is None: @@ -220,23 +220,6 @@ var._q_invariant = True else: var._q_invariant = False - for rel in select.iget_nodes(Relation): - if rel.neged(strict=True) and not rel.is_types_restriction(): - rschema = getrschema(rel.r_type) - if not rschema.final: - # if one of the relation's variable is ambiguous but not - # invariant, an intersection will be necessary - for vref in rel.get_nodes(VariableRef): - var = vref.variable - if (not var._q_invariant and var.valuable_references() == 1 - and len(var.stinfo['possibletypes']) > 1): - select.need_intersect = True - break - else: - continue - break - else: - select.need_intersect = False class SQLGenAnnotator(object): @@ -270,7 +253,7 @@ def is_ambiguous(self, var): # ignore has_text relation if len([rel for rel in var.stinfo['relations'] - if rel.sqlscope is var.sqlscope and rel.r_type == 'has_text']) == 1: + if rel.scope is var.scope and rel.r_type == 'has_text']) == 1: return False try: data = var.stmt._deamb_data @@ -353,7 +336,7 @@ if isinstance(term, VariableRef) and self.is_ambiguous(term.variable): var = term.variable if len(var.stinfo['relations']) == 1 \ - or rel.sqlscope is var.sqlscope or rel.r_type == 'identity': + or rel.scope is var.scope or rel.r_type == 'identity': self.restrict(var, frozenset(etypes_func())) try: self.maydeambrels[var].add(rel) diff -r 2604545d7dd9 -r 61b28589d33f server/server.py --- a/server/server.py Sat May 29 10:06:07 2010 +0000 +++ b/server/server.py Sat May 29 10:18:02 2010 +0200 @@ -15,9 +15,8 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -"""Pyro RQL server +"""Pyro RQL server""" -""" __docformat__ = "restructuredtext en" import os @@ -26,6 +25,8 @@ import warnings from time import localtime, mktime +from logilab.common.daemon import daemonize + from cubicweb.cwconfig import CubicWebConfiguration from cubicweb.server.repository import Repository @@ -83,7 +84,6 @@ self.quiting = None # event queue self.events = [] - # start repository looping tasks def add_event(self, event): """add an event to the loop""" @@ -103,6 +103,7 @@ def run(self, req_timeout=5.0): """enter the service loop""" + # start repository looping tasks self.repo.start_looping_tasks() while self.quiting is None: try: @@ -130,35 +131,7 @@ signal.signal(signal.SIGINT, lambda x, y, s=self: s.quit()) signal.signal(signal.SIGTERM, lambda x, y, s=self: s.quit()) - def daemonize(self, pid_file=None): - """daemonize the process""" - # fork so the parent can exist - if (os.fork()): - return -1 - # deconnect from tty and create a new session - os.setsid() - # fork again so the parent, (the session group leader), can exit. - # as a non-session group leader, we can never regain a controlling - # terminal. - if (os.fork()): - return -1 - # move to the root to avoit mount pb - os.chdir('/') - # set paranoid umask - os.umask(077) - if pid_file is not None: - # write pid in a file - f = open(pid_file, 'w') - f.write(str(os.getpid())) - f.close() - # filter warnings - warnings.filterwarnings('ignore') - # close standard descriptors - sys.stdin.close() - sys.stdout.close() - sys.stderr.close() - from logging import getLogger from cubicweb import set_log_methods LOGGER = getLogger('cubicweb.reposerver') -set_log_methods(CubicWebConfiguration, LOGGER) +set_log_methods(RepositoryServer, LOGGER) diff -r 2604545d7dd9 -r 61b28589d33f server/serverctl.py --- a/server/serverctl.py Sat May 29 10:06:07 2010 +0000 +++ b/server/serverctl.py Sat May 29 10:18:02 2010 +0200 @@ -43,7 +43,7 @@ given server.serverconfig """ from getpass import getpass - from logilab.common.db import get_connection + from logilab.database import get_connection dbhost = source.get('db-host') if dbname is None: dbname = source['db-name'] @@ -317,8 +317,9 @@ create_db = self.config.create_db helper = get_db_helper(driver) if driver == 'sqlite': - if os.path.exists(dbname) and automatic or \ - ASK.confirm('Database %s already exists -- do you want to drop it ?' % dbname): + if os.path.exists(dbname) and ( + automatic or + ASK.confirm('Database %s already exists. Drop it?' % dbname)): os.unlink(dbname) elif create_db: print '\n'+underline_title('Creating the system database') @@ -392,7 +393,7 @@ def run(self, args): print '\n'+underline_title('Initializing the system database') from cubicweb.server import init_repository - from logilab.common.db import get_connection + from logilab.database import get_connection appid = pop_arg(args, msg='No instance specified !') config = ServerConfiguration.config_for(appid) try: @@ -525,6 +526,7 @@ ) def run(self, args): + from logilab.common.daemon import daemonize from cubicweb.server.server import RepositoryServer appid = pop_arg(args, msg='No instance specified !') config = ServerConfiguration.config_for(appid) @@ -544,7 +546,7 @@ piddir = os.path.dirname(pidfile) if not os.path.exists(piddir): os.makedirs(piddir) - if not debug and server.daemonize(pidfile) == -1: + if not debug and daemonize(pidfile) == -1: return uid = config['uid'] if uid is not None: diff -r 2604545d7dd9 -r 61b28589d33f server/sources/native.py --- a/server/sources/native.py Sat May 29 10:06:07 2010 +0000 +++ b/server/sources/native.py Sat May 29 10:18:02 2010 +0200 @@ -33,6 +33,7 @@ from datetime import datetime from base64 import b64decode, b64encode from contextlib import contextmanager +from os.path import abspath from logilab.common.compat import any from logilab.common.cache import Cache @@ -264,6 +265,7 @@ if self.dbdriver == 'sqlite' and \ not getattr(repo.config, 'no_sqlite_wrap', False): from cubicweb.server.sources.extlite import ConnectionWrapper + self.dbhelper.dbname = abspath(self.dbhelper.dbname) self.get_connection = lambda: ConnectionWrapper(self) self.check_connection = lambda cnx: cnx def pool_reset(cnx): diff -r 2604545d7dd9 -r 61b28589d33f server/sources/pyrorql.py --- a/server/sources/pyrorql.py Sat May 29 10:06:07 2010 +0000 +++ b/server/sources/pyrorql.py Sat May 29 10:18:02 2010 +0200 @@ -38,7 +38,7 @@ from cubicweb.cwconfig import register_persistent_options from cubicweb.server.sources import (AbstractSource, ConnectionWrapper, TimedCache, dbg_st_search, dbg_results) - +from cubicweb.server.msplanner import neged_relation def uidtype(union, col, etype, args): select, col = union.locate_subquery(col, etype, args) @@ -476,7 +476,10 @@ return def visit_exists(self, node): - return 'EXISTS(%s)' % node.children[0].accept(self) + rql = node.children[0].accept(self) + if rql: + return 'EXISTS(%s)' % rql + return def visit_relation(self, node): try: @@ -486,7 +489,7 @@ restr, lhs = self.process_eid_const(node.children[0]) except UnknownEid: # can safely skip not relation with an unsupported eid - if node.neged(strict=True): + if neged_relation(node): return raise else: @@ -494,7 +497,7 @@ restr = None except UnknownEid: # can safely skip not relation with an unsupported eid - if node.neged(strict=True): + if neged_relation(node): return # XXX what about optional relation or outer NOT EXISTS() raise @@ -511,7 +514,7 @@ rhs = node.children[1].accept(self) except UnknownEid: # can safely skip not relation with an unsupported eid - if node.neged(strict=True): + if neged_relation(node): return # XXX what about optional relation or outer NOT EXISTS() raise diff -r 2604545d7dd9 -r 61b28589d33f server/sources/rql2sql.py --- a/server/sources/rql2sql.py Sat May 29 10:06:07 2010 +0000 +++ b/server/sources/rql2sql.py Sat May 29 10:18:02 2010 +0200 @@ -44,7 +44,7 @@ by Troels Arvin. Features SQL ISO Standard, PG, mysql, Oracle, MS SQL, DB2 and Informix. -.. _Comparison of different SQL implementations: http://www.troels.arvin.dk/db/rdbms +.. _Comparison of different SQL implementations: http://www.troels.arvin.dk/db/rdbms """ @@ -112,7 +112,7 @@ unstable.remove(varname) torewrite.add(var) newselect = Select() - newselect.need_distinct = newselect.need_intersect = False + newselect.need_distinct = False myunion = Union() myunion.append(newselect) # extract aliases / selection @@ -316,13 +316,15 @@ # IGenerator implementation for RQL->SQL ####################################### class StateInfo(object): - def __init__(self, existssols, unstablevars): + def __init__(self, select, existssols, unstablevars): self.existssols = existssols self.unstablevars = unstablevars self.subtables = {} self.needs_source_cb = None self.subquery_source_cb = None self.source_cb_funcs = set() + self.scopes = {select: 0} + self.scope_nodes = [] def reset(self, solution): """reset some visit variables""" @@ -381,12 +383,16 @@ self.solution = origsol self.tables = origtables - def push_scope(self): + def push_scope(self, scope_node): + self.scope_nodes.append(scope_node) + self.scopes[scope_node] = len(self.actual_tables) self.actual_tables.append([]) self._restr_stack.append(self.restrictions) self.restrictions = [] def pop_scope(self): + del self.scopes[self.scope_nodes[-1]] + self.scope_nodes.pop() restrictions = self.restrictions self.restrictions = self._restr_stack.pop() return restrictions, self.actual_tables.pop() @@ -442,7 +448,7 @@ self._varmap = varmap self._query_attrs = {} self._state = None - self._not_scope_offset = 0 + # self._not_scope_offset = 0 try: # union query for each rqlst / solution sql = self.union_sql(union) @@ -509,7 +515,7 @@ needwrap = True else: existssols, unstable = {}, () - state = StateInfo(existssols, unstable) + state = StateInfo(select, existssols, unstable) if self._state is not None: # state from a previous unioned select state.merge_source_cbs(self._state.needs_source_cb) @@ -622,12 +628,7 @@ elif self._state.restrictions and self.dbhelper.needs_from_clause: sql.insert(1, 'FROM (SELECT 1) AS _T') sqls.append('\n'.join(sql)) - if select.need_intersect: - #if distinct or not self.dbhelper.intersect_all_support: - return '\nINTERSECT\n'.join(sqls) - #else: - # return '\nINTERSECT ALL\n'.join(sqls) - elif distinct: + if distinct: return '\nUNION\n'.join(sqls) else: return '\nUNION ALL\n'.join(sqls) @@ -682,32 +683,11 @@ return '' def visit_not(self, node): - self._state.push_scope() - if isinstance(node.children[0], Relation): - self._not_scope_offset += 1 csql = node.children[0].accept(self) - if isinstance(node.children[0], Relation): - self._not_scope_offset -= 1 - sqls, tables = self._state.pop_scope() if node in self._state.done or not csql: # already processed or no sql generated by children - self._state.actual_tables[-1] += tables - self._state.restrictions += sqls return csql - if isinstance(node.children[0], Exists): - assert not sqls, (sqls, str(node.stmt)) - assert not tables, (tables, str(node.stmt)) - return 'NOT %s' % csql - sqls.append(csql) - if tables: - select = 'SELECT 1 FROM %s' % ','.join(tables) - else: - select = 'SELECT 1' - if sqls: - sql = 'NOT EXISTS(%s WHERE %s)' % (select, ' AND '.join(sqls)) - else: - sql = 'NOT EXISTS(%s)' % select - return sql + return 'NOT (%s)' % csql def visit_exists(self, exists): """generate SQL name for a exists subquery""" @@ -721,7 +701,7 @@ return 'EXISTS(%s)' % ' UNION '.join(sqls) def _visit_exists(self, exists): - self._state.push_scope() + self._state.push_scope(exists) restriction = exists.children[0].accept(self) restrictions, tables = self._state.pop_scope() if restriction: @@ -762,9 +742,6 @@ else: # no variables in the RHS sql = self._visit_attribute_relation(relation) - if relation.neged(strict=True): - self._state.done.add(relation.parent) - sql = 'NOT (%s)' % sql else: if rtype == 'is' and rhs.operator == 'IS': # special case "C is NULL" @@ -833,9 +810,6 @@ if relation.r_type == 'identity': # special case "X identity Y" lhs, rhs = relation.get_parts() - if isinstance(relation.parent, Not): - self._state.done.add(relation.parent) - return 'NOT %s%s' % (lhs.accept(self), rhs.accept(self)) return '%s%s' % (lhs.accept(self), rhs.accept(self)) lhsvar, lhsconst, rhsvar, rhsconst = relation_info(relation) rid = self._relation_table(relation) @@ -1041,7 +1015,7 @@ else: not_ = False return self.dbhelper.fti_restriction_sql(alias, const.eval(self._args), - jointo, not_) + restriction + jointo, not_) + restriction def visit_comparison(self, cmp): """generate SQL for a comparison""" @@ -1203,25 +1177,26 @@ pass return '' + def _temp_table_scope(self, select, table): + scope = 9999 + for var, sql in self._varmap.iteritems(): + if table == sql.split('.', 1)[0]: + try: + scope = min(scope, self._state.scopes[select.defined_vars[var].scope]) + except KeyError: + scope = 0 # XXX + if scope == 0: + return 0 + return 0 + def _var_info(self, var): - # if current var or one of its attribute is selected , it *must* - # appear in the toplevel's FROM even if we're currently visiting - # a EXISTS node - if var.sqlscope is var.stmt: - scope = 0 - # don't consider not_scope_offset if the variable is only used in one - # relation - elif len(var.stinfo['relations']) > 1: - scope = -1 - self._not_scope_offset - else: - scope = -1 try: sql = self._varmap[var.name] tablealias = sql.split('.', 1)[0] - if scope < 0: - scope = self._varmap_table_scope(var.stmt, tablealias) + scope = self._temp_table_scope(var.stmt, tablealias) self.add_table(tablealias, scope=scope) except KeyError: + scope = self._state.scopes[var.scope] etype = self._state.solution[var.name] # XXX this check should be moved in rql.stcheck if self.schema.eschema(etype).final: @@ -1235,7 +1210,7 @@ def _inlined_var_sql(self, var, rtype): try: sql = self._varmap['%s.%s' % (var.name, rtype)] - scope = var.sqlscope is var.stmt and 0 or -1 + scope = self._state.scopes[var.scope] self.add_table(sql.split('.', 1)[0], scope=scope) except KeyError: sql = '%s.%s%s' % (self._var_table(var), SQL_PREFIX, rtype) @@ -1358,7 +1333,7 @@ break # XXX may have a principal without being invariant for this generation, # not sure this is a pb or not - if var.stinfo.get('principal') is relation and var.sqlscope is var.stmt: + if var.stinfo.get('principal') is relation and var.scope is var.stmt: scope = 0 break else: @@ -1379,15 +1354,3 @@ alias = self.alias_and_add_table(self.dbhelper.fti_table) relation._q_sqltable = alias return alias - - def _varmap_table_scope(self, select, table): - """since a varmap table may be used for multiple variable, its scope is - the most outer scope of each variables - """ - scope = -1 - for varname, alias in self._varmap.iteritems(): - # check '.' in varname since there are 'X.attribute' keys in varmap - if not '.' in varname and alias.split('.', 1)[0] == table: - if select.defined_vars[varname].sqlscope is select: - return 0 - return scope diff -r 2604545d7dd9 -r 61b28589d33f server/sources/storages.py --- a/server/sources/storages.py Sat May 29 10:06:07 2010 +0000 +++ b/server/sources/storages.py Sat May 29 10:18:02 2010 +0200 @@ -82,7 +82,7 @@ XXX subject to race condition. """ - path = osp.join(dirpath, basename) + path = osp.join(dirpath, basename.replace(osp.sep, '-')) if not osp.isfile(path): return path base, ext = osp.splitext(path) @@ -125,17 +125,18 @@ def entity_updated(self, entity, attr): """an entity using this storage for attr has been updatded""" + oldpath = self.current_fs_path(entity, attr) if entity._cw.transaction_data.get('fs_importing'): - oldpath = self.current_fs_path(entity, attr) fpath = entity[attr].getvalue() - if oldpath != fpath: - hook.set_operation(entity._cw, 'bfss_deleted', oldpath, - DeleteFileOp) binary = Binary(file(fpath).read()) else: binary = entity.pop(attr) - fpath = self.current_fs_path(entity, attr) + fpath = self.new_fs_path(entity, attr) UpdateFileOp(entity._cw, filepath=fpath, filedata=binary.getvalue()) + if oldpath != fpath: + entity[attr] = Binary(fpath) + hook.set_operation(entity._cw, 'bfss_deleted', oldpath, + DeleteFileOp) return binary def entity_deleted(self, entity, attr): diff -r 2604545d7dd9 -r 61b28589d33f server/test/unittest_msplanner.py --- a/server/test/unittest_msplanner.py Sat May 29 10:06:07 2010 +0000 +++ b/server/test/unittest_msplanner.py Sat May 29 10:18:02 2010 +0200 @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -""" - -""" from cubicweb.devtools import init_test_database from cubicweb.devtools.repotest import BasePlannerTC, test_plan @@ -748,7 +745,6 @@ ]) def test_not_identity(self): - # both system and rql support all variables, can be self._test('Any X WHERE NOT X identity U, U eid %s' % self.session.user.eid, [('OneFetchStep', [('Any X WHERE NOT X identity 5, X is CWUser', [{'X': 'CWUser'}])], @@ -1105,7 +1101,7 @@ [('Any L,X WHERE X login L, X is CWUser', [{'X': 'CWUser', 'L': 'String'}])], [self.ldap, self.system], None, {'X': 'table2.C1', 'X.login': 'table2.C0', 'L': 'table2.C0'}, []), ('OneFetchStep', - [('Any G,L WHERE X in_group G, X login L, G name "managers", (EXISTS(X copain T, T login L, T is CWUser)) OR (EXISTS(X in_state S, S name "pascontent", NOT X copain T2, S is State, T2 is CWUser)), G is CWGroup, X is CWUser', + [('Any G,L WHERE X in_group G, X login L, G name "managers", (EXISTS(X copain T, T login L, T is CWUser)) OR (EXISTS(X in_state S, S name "pascontent", NOT EXISTS(X copain T2), S is State)), G is CWGroup, T2 is CWUser, X is CWUser', [{'G': 'CWGroup', 'L': 'String', 'S': 'State', 'T': 'CWUser', 'T2': 'CWUser', 'X': 'CWUser'}])], None, None, [self.system], {'T2': 'table1.C0', 'L': 'table2.C0', @@ -1222,7 +1218,7 @@ # in the source where %(x)s is not coming from and will be removed during rql # generation for the external source self._test('Any SN WHERE NOT X in_state S, X eid %(x)s, S name SN', - [('OneFetchStep', [('Any SN WHERE NOT 5 in_state S, S name SN, S is State', + [('OneFetchStep', [('Any SN WHERE NOT EXISTS(5 in_state S), S name SN, S is State', [{'S': 'State', 'SN': 'String'}])], None, None, [self.cards, self.system], {}, [])], {'x': ueid}) @@ -1233,7 +1229,7 @@ # the same plan may be used, since we won't find any record in the system source # linking 9999999 to a state self._test('Any SN WHERE NOT X in_state S, X eid %(x)s, S name SN', - [('OneFetchStep', [('Any SN WHERE NOT 999999 in_state S, S name SN, S is State', + [('OneFetchStep', [('Any SN WHERE NOT EXISTS(999999 in_state S), S name SN, S is State', [{'S': 'State', 'SN': 'String'}])], None, None, [self.cards, self.system], {}, [])], {'x': 999999}) @@ -1246,12 +1242,12 @@ []), ('IntersectStep', None, None, [('OneFetchStep', - [('Any SN WHERE NOT X in_state S, S name SN, S is State, X is Note', + [('Any SN WHERE NOT EXISTS(X in_state S, X is Note), S name SN, S is State', [{'S': 'State', 'SN': 'String', 'X': 'Note'}])], None, None, [self.cards, self.system], {}, []), ('OneFetchStep', - [('Any SN WHERE NOT X in_state S, S name SN, S is State, X is IN(Affaire, CWUser)', + [('Any SN WHERE NOT EXISTS(X in_state S, X is IN(Affaire, CWUser)), S name SN, S is State', [{'S': 'State', 'SN': 'String', 'X': 'Affaire'}, {'S': 'State', 'SN': 'String', 'X': 'CWUser'}])], None, None, [self.system], {'S': 'table0.C1', 'S.name': 'table0.C0', 'SN': 'table0.C0'}, @@ -1505,7 +1501,7 @@ self._test('Any Y WHERE X eid %(x)s, NOT X multisource_crossed_rel Y', [('FetchStep', [('Any Y WHERE Y is Note', [{'Y': 'Note'}])], [self.cards, self.system], None, {'Y': 'table0.C0'}, []), - ('OneFetchStep', [('Any Y WHERE NOT 999999 multisource_crossed_rel Y, Y is Note', + ('OneFetchStep', [('Any Y WHERE NOT EXISTS(999999 multisource_crossed_rel Y), Y is Note', [{'Y': 'Note'}])], None, None, [self.system], {'Y': 'table0.C0'}, [])], @@ -1633,7 +1629,7 @@ repo._type_source_cache[999999] = ('Note', 'system', 999999) self._test('DELETE Note X WHERE X eid %(x)s, NOT Y multisource_rel X', [('DeleteEntitiesStep', - [('OneFetchStep', [('Any 999999 WHERE NOT Y multisource_rel 999999, Y is IN(Card, Note)', + [('OneFetchStep', [('Any 999999 WHERE NOT EXISTS(Y multisource_rel 999999), Y is IN(Card, Note)', [{'Y': 'Card'}, {'Y': 'Note'}])], None, None, [self.system], {}, []) ]) @@ -2185,7 +2181,7 @@ self.repo._type_source_cache[999998] = ('Note', 'vcs', 999998) self.repo._type_source_cache[999999] = ('Note', 'vcs', 999999) self._test('Any X, Y WHERE NOT X multisource_rel Y, X eid 999998, Y eid 999999', - [('OneFetchStep', [('Any 999998,999999 WHERE NOT 999998 multisource_rel 999999', [{}])], + [('OneFetchStep', [('Any 999998,999999 WHERE NOT EXISTS(999998 multisource_rel 999999)', [{}])], None, None, [self.vcs], {}, []) ]) diff -r 2604545d7dd9 -r 61b28589d33f server/test/unittest_multisources.py --- a/server/test/unittest_multisources.py Sat May 29 10:06:07 2010 +0000 +++ b/server/test/unittest_multisources.py Sat May 29 10:18:02 2010 +0200 @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -""" - -""" from os.path import dirname, join, abspath from datetime import datetime, timedelta diff -r 2604545d7dd9 -r 61b28589d33f server/test/unittest_repository.py --- a/server/test/unittest_repository.py Sat May 29 10:06:07 2010 +0000 +++ b/server/test/unittest_repository.py Sat May 29 10:18:02 2010 +0200 @@ -239,10 +239,11 @@ if not r.type in ('eid', 'is', 'is_instance_of', 'identity', 'creation_date', 'modification_date', 'cwuri', 'owned_by', 'created_by', - 'update_permission', 'read_permission')], + 'update_permission', 'read_permission', + 'in_basket')], ['relation_type', 'from_entity', 'to_entity', - 'in_basket', 'constrained_by', + 'constrained_by', 'cardinality', 'ordernum', 'indexed', 'fulltextindexed', 'internationalizable', 'defaultval', 'description', 'description_format']) diff -r 2604545d7dd9 -r 61b28589d33f server/test/unittest_rql2sql.py --- a/server/test/unittest_rql2sql.py Sat May 29 10:06:07 2010 +0000 +++ b/server/test/unittest_rql2sql.py Sat May 29 10:18:02 2010 +0200 @@ -15,10 +15,6 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -""" - -""" - """unit tests for module cubicweb.server.sources.rql2sql""" import sys @@ -180,7 +176,7 @@ "NOT EXISTS(X owned_by U, U in_group G, G name 'lulufanclub' OR G name 'managers');", '''SELECT _X.cw_eid FROM cw_Personne AS _X -WHERE _X.cw_prenom=lulu AND NOT EXISTS(SELECT 1 FROM owned_by_relation AS rel_owned_by0, in_group_relation AS rel_in_group1, cw_CWGroup AS _G WHERE rel_owned_by0.eid_from=_X.cw_eid AND rel_in_group1.eid_from=rel_owned_by0.eid_to AND rel_in_group1.eid_to=_G.cw_eid AND ((_G.cw_name=lulufanclub) OR (_G.cw_name=managers)))'''), +WHERE _X.cw_prenom=lulu AND NOT (EXISTS(SELECT 1 FROM owned_by_relation AS rel_owned_by0, in_group_relation AS rel_in_group1, cw_CWGroup AS _G WHERE rel_owned_by0.eid_from=_X.cw_eid AND rel_in_group1.eid_from=rel_owned_by0.eid_to AND rel_in_group1.eid_to=_G.cw_eid AND ((_G.cw_name=lulufanclub) OR (_G.cw_name=managers))))'''), @@ -276,7 +272,7 @@ ('Any O WHERE NOT S ecrit_par O, S eid 1, S inline1 P, O inline2 P', '''SELECT _O.cw_eid FROM cw_Note AS _S, cw_Personne AS _O -WHERE NOT EXISTS(SELECT 1 WHERE _S.cw_ecrit_par=_O.cw_eid) AND _S.cw_eid=1 AND _O.cw_inline2=_S.cw_inline1'''), +WHERE NOT (_S.cw_ecrit_par=_O.cw_eid) AND _S.cw_eid=1 AND _O.cw_inline2=_S.cw_inline1'''), ('DISTINCT Any S ORDERBY stockproc(SI) WHERE NOT S ecrit_par O, S para SI', '''SELECT T1.C0 FROM (SELECT DISTINCT _S.cw_eid AS C0, STOCKPROC(_S.cw_para) AS C1 @@ -299,7 +295,7 @@ (' Any X,U WHERE C owned_by U, NOT X owned_by U, C eid 1, X eid 2', '''SELECT 2, rel_owned_by0.eid_to FROM owned_by_relation AS rel_owned_by0 -WHERE rel_owned_by0.eid_from=1 AND NOT EXISTS(SELECT 1 FROM owned_by_relation AS rel_owned_by1 WHERE rel_owned_by1.eid_from=2 AND rel_owned_by0.eid_to=rel_owned_by1.eid_to)'''), +WHERE rel_owned_by0.eid_from=1 AND NOT (EXISTS(SELECT 1 FROM owned_by_relation AS rel_owned_by1 WHERE rel_owned_by1.eid_from=2 AND rel_owned_by0.eid_to=rel_owned_by1.eid_to))'''), ('Any GN WHERE X in_group G, G name GN, (G name "managers" OR EXISTS(X copain T, T login in ("comme", "cochon")))', '''SELECT _G.cw_name @@ -353,7 +349,7 @@ ('Any L WHERE X login "admin", NOT X identity Y, Y login L', '''SELECT _Y.cw_login FROM cw_CWUser AS _X, cw_CWUser AS _Y -WHERE _X.cw_login=admin AND NOT _X.cw_eid=_Y.cw_eid'''), +WHERE _X.cw_login=admin AND NOT (_X.cw_eid=_Y.cw_eid)'''), ('Any L WHERE X login "admin", X identity Y?, Y login L', '''SELECT _Y.cw_login @@ -391,31 +387,31 @@ ('DISTINCT Any X,Y WHERE X name "CWGroup", Y eid IN(1, 2, 3), NOT EXISTS(X read_permission Y)', '''SELECT DISTINCT _X.cw_eid, _Y.cw_eid FROM cw_CWEType AS _X, cw_CWGroup AS _Y -WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid) +WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT (EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid)) UNION SELECT DISTINCT _X.cw_eid, _Y.cw_eid FROM cw_CWEType AS _X, cw_RQLExpression AS _Y -WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid)'''), +WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT (EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid))'''), # should generate the same query as above ('DISTINCT Any X,Y WHERE X name "CWGroup", Y eid IN(1, 2, 3), NOT X read_permission Y', '''SELECT DISTINCT _X.cw_eid, _Y.cw_eid FROM cw_CWEType AS _X, cw_CWGroup AS _Y -WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid) +WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT (EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid)) UNION SELECT DISTINCT _X.cw_eid, _Y.cw_eid FROM cw_CWEType AS _X, cw_RQLExpression AS _Y -WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid)'''), +WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT (EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid))'''), # neged relation, can't be inveriant ('Any X,Y WHERE X name "CWGroup", Y eid IN(1, 2, 3), NOT X read_permission Y', '''SELECT _X.cw_eid, _Y.cw_eid FROM cw_CWEType AS _X, cw_CWGroup AS _Y -WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid) +WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT (EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid)) UNION ALL SELECT _X.cw_eid, _Y.cw_eid FROM cw_CWEType AS _X, cw_RQLExpression AS _Y -WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid)'''), +WHERE _X.cw_name=CWGroup AND _Y.cw_eid IN(1, 2, 3) AND NOT (EXISTS(SELECT 1 FROM read_permission_relation AS rel_read_permission0 WHERE rel_read_permission0.eid_from=_X.cw_eid AND rel_read_permission0.eid_to=_Y.cw_eid))'''), ('Any MAX(X)+MIN(X), N GROUPBY N WHERE X name N, X is IN (Basket, Folder, Tag);', '''SELECT (MAX(T1.C0) + MIN(T1.C0)), T1.C1 FROM (SELECT _X.cw_eid AS C0, _X.cw_name AS C1 @@ -552,7 +548,7 @@ 'EXISTS(A use_email O, EXISTS(A identity B, NOT B in_group D, D name "guests", D is CWGroup), A is CWUser), B eid 2', '''SELECT _O.cw_eid, _O.cw_address, _O.cw_alias, _O.cw_modification_date FROM cw_EmailAddress AS _O -WHERE NOT EXISTS(SELECT 1 FROM use_email_relation AS rel_use_email0 WHERE rel_use_email0.eid_from=1 AND rel_use_email0.eid_to=_O.cw_eid) AND EXISTS(SELECT 1 FROM use_email_relation AS rel_use_email1 WHERE rel_use_email1.eid_to=_O.cw_eid AND EXISTS(SELECT 1 FROM cw_CWGroup AS _D WHERE rel_use_email1.eid_from=2 AND NOT EXISTS(SELECT 1 FROM in_group_relation AS rel_in_group2 WHERE rel_in_group2.eid_from=2 AND rel_in_group2.eid_to=_D.cw_eid) AND _D.cw_name=guests)) +WHERE NOT (EXISTS(SELECT 1 FROM use_email_relation AS rel_use_email0 WHERE rel_use_email0.eid_from=1 AND rel_use_email0.eid_to=_O.cw_eid)) AND EXISTS(SELECT 1 FROM use_email_relation AS rel_use_email1 WHERE rel_use_email1.eid_to=_O.cw_eid AND EXISTS(SELECT 1 FROM cw_CWGroup AS _D WHERE rel_use_email1.eid_from=2 AND NOT (EXISTS(SELECT 1 FROM in_group_relation AS rel_in_group2 WHERE rel_in_group2.eid_from=2 AND rel_in_group2.eid_to=_D.cw_eid)) AND _D.cw_name=guests)) ORDER BY 4 DESC'''), @@ -603,17 +599,17 @@ ("Personne X WHERE NOT X evaluee Y;", '''SELECT _X.cw_eid FROM cw_Personne AS _X -WHERE NOT EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_X.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_X.cw_eid))'''), ("Note N WHERE NOT X evaluee N, X eid 0", '''SELECT _N.cw_eid FROM cw_Note AS _N -WHERE NOT EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=0 AND rel_evaluee0.eid_to=_N.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=0 AND rel_evaluee0.eid_to=_N.cw_eid))'''), ('Any X WHERE NOT X travaille S, X is Personne', '''SELECT _X.cw_eid FROM cw_Personne AS _X -WHERE NOT EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_from=_X.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_from=_X.cw_eid))'''), ("Personne P where not P datenaiss TODAY", '''SELECT _P.cw_eid @@ -623,16 +619,16 @@ ("Personne P where NOT P concerne A", '''SELECT _P.cw_eid FROM cw_Personne AS _P -WHERE NOT EXISTS(SELECT 1 FROM concerne_relation AS rel_concerne0 WHERE rel_concerne0.eid_from=_P.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM concerne_relation AS rel_concerne0 WHERE rel_concerne0.eid_from=_P.cw_eid))'''), ("Affaire A where not P concerne A", '''SELECT _A.cw_eid FROM cw_Affaire AS _A -WHERE NOT EXISTS(SELECT 1 FROM concerne_relation AS rel_concerne0 WHERE rel_concerne0.eid_to=_A.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM concerne_relation AS rel_concerne0 WHERE rel_concerne0.eid_to=_A.cw_eid))'''), ("Personne P where not P concerne A, A sujet ~= 'TEST%'", '''SELECT _P.cw_eid FROM cw_Affaire AS _A, cw_Personne AS _P -WHERE NOT EXISTS(SELECT 1 FROM concerne_relation AS rel_concerne0 WHERE rel_concerne0.eid_from=_P.cw_eid AND rel_concerne0.eid_to=_A.cw_eid) AND _A.cw_sujet ILIKE TEST%'''), +WHERE NOT (EXISTS(SELECT 1 FROM concerne_relation AS rel_concerne0 WHERE rel_concerne0.eid_from=_P.cw_eid AND rel_concerne0.eid_to=_A.cw_eid)) AND _A.cw_sujet ILIKE TEST%'''), ('Any S WHERE NOT T eid 28258, T tags S', '''SELECT rel_tags0.eid_to @@ -660,33 +656,33 @@ ('Note X WHERE NOT Y evaluee X', '''SELECT _X.cw_eid FROM cw_Note AS _X -WHERE NOT EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_to=_X.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_to=_X.cw_eid))'''), ('Any Y WHERE NOT Y evaluee X', '''SELECT _Y.cw_eid FROM cw_CWUser AS _Y -WHERE NOT EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_Y.cw_eid) +WHERE NOT (EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_Y.cw_eid)) UNION ALL SELECT _Y.cw_eid FROM cw_Division AS _Y -WHERE NOT EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_Y.cw_eid) +WHERE NOT (EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_Y.cw_eid)) UNION ALL SELECT _Y.cw_eid FROM cw_Personne AS _Y -WHERE NOT EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_Y.cw_eid) +WHERE NOT (EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_Y.cw_eid)) UNION ALL SELECT _Y.cw_eid FROM cw_Societe AS _Y -WHERE NOT EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_Y.cw_eid) +WHERE NOT (EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_Y.cw_eid)) UNION ALL SELECT _Y.cw_eid FROM cw_SubDivision AS _Y -WHERE NOT EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_Y.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0 WHERE rel_evaluee0.eid_from=_Y.cw_eid))'''), ('Any X WHERE NOT Y evaluee X, Y is CWUser', '''SELECT _X.cw_eid FROM cw_Note AS _X -WHERE NOT EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0,cw_CWUser AS _Y WHERE rel_evaluee0.eid_from=_Y.cw_eid AND rel_evaluee0.eid_to=_X.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM evaluee_relation AS rel_evaluee0, cw_CWUser AS _Y WHERE rel_evaluee0.eid_from=_Y.cw_eid AND rel_evaluee0.eid_to=_X.cw_eid))'''), ('Any X,RT WHERE X relation_type RT, NOT X is CWAttribute', '''SELECT _X.cw_eid, _X.cw_relation_type @@ -701,17 +697,13 @@ ('Any S WHERE NOT X in_state S, X is IN(Affaire, CWUser)', '''SELECT _S.cw_eid FROM cw_State AS _S -WHERE NOT EXISTS(SELECT 1 FROM cw_Affaire AS _X WHERE _X.cw_in_state=_S.cw_eid) -INTERSECT -SELECT _S.cw_eid -FROM cw_State AS _S -WHERE NOT EXISTS(SELECT 1 FROM cw_CWUser AS _X WHERE _X.cw_in_state=_S.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM cw_Affaire AS _X WHERE _X.cw_in_state=_S.cw_eid UNION SELECT 1 FROM cw_CWUser AS _X WHERE _X.cw_in_state=_S.cw_eid))'''), ('Any S WHERE NOT(X in_state S, S name "somename"), X is CWUser', '''SELECT _S.cw_eid FROM cw_State AS _S -WHERE NOT EXISTS(SELECT 1 FROM cw_CWUser AS _X WHERE _X.cw_in_state=_S.cw_eid AND _S.cw_name=somename)'''), - +WHERE NOT (EXISTS(SELECT 1 FROM cw_CWUser AS _X WHERE _X.cw_in_state=_S.cw_eid AND _S.cw_name=somename))'''), + # XXXFIXME fail # ('Any X,RT WHERE X relation_type RT?, NOT X is CWAttribute', # '''SELECT _X.cw_eid, _X.cw_relation_type @@ -844,7 +836,7 @@ ('Any O,AD WHERE NOT S inline1 O, S eid 123, O todo_by AD?', '''SELECT _O.cw_eid, rel_todo_by0.eid_to FROM cw_Affaire AS _O LEFT OUTER JOIN todo_by_relation AS rel_todo_by0 ON (rel_todo_by0.eid_from=_O.cw_eid), cw_Note AS _S -WHERE NOT EXISTS(SELECT 1 WHERE _S.cw_inline1=_O.cw_eid) AND _S.cw_eid=123''') +WHERE NOT (_S.cw_inline1=_O.cw_eid) AND _S.cw_eid=123''') ] VIRTUAL_VARS = [ @@ -919,7 +911,7 @@ FROM cw_Personne AS _P'''), ] -SYMETRIC = [ +SYMMETRIC = [ ('Any P WHERE X eid 0, X connait P', '''SELECT DISTINCT _P.cw_eid FROM connait_relation AS rel_connait0, cw_Personne AS _P @@ -941,17 +933,17 @@ ('Any P WHERE X eid 0, NOT X connait P', '''SELECT _P.cw_eid FROM cw_Personne AS _P -WHERE NOT EXISTS(SELECT 1 FROM connait_relation AS rel_connait0 WHERE (rel_connait0.eid_from=0 AND rel_connait0.eid_to=_P.cw_eid OR rel_connait0.eid_to=0 AND rel_connait0.eid_from=_P.cw_eid))'''), +WHERE NOT (EXISTS(SELECT 1 FROM connait_relation AS rel_connait0 WHERE (rel_connait0.eid_from=0 AND rel_connait0.eid_to=_P.cw_eid OR rel_connait0.eid_to=0 AND rel_connait0.eid_from=_P.cw_eid)))'''), ('Any P WHERE NOT X connait P', '''SELECT _P.cw_eid FROM cw_Personne AS _P -WHERE NOT EXISTS(SELECT 1 FROM connait_relation AS rel_connait0 WHERE (rel_connait0.eid_to=_P.cw_eid OR rel_connait0.eid_from=_P.cw_eid))'''), +WHERE NOT (EXISTS(SELECT 1 FROM connait_relation AS rel_connait0 WHERE (rel_connait0.eid_to=_P.cw_eid OR rel_connait0.eid_from=_P.cw_eid)))'''), ('Any X WHERE NOT X connait P', '''SELECT _X.cw_eid FROM cw_Personne AS _X -WHERE NOT EXISTS(SELECT 1 FROM connait_relation AS rel_connait0 WHERE (rel_connait0.eid_from=_X.cw_eid OR rel_connait0.eid_to=_X.cw_eid))'''), +WHERE NOT (EXISTS(SELECT 1 FROM connait_relation AS rel_connait0 WHERE (rel_connait0.eid_from=_X.cw_eid OR rel_connait0.eid_to=_X.cw_eid)))'''), ('Any P WHERE X connait P, P nom "nom"', '''SELECT DISTINCT _P.cw_eid @@ -988,7 +980,12 @@ ('Any N WHERE NOT N ecrit_par P, P nom "toto"', '''SELECT _N.cw_eid FROM cw_Note AS _N, cw_Personne AS _P -WHERE NOT EXISTS(SELECT 1 WHERE _N.cw_ecrit_par=_P.cw_eid) AND _P.cw_nom=toto'''), +WHERE NOT (_N.cw_ecrit_par=_P.cw_eid) AND _P.cw_nom=toto'''), + + ('Any P WHERE NOT N ecrit_par P, P nom "toto"', + '''SELECT _P.cw_eid +FROM cw_Personne AS _P +WHERE NOT (EXISTS(SELECT 1 FROM cw_Note AS _N WHERE _N.cw_ecrit_par=_P.cw_eid)) AND _P.cw_nom=toto'''), ('Any P WHERE N ecrit_par P, N eid 0', '''SELECT _N.cw_ecrit_par @@ -1003,7 +1000,7 @@ ('Any P WHERE NOT N ecrit_par P, P is Personne, N eid 512', '''SELECT _P.cw_eid FROM cw_Note AS _N, cw_Personne AS _P -WHERE NOT EXISTS(SELECT 1 WHERE _N.cw_ecrit_par=_P.cw_eid) AND _N.cw_eid=512'''), +WHERE NOT (_N.cw_ecrit_par=_P.cw_eid) AND _N.cw_eid=512'''), ('Any S,ES,T WHERE S state_of ET, ET name "CWUser", ES allowed_transition T, T destination_state S', '''SELECT _T.cw_destination_state, rel_allowed_transition1.eid_from, _T.cw_eid @@ -1025,7 +1022,7 @@ ('Any X WHERE NOT Y for_user X, X eid 123', '''SELECT 123 -WHERE NOT EXISTS(SELECT 1 FROM cw_CWProperty AS _Y WHERE _Y.cw_for_user=123) +WHERE NOT (EXISTS(SELECT 1 FROM cw_CWProperty AS _Y WHERE _Y.cw_for_user=123)) '''), ] @@ -1034,46 +1031,34 @@ ('Any SN WHERE NOT X in_state S, S name SN', '''SELECT _S.cw_name FROM cw_State AS _S -WHERE NOT EXISTS(SELECT 1 FROM cw_Affaire AS _X WHERE _X.cw_in_state=_S.cw_eid) -INTERSECT -SELECT _S.cw_name -FROM cw_State AS _S -WHERE NOT EXISTS(SELECT 1 FROM cw_CWUser AS _X WHERE _X.cw_in_state=_S.cw_eid) -INTERSECT -SELECT _S.cw_name -FROM cw_State AS _S -WHERE NOT EXISTS(SELECT 1 FROM cw_Note AS _X WHERE _X.cw_in_state=_S.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM cw_Affaire AS _X WHERE _X.cw_in_state=_S.cw_eid UNION SELECT 1 FROM cw_Note AS _X WHERE _X.cw_in_state=_S.cw_eid UNION SELECT 1 FROM cw_CWUser AS _X WHERE _X.cw_in_state=_S.cw_eid))'''), ('Any PN WHERE NOT X travaille S, X nom PN, S is IN(Division, Societe)', '''SELECT _X.cw_nom FROM cw_Personne AS _X -WHERE NOT EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0,cw_Division AS _S WHERE rel_travaille0.eid_from=_X.cw_eid AND rel_travaille0.eid_to=_S.cw_eid) -INTERSECT -SELECT _X.cw_nom -FROM cw_Personne AS _X -WHERE NOT EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0,cw_Societe AS _S WHERE rel_travaille0.eid_from=_X.cw_eid AND rel_travaille0.eid_to=_S.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0, cw_Division AS _S WHERE rel_travaille0.eid_from=_X.cw_eid AND rel_travaille0.eid_to=_S.cw_eid UNION SELECT 1 FROM travaille_relation AS rel_travaille1, cw_Societe AS _S WHERE rel_travaille1.eid_from=_X.cw_eid AND rel_travaille1.eid_to=_S.cw_eid))'''), ('Any PN WHERE NOT X travaille S, S nom PN, S is IN(Division, Societe)', '''SELECT _S.cw_nom FROM cw_Division AS _S -WHERE NOT EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_to=_S.cw_eid) +WHERE NOT (EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_to=_S.cw_eid)) UNION ALL SELECT _S.cw_nom FROM cw_Societe AS _S -WHERE NOT EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_to=_S.cw_eid)'''), +WHERE NOT (EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_to=_S.cw_eid))'''), ('Personne X WHERE NOT X travaille S, S nom "chouette"', '''SELECT _X.cw_eid FROM cw_Division AS _S, cw_Personne AS _X -WHERE NOT EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_from=_X.cw_eid AND rel_travaille0.eid_to=_S.cw_eid) AND _S.cw_nom=chouette +WHERE NOT (EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_from=_X.cw_eid AND rel_travaille0.eid_to=_S.cw_eid)) AND _S.cw_nom=chouette UNION ALL SELECT _X.cw_eid FROM cw_Personne AS _X, cw_Societe AS _S -WHERE NOT EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_from=_X.cw_eid AND rel_travaille0.eid_to=_S.cw_eid) AND _S.cw_nom=chouette +WHERE NOT (EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_from=_X.cw_eid AND rel_travaille0.eid_to=_S.cw_eid)) AND _S.cw_nom=chouette UNION ALL SELECT _X.cw_eid FROM cw_Personne AS _X, cw_SubDivision AS _S -WHERE NOT EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_from=_X.cw_eid AND rel_travaille0.eid_to=_S.cw_eid) AND _S.cw_nom=chouette'''), +WHERE NOT (EXISTS(SELECT 1 FROM travaille_relation AS rel_travaille0 WHERE rel_travaille0.eid_from=_X.cw_eid AND rel_travaille0.eid_to=_S.cw_eid)) AND _S.cw_nom=chouette'''), ('Any X WHERE X is ET, ET eid 2', '''SELECT rel_is0.eid_from @@ -1345,7 +1330,7 @@ self.assertRaises(BadRQLQuery, self.o.generate, rqlst) def test_symmetric(self): - for t in self._parse(SYMETRIC): + for t in self._parse(SYMMETRIC): yield t def test_inline(self): @@ -1393,7 +1378,7 @@ WHERE EXISTS(SELECT 1 FROM cw_CWGroup AS _T WHERE _T.cw_name=managers)'''), ('Any X,Y WHERE NOT X created_by Y, X eid 5, Y eid 6', '''SELECT 5, 6 -WHERE NOT EXISTS(SELECT 1 FROM created_by_relation AS rel_created_by0 WHERE rel_created_by0.eid_from=5 AND rel_created_by0.eid_to=6)'''), +WHERE NOT (EXISTS(SELECT 1 FROM created_by_relation AS rel_created_by0 WHERE rel_created_by0.eid_from=5 AND rel_created_by0.eid_to=6))'''), ] for t in self._parse(queries): yield t @@ -1439,7 +1424,7 @@ self.o = SQLGenerator(schema, dbhelper) def _norm_sql(self, sql): - return sql.strip().replace(' ILIKE ', ' LIKE ').replace('\nINTERSECT ALL\n', '\nINTERSECT\n') + return sql.strip().replace(' ILIKE ', ' LIKE ') def test_date_extraction(self): self._check("Any MONTH(D) WHERE P is Personne, P creation_date D", @@ -1571,7 +1556,7 @@ ('Any X,Y WHERE NOT X created_by Y, X eid 5, Y eid 6', '''SELECT 5, 6 FROM (SELECT 1) AS _T -WHERE NOT EXISTS(SELECT 1 FROM created_by_relation AS rel_created_by0 WHERE rel_created_by0.eid_from=5 AND rel_created_by0.eid_to=6)'''), +WHERE NOT (EXISTS(SELECT 1 FROM created_by_relation AS rel_created_by0 WHERE rel_created_by0.eid_from=5 AND rel_created_by0.eid_to=6))'''), ] for t in self._parse(queries): yield t diff -r 2604545d7dd9 -r 61b28589d33f server/test/unittest_storage.py --- a/server/test/unittest_storage.py Sat May 29 10:06:07 2010 +0000 +++ b/server/test/unittest_storage.py Sat May 29 10:18:02 2010 +0200 @@ -21,7 +21,7 @@ from __future__ import with_statement -from logilab.common.testlib import unittest_main +from logilab.common.testlib import unittest_main, tag from cubicweb.devtools.testlib import CubicWebTC import os.path as osp @@ -180,7 +180,7 @@ self.assertEquals(f1.data.getvalue(), file(filepath).read(), 'files content differ') - + @tag('Storage', 'BFSS', 'update') def test_bfss_update_with_existing_data(self): # use self.session to use server-side cache f1 = self.session.create_entity('File', data=Binary('some data'), @@ -194,6 +194,52 @@ f2 = self.execute('Any F WHERE F eid %(f)s, F is File', {'f': f1.eid}).get_entity(0, 0) self.assertEquals(f2.data.getvalue(), 'some other data') + @tag('Storage', 'BFSS', 'update', 'extension', 'commit') + def test_bfss_update_with_different_extension_commited(self): + # use self.session to use server-side cache + f1 = self.session.create_entity('File', data=Binary('some data'), + data_format=u'text/plain', data_name=u'foo.txt') + # NOTE: do not use set_attributes() which would automatically + # update f1's local dict. We want the pure rql version to work + self.commit() + old_path = self.fspath(f1) + self.failUnless(osp.isfile(old_path)) + self.assertEquals(osp.splitext(old_path)[1], '.txt') + self.execute('SET F data %(d)s, F data_name %(dn)s, F data_format %(df)s WHERE F eid %(f)s', + {'d': Binary('some other data'), 'f': f1.eid, 'dn': u'bar.jpg', 'df': u'image/jpeg'}) + self.commit() + # the new file exists with correct extension + # the old file is dead + f2 = self.execute('Any F WHERE F eid %(f)s, F is File', {'f': f1.eid}).get_entity(0, 0) + new_path = self.fspath(f2) + self.failIf(osp.isfile(old_path)) + self.failUnless(osp.isfile(new_path)) + self.assertEquals(osp.splitext(new_path)[1], '.jpg') + + @tag('Storage', 'BFSS', 'update', 'extension', 'rollback') + def test_bfss_update_with_different_extension_rollbacked(self): + # use self.session to use server-side cache + f1 = self.session.create_entity('File', data=Binary('some data'), + data_format=u'text/plain', data_name=u'foo.txt') + # NOTE: do not use set_attributes() which would automatically + # update f1's local dict. We want the pure rql version to work + self.commit() + old_path = self.fspath(f1) + old_data = f1.data.getvalue() + self.failUnless(osp.isfile(old_path)) + self.assertEquals(osp.splitext(old_path)[1], '.txt') + self.execute('SET F data %(d)s, F data_name %(dn)s, F data_format %(df)s WHERE F eid %(f)s', + {'d': Binary('some other data'), 'f': f1.eid, 'dn': u'bar.jpg', 'df': u'image/jpeg'}) + self.rollback() + # the new file exists with correct extension + # the old file is dead + f2 = self.execute('Any F WHERE F eid %(f)s, F is File', {'f': f1.eid}).get_entity(0, 0) + new_path = self.fspath(f2) + new_data = f2.data.getvalue() + self.failUnless(osp.isfile(new_path)) + self.assertEquals(osp.splitext(new_path)[1], '.txt') + self.assertEquals(old_path, new_path) + self.assertEquals(old_data, new_data) def test_bfss_update_with_fs_importing(self): # use self.session to use server-side cache diff -r 2604545d7dd9 -r 61b28589d33f server/utils.py --- a/server/utils.py Sat May 29 10:06:07 2010 +0000 +++ b/server/utils.py Sat May 29 10:18:02 2010 +0200 @@ -148,7 +148,8 @@ self._t.cancel() def join(self): - self._t.join() + if self._t.isAlive(): + self._t.join() class RepoThread(Thread): diff -r 2604545d7dd9 -r 61b28589d33f skeleton/MANIFEST.in --- a/skeleton/MANIFEST.in Sat May 29 10:06:07 2010 +0000 +++ b/skeleton/MANIFEST.in Sat May 29 10:18:02 2010 +0200 @@ -2,3 +2,4 @@ include */*.py recursive-include data external_resources *.gif *.png *.css *.ico *.js recursive-include i18n *.pot *.po +recursive-include wdoc * diff -r 2604545d7dd9 -r 61b28589d33f skeleton/__pkginfo__.py.tmpl --- a/skeleton/__pkginfo__.py.tmpl Sat May 29 10:06:07 2010 +0000 +++ b/skeleton/__pkginfo__.py.tmpl Sat May 29 10:18:02 2010 +0200 @@ -34,7 +34,7 @@ [THIS_CUBE_DIR, [fname for fname in glob('*.py') if fname != 'setup.py']], ] # check for possible extended cube layout -for dname in ('entities', 'views', 'sobjects', 'hooks', 'schema', 'data', 'i18n', 'migration'): +for dname in ('entities', 'views', 'sobjects', 'hooks', 'schema', 'data', 'wdoc', 'i18n', 'migration'): if isdir(dname): data_files.append([join(THIS_CUBE_DIR, dname), listdir(dname)]) # Note: here, you'll need to add subdirectories if you want diff -r 2604545d7dd9 -r 61b28589d33f test/unittest_entity.py --- a/test/unittest_entity.py Sat May 29 10:06:07 2010 +0000 +++ b/test/unittest_entity.py Sat May 29 10:18:02 2010 +0200 @@ -251,7 +251,7 @@ email = self.execute('Any X WHERE X eid %(x)s', {'x': email.eid}).get_entity(0, 0) rql = email.unrelated_rql('use_email', 'CWUser', 'object')[0] self.assertEquals(rql, 'Any S,AA,AB,AC,AD ORDERBY AA ' - 'WHERE NOT S use_email O, O eid %(x)s, S is CWUser, S login AA, S firstname AB, S surname AC, S modification_date AD, ' + 'WHERE NOT EXISTS(S use_email O), O eid %(x)s, S is CWUser, S login AA, S firstname AB, S surname AC, S modification_date AD, ' 'A eid %(B)s, EXISTS(S identity A, NOT A in_group C, C name "guests", C is CWGroup)') #rql = email.unrelated_rql('use_email', 'Person', 'object')[0] #self.assertEquals(rql, '') diff -r 2604545d7dd9 -r 61b28589d33f test/unittest_rqlrewrite.py --- a/test/unittest_rqlrewrite.py Sat May 29 10:06:07 2010 +0000 +++ b/test/unittest_rqlrewrite.py Sat May 29 10:18:02 2010 +0200 @@ -24,7 +24,7 @@ from rql import parse, nodes, RQLHelper from cubicweb import Unauthorized -from cubicweb.schema import RRQLExpression +from cubicweb.schema import RRQLExpression, ERQLExpression from cubicweb.rqlrewrite import RQLRewriter from cubicweb.devtools import repotest, TestServerConfiguration @@ -350,6 +350,20 @@ self.failUnlessEqual(rqlst.as_string(), u"Any C WHERE C is Card, EXISTS(C owned_by A, A is CWUser)") + def test_rqlexpr_not_relation1(self): + constraint = RRQLExpression('X owned_by Z, Z login "hop"', 'X') + rqlst = parse('Affaire A WHERE NOT EXISTS(A documented_by C)') + rewrite(rqlst, {('C', 'X'): (constraint,)}, {}, 'X') + self.failUnlessEqual(rqlst.as_string(), + u'Any A WHERE NOT EXISTS(A documented_by C, EXISTS(C owned_by B, B login "hop", B is CWUser), C is Card), A is Affaire') + + def test_rqlexpr_not_relation2(self): + constraint = RRQLExpression('X owned_by Z, Z login "hop"', 'X') + rqlst = rqlhelper.parse('Affaire A WHERE NOT A documented_by C', annotate=False) + rewrite(rqlst, {('C', 'X'): (constraint,)}, {}, 'X') + self.failUnlessEqual(rqlst.as_string(), + u'Any A WHERE NOT EXISTS(A documented_by C, EXISTS(C owned_by B, B login "hop", B is CWUser), C is Card), A is Affaire') + if __name__ == '__main__': unittest_main() diff -r 2604545d7dd9 -r 61b28589d33f view.py --- a/view.py Sat May 29 10:06:07 2010 +0000 +++ b/view.py Sat May 29 10:18:02 2010 +0200 @@ -15,10 +15,8 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -"""abstract views and templates classes for CubicWeb web client +"""abstract views and templates classes for CubicWeb web client""" - -""" __docformat__ = "restructuredtext en" _ = unicode @@ -74,6 +72,7 @@ cubicweb:tindex CDATA #IMPLIED cubicweb:tlunit CDATA #IMPLIED cubicweb:type CDATA #IMPLIED + cubicweb:unselimg CDATA #IMPLIED cubicweb:uselabel CDATA #IMPLIED cubicweb:value CDATA #IMPLIED cubicweb:variables CDATA #IMPLIED diff -r 2604545d7dd9 -r 61b28589d33f web/application.py --- a/web/application.py Sat May 29 10:06:07 2010 +0000 +++ b/web/application.py Sat May 29 10:18:02 2010 +0200 @@ -233,12 +233,15 @@ return session def _update_last_login_time(self, req): + # XXX should properly detect missing permission / non writeable source + # and avoid "except (RepositoryError, Unauthorized)" below + if req.user.metainformation()['source']['adapter'] == 'ldapuser': + return try: req.execute('SET X last_login_time NOW WHERE X eid %(x)s', {'x' : req.user.eid}) req.cnx.commit() except (RepositoryError, Unauthorized): - # ldap user are not writeable for instance req.cnx.rollback() except: req.cnx.rollback() @@ -379,6 +382,8 @@ controller = self.vreg['controllers'].select(ctrlid, req, appli=self) except NoSelectableObject: + if ctrlid == 'login': + raise Unauthorized(req._('log out first')) raise Unauthorized(req._('not authorized')) req.update_search_state() result = controller.publish(rset=rset) diff -r 2604545d7dd9 -r 61b28589d33f web/facet.py --- a/web/facet.py Sat May 29 10:06:07 2010 +0000 +++ b/web/facet.py Sat May 29 10:18:02 2010 +0200 @@ -467,6 +467,7 @@ attrtype = 'String' # type of comparison: default is an exact match on the attribute value comparator = '=' # could be '<', '<=', '>', '>=' + i18nable = True def vocabulary(self): """return vocabulary for this facet, eg a list of 2-uple (label, value) @@ -491,7 +492,10 @@ return rset and self.rset_vocabulary(rset) def rset_vocabulary(self, rset): - _ = self._cw._ + if self.i18nable: + _ = self._cw._ + else: + _ = unicode return [(_(value), value) for value, in rset] def support_and(self): diff -r 2604545d7dd9 -r 61b28589d33f web/form.py --- a/web/form.py Sat May 29 10:06:07 2010 +0000 +++ b/web/form.py Sat May 29 10:18:02 2010 +0200 @@ -15,9 +15,7 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -"""abstract form classes for CubicWeb web client - -""" +"""abstract form classes for CubicWeb web client""" __docformat__ = "restructuredtext en" from warnings import warn @@ -80,8 +78,6 @@ __metaclass__ = metafieldsform __registry__ = 'forms' - internal_fields = ('__errorurl',) + controller.NAV_FORM_PARAMETERS - parent_form = None force_session_key = None domid = 'form' diff -r 2604545d7dd9 -r 61b28589d33f web/test/unittest_application.py --- a/web/test/unittest_application.py Sat May 29 10:06:07 2010 +0000 +++ b/web/test/unittest_application.py Sat May 29 10:18:02 2010 +0200 @@ -1,4 +1,3 @@ -# -*- coding: iso-8859-1 -*- # copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # @@ -16,9 +15,7 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -"""unit tests for cubicweb.web.application - -""" +"""unit tests for cubicweb.web.application""" import base64, Cookie import sys @@ -27,7 +24,7 @@ from logilab.common.testlib import TestCase, unittest_main from logilab.common.decorators import clear_cache -from cubicweb import AuthenticationError +from cubicweb import AuthenticationError, Unauthorized from cubicweb.devtools.testlib import CubicWebTC from cubicweb.devtools.fake import FakeRequest from cubicweb.web import LogOut, Redirect, INTERNAL_FIELD_VALUE @@ -299,6 +296,11 @@ self.commit() self.assertEquals(vreg.property_value('ui.language'), 'en') + def test_login_not_available_to_authenticated(self): + req = self.request() + ex = self.assertRaises(Unauthorized, self.app_publish, req, 'login') + self.assertEquals(str(ex), 'log out first') + def test_fb_login_concept(self): """see data/views.py""" self.set_option('auth-mode', 'cookie') diff -r 2604545d7dd9 -r 61b28589d33f web/test/unittest_views_basetemplates.py --- a/web/test/unittest_views_basetemplates.py Sat May 29 10:06:07 2010 +0000 +++ b/web/test/unittest_views_basetemplates.py Sat May 29 10:18:02 2010 +0200 @@ -15,9 +15,6 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -""" - -""" from cubicweb.devtools.testlib import CubicWebTC from cubicweb.devtools.htmlparser import DTDValidator @@ -26,7 +23,10 @@ def _login_labels(self): valid = self.content_type_validators.get('text/html', DTDValidator)() + req = self.request() + req.cnx.anonymous_connection = True page = valid.parse_string(self.vreg['views'].main_template(self.request(), 'login')) + req.cnx.anonymous_connection = False return page.find_tag('label') def test_label(self): diff -r 2604545d7dd9 -r 61b28589d33f web/test/unittest_views_searchrestriction.py --- a/web/test/unittest_views_searchrestriction.py Sat May 29 10:06:07 2010 +0000 +++ b/web/test/unittest_views_searchrestriction.py Sat May 29 10:18:02 2010 +0200 @@ -85,7 +85,7 @@ try: self.assertEquals(self._generate(select, 'in_state', 'subject', 'name'), "DISTINCT Any A,B ORDERBY B WHERE V is CWUser, " - "NOT V in_state VS, VS name 'published', " + "NOT EXISTS(V in_state VS), VS name 'published', " "V in_state A, A name B") finally: for rdefs in rschema.rdefs.values(): diff -r 2604545d7dd9 -r 61b28589d33f web/views/autoform.py --- a/web/views/autoform.py Sat May 29 10:06:07 2010 +0000 +++ b/web/views/autoform.py Sat May 29 10:18:02 2010 +0200 @@ -125,6 +125,7 @@ from logilab.mtconverter import xml_escape from logilab.common.decorators import iclassmethod, cached +from logilab.common.deprecation import deprecated from cubicweb import typed_eid, neg_role, uilib from cubicweb.schema import display_name @@ -643,6 +644,20 @@ # set this to a list of [(relation, role)] if you want to explictily tell # which relations should be edited display_fields = None + # action on the form tag + _default_form_action_path = 'validateform' + + # pre 3.8.3 compat + @property + def set_action(self, action): + self._action = action + @deprecated('[3.9] use form.form_action()') + def get_action(self): + try: + return self._action + except AttributeError: + return self._cw.build_url(self._default_form_action_path) + action = property(get_action, set_action) @iclassmethod def field_by_name(cls_or_self, name, role=None, eschema=None): @@ -713,21 +728,6 @@ return None return self.maxrelitems + 1 - def action(self): - """return the form's action attribute. Default to validateform if not - explicitly overriden. - """ - try: - return self._action - except AttributeError: - return self._cw.build_url('validateform') - - def set_action(self, value): - """override default action""" - self._action = value - - action = property(action, set_action) - # autoform specific fields ################################################# def _generic_relations_field(self): diff -r 2604545d7dd9 -r 61b28589d33f web/views/basecontrollers.py --- a/web/views/basecontrollers.py Sat May 29 10:06:07 2010 +0000 +++ b/web/views/basecontrollers.py Sat May 29 10:18:02 2010 +0200 @@ -31,7 +31,7 @@ from cubicweb import (NoSelectableObject, ObjectNotFound, ValidationError, AuthenticationError, typed_eid) from cubicweb.utils import CubicWebJsonEncoder -from cubicweb.selectors import authenticated_user, match_form_params +from cubicweb.selectors import authenticated_user, anonymous_user, match_form_params from cubicweb.mail import format_mail from cubicweb.web import Redirect, RemoteCallFailed, DirectResponse, json_dumps, json from cubicweb.web.controller import Controller @@ -78,6 +78,7 @@ class LoginController(Controller): __regid__ = 'login' + __select__ = anonymous_user() def publish(self, rset=None): """log in the instance""" @@ -314,6 +315,9 @@ for name, value in zip(names, values): # remove possible __action_xxx inputs if name.startswith('__action'): + if action is None: + # strip '__action_' to get the actual action name + action = name[9:] continue # form.setdefault(name, []).append(value) if name in form: diff -r 2604545d7dd9 -r 61b28589d33f web/views/basetemplates.py --- a/web/views/basetemplates.py Sat May 29 10:06:07 2010 +0000 +++ b/web/views/basetemplates.py Sat May 29 10:18:02 2010 +0200 @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # @@ -16,16 +15,15 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -"""default templates for CubicWeb web client +"""default templates for CubicWeb web client""" -""" __docformat__ = "restructuredtext en" from logilab.mtconverter import xml_escape from logilab.common.deprecation import class_renamed from cubicweb.appobject import objectify_selector -from cubicweb.selectors import match_kwargs, no_cnx +from cubicweb.selectors import match_kwargs, no_cnx, anonymous_user from cubicweb.view import View, MainTemplate, NOINDEX, NOFOLLOW from cubicweb.utils import UStringIO from cubicweb.schema import display_name @@ -60,6 +58,7 @@ class LogInTemplate(LogInOutTemplate): __regid__ = 'login' + __select__ = anonymous_user() title = 'log in' def content(self, w): @@ -80,6 +79,7 @@ xml_escape(indexurl), self._cw._('go back to the index page'))) + @objectify_selector def templatable_view(cls, req, rset, *args, **kwargs): view = kwargs.pop('view', None) @@ -446,9 +446,10 @@ form_buttons = [fw.SubmitButton(label=_('log in'), attrs={'class': 'loginButton'})] - @property - def action(self): - return xml_escape(login_form_url(self._cw)) + def form_action(self): + if self.action is None: + return login_form_url(self._cw) + return super(LogForm, self).form_action() class LogFormView(View): diff -r 2604545d7dd9 -r 61b28589d33f web/views/cwproperties.py --- a/web/views/cwproperties.py Sat May 29 10:06:07 2010 +0000 +++ b/web/views/cwproperties.py Sat May 29 10:18:02 2010 +0200 @@ -121,9 +121,11 @@ # user's preference but not site's configuration for key in vreg.user_property_keys(self.__regid__=='systempropertiesform'): parts = key.split('.') - if parts[0] in vreg: + if parts[0] in vreg and len(parts) >= 3: # appobject configuration - reg, oid, propid = parts + reg = parts[0] + propid = parts[-1] + oid = '.'.join(parts[1:-1]) groupedopts.setdefault(reg, {}).setdefault(oid, []).append(key) else: mainopts.setdefault(parts[0], []).append(key) diff -r 2604545d7dd9 -r 61b28589d33f web/views/editforms.py --- a/web/views/editforms.py Sat May 29 10:06:07 2010 +0000 +++ b/web/views/editforms.py Sat May 29 10:18:02 2010 +0200 @@ -17,8 +17,8 @@ # with CubicWeb. If not, see . """Set of HTML automatic forms to create, delete, copy or edit a single entity or a list of entities of the same type +""" -""" __docformat__ = "restructuredtext en" _ = unicode @@ -27,10 +27,11 @@ from logilab.mtconverter import xml_escape from logilab.common.decorators import cached +from cubicweb import tags from cubicweb.selectors import (match_kwargs, one_line_rset, non_final_entity, specified_etype_implements, implements, yes) from cubicweb.view import EntityView -from cubicweb import tags +from cubicweb.schema import display_name from cubicweb.web import uicfg, stdmsgs, eid_param, dumps, \ formfields as ff, formwidgets as fw from cubicweb.web.form import FormViewMixIn, FieldNotFound @@ -306,7 +307,8 @@ self._cw.add_js('cubicweb.edition.js') self._cw.add_css('cubicweb.form.css') if default is None: - default = xml_escape(self._cw._('')) + default = xml_escape(self._cw._('<%s not specified>') + % display_name(self._cw, rtype, role)) schema = self._cw.vreg.schema entity = self.cw_rset.get_entity(row, col) rschema = schema.rschema(rtype) diff -r 2604545d7dd9 -r 61b28589d33f web/views/formrenderers.py --- a/web/views/formrenderers.py Sat May 29 10:06:07 2010 +0000 +++ b/web/views/formrenderers.py Sat May 29 10:18:02 2010 +0200 @@ -174,12 +174,8 @@ enctype = 'multipart/form-data' else: enctype = 'application/x-www-form-urlencoded' - if form.action is None: - action = self._cw.build_url('edit') - else: - action = form.action tag = ('
. -"""security management and error screens +"""security management and error screens""" - -""" __docformat__ = "restructuredtext en" _ = unicode diff -r 2604545d7dd9 -r 61b28589d33f web/views/primary.py --- a/web/views/primary.py Sat May 29 10:06:07 2010 +0000 +++ b/web/views/primary.py Sat May 29 10:18:02 2010 +0200 @@ -176,7 +176,8 @@ warn('[3.5] box views should now be defined as a 4-uple (label, rset, vid, dispctrl), ' 'please update %s' % self.__class__.__name__, DeprecationWarning) - label, rset, vid = box + label, rset, vid = box + dispctrl = {} self.w(u'
') self.wview(vid, rset, title=label, initargs={'dispctrl': dispctrl}) self.w(u'
') diff -r 2604545d7dd9 -r 61b28589d33f web/views/schema.py --- a/web/views/schema.py Sat May 29 10:06:07 2010 +0000 +++ b/web/views/schema.py Sat May 29 10:18:02 2010 +0200 @@ -138,9 +138,9 @@ """display schema information (graphically, listing tables...) in tabs""" __regid__ = 'schema' title = _('instance schema') - tabs = [_('schema-image'), _('schema-entity-types'), + tabs = [_('schema-diagram'), _('schema-entity-types'), _('schema-relation-types'), _('schema-security')] - default_tab = 'schema-image' + default_tab = 'schema-diagram' def call(self): self.w(u'

%s

' % _('Schema of the data model')) @@ -148,7 +148,7 @@ class SchemaImageTab(StartupView): - __regid__ = 'schema-image' + __regid__ = 'schema-diagram' def call(self): self.w(_(u'
This schema of the data model excludes the ' diff -r 2604545d7dd9 -r 61b28589d33f web/views/startup.py --- a/web/views/startup.py Sat May 29 10:06:07 2010 +0000 +++ b/web/views/startup.py Sat May 29 10:18:02 2010 +0200 @@ -97,7 +97,8 @@ self.startupviews_table() def startupviews_table(self): - for v in self._cw.vreg['views'].possible_views(self._cw, None): + views = self._cw.vreg['views'].possible_views(self._cw, None) + for v in sorted(views, key=lambda x: self._cw._(x.title)): if v.category != 'startupview' or v.__regid__ in ('index', 'tree', 'manage'): continue self.w('

%s

' % ( diff -r 2604545d7dd9 -r 61b28589d33f web/views/tableview.py --- a/web/views/tableview.py Sat May 29 10:06:07 2010 +0000 +++ b/web/views/tableview.py Sat May 29 10:18:02 2010 +0200 @@ -15,10 +15,7 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -"""generic table view, including filtering abilities - - -""" +"""generic table view, including filtering abilities""" __docformat__ = "restructuredtext en" try: @@ -79,7 +76,7 @@ # drop False / None values from vidargs vidargs = dict((k, v) for k, v in vidargs.iteritems() if v) w(u'' % - xml_escape(dumps([divid, 'table', False, vidargs]))) + xml_escape(dumps([divid, self.__regid__, False, vidargs]))) w(u'
' % (divid, hidden and 'hidden' or '')) w(u'' % divid) w(u'') diff -r 2604545d7dd9 -r 61b28589d33f web/wdoc/ChangeLog_en --- a/web/wdoc/ChangeLog_en Sat May 29 10:06:07 2010 +0000 +++ b/web/wdoc/ChangeLog_en Sat May 29 10:18:02 2010 +0200 @@ -4,6 +4,36 @@ .. _SPARQL: http://www.w3.org/TR/rdf-sparql-query/ .. _schema: schema .. _OWL: http://www.w3.org/TR/owl-features/ +.. _pdfexport: http://www.cubicweb.org/project/cubicweb-pdfexport + +2010-04-20 -- 3.8.0 + * nicer schema_ and workflow views (clickable image!) + + * more power to undo, though not yet complete (missing entity updates, soon available...) + + * pdf export functionnality moved to its own cube. If it's no more + present on this site while you found it useful, ask you site + administrator to install the pdfexport_ cube. + + +2010-03-16 -- 3.7.0 + * experimental support for undoing of deletion. If you're not proposed to *undo* + deletion of one or several entities, ask you site administrator to activate + the feature. + + +2010-02-10 -- 3.6.0 + * nice 'demo widget' to edit bookmark's path, e.g. a relative url, splitted + as path and parameters and dealing nicely with url encodings. Try to + edit your bookmarks! + + * hell lot of refactorings, but you should hopefuly not see that from the outside + +2009-09-17 -- 3.5.0 + + * selectable workflows: authorized users may change the workflow used + by some workflowable entities + 2009-08-07 -- 3.4.0 diff -r 2604545d7dd9 -r 61b28589d33f web/wdoc/ChangeLog_fr --- a/web/wdoc/ChangeLog_fr Sat May 29 10:06:07 2010 +0000 +++ b/web/wdoc/ChangeLog_fr Sat May 29 10:18:02 2010 +0200 @@ -4,6 +4,45 @@ .. _SPARQL: http://www.w3.org/TR/rdf-sparql-query/ .. _schema: schema .. _OWL: http://www.w3.org/TR/owl-features/ +.. _pdfexport: http://www.cubicweb.org/project/cubicweb-pdfexport + +2010-04-20 -- 3.8.0 + + * amélioration des vues de schema_ et des vues de workflows + (images clickable !) + + * meilleure support du "undo", mais il manque toujours le support + sur la modification d'entité (bientôt...) + + * la fonctionnalité d'export d'pdf a été déplacé dans son propre + cube. Si cette fonctionalité n'est plus disponible sur ce site et + que vous la trouviez utile, demander à l'administrateur + d'installer le cube pdfexport_. + + +2010-03-16 -- 3.7.0 + + * support experimental pour l'annulation ("undo") de la + suppression. Si, après une suppression d'une ou plusieurs + entités, on ne vous propose pas d'annuler l'opération, demander à + l'administrateur d'activé la fonctionnalité + + +2010-02-10 -- 3.6.0 + + * nouvelle widget (de démonstration :) pour éditer le chemin des + signets. Celui-ci, une url relative finalement, est décomposée de + chemin et paramètres que vous pouvez éditer individuellement et + surtout lisiblement car la gestion de l'échappement de l'url est + géré de manière transparente + + * beaucoup de refactoring, mais vous ne devriez rien remarquer :) + +2009-09-17 -- 3.5.0 + + * workflow sélectionnable: les utilisateurs autorisés peuvent + changer le workflow à utilister pour les entités le supportant + 2009-08-07 -- 3.4.0 diff -r 2604545d7dd9 -r 61b28589d33f web/webconfig.py --- a/web/webconfig.py Sat May 29 10:06:07 2010 +0000 +++ b/web/webconfig.py Sat May 29 10:18:02 2010 +0200 @@ -83,20 +83,20 @@ {'type' : 'string', 'default': None, 'help': 'login of the CubicWeb user account to use for anonymous user (if you want to allow anonymous)', - 'group': 'main', 'level': 1, + 'group': 'web', 'level': 1, }), ('anonymous-password', {'type' : 'string', 'default': None, 'help': 'password of the CubicWeb user account to use for anonymous user, ' 'if anonymous-user is set', - 'group': 'main', 'level': 1, + 'group': 'web', 'level': 1, }), ('query-log-file', {'type' : 'string', 'default': None, 'help': 'web instance query log file', - 'group': 'main', 'level': 3, + 'group': 'web', 'level': 3, }), # web configuration ('https-url',