# HG changeset patch # User Julien Cristau # Date 1426686107 -3600 # Node ID 3231fd2fa7a5d7f8e0065195e27e57fdf231da0e # Parent 512ba8f37bd4d2693b81bdec5d122ec03f865af3# Parent cc75eb44d95af83f08fbdfbc6809e2c0857790c8 merge 3.19.11 into 3.20 diff -r 512ba8f37bd4 -r 3231fd2fa7a5 .hgtags --- a/.hgtags Thu Mar 12 14:21:36 2015 +0100 +++ b/.hgtags Wed Mar 18 14:41:47 2015 +0100 @@ -401,6 +401,9 @@ 3bab0b9b0ee7355a6fea45c2adca88bffe130e5d cubicweb-version-3.19.10 3bab0b9b0ee7355a6fea45c2adca88bffe130e5d cubicweb-debian-version-3.19.10-1 3bab0b9b0ee7355a6fea45c2adca88bffe130e5d cubicweb-centos-version-3.19.10-1 +1ae64186af9448dffbeebdef910c8c7391c04313 cubicweb-version-3.19.11 +1ae64186af9448dffbeebdef910c8c7391c04313 cubicweb-debian-version-3.19.11-1 +1ae64186af9448dffbeebdef910c8c7391c04313 cubicweb-centos-version-3.19.11-1 7e6b7739afe6128589ad51b0318decb767cbae36 cubicweb-version-3.20.0 7e6b7739afe6128589ad51b0318decb767cbae36 cubicweb-debian-version-3.20.0-1 7e6b7739afe6128589ad51b0318decb767cbae36 cubicweb-centos-version-3.20.0-1 diff -r 512ba8f37bd4 -r 3231fd2fa7a5 __init__.py --- a/__init__.py Thu Mar 12 14:21:36 2015 +0100 +++ b/__init__.py Wed Mar 18 14:41:47 2015 +0100 @@ -69,12 +69,12 @@ class Binary(StringIO): """customize StringIO to make sure we don't use unicode""" def __init__(self, buf=''): - assert isinstance(buf, (str, buffer)), \ + assert isinstance(buf, (str, buffer, bytearray)), \ "Binary objects must use raw strings, not %s" % buf.__class__ StringIO.__init__(self, buf) def write(self, data): - assert isinstance(data, (str, buffer)), \ + assert isinstance(data, (str, buffer, bytearray)), \ "Binary objects must use raw strings, not %s" % data.__class__ StringIO.write(self, data) diff -r 512ba8f37bd4 -r 3231fd2fa7a5 __pkginfo__.py diff -r 512ba8f37bd4 -r 3231fd2fa7a5 cubicweb.spec diff -r 512ba8f37bd4 -r 3231fd2fa7a5 dataimport.py --- a/dataimport.py Thu Mar 12 14:21:36 2015 +0100 +++ b/dataimport.py Wed Mar 18 14:41:47 2015 +0100 @@ -69,6 +69,7 @@ import cPickle import os.path as osp import inspect +from base64 import b64encode from collections import defaultdict from copy import copy from datetime import date, datetime, time diff -r 512ba8f37bd4 -r 3231fd2fa7a5 debian/changelog --- a/debian/changelog Thu Mar 12 14:21:36 2015 +0100 +++ b/debian/changelog Wed Mar 18 14:41:47 2015 +0100 @@ -28,6 +28,12 @@ -- Julien Cristau Tue, 06 Jan 2015 18:11:03 +0100 +cubicweb (3.19.11-1) unstable; urgency=low + + * new upstream release + + -- Julien Cristau Wed, 18 Mar 2015 11:55:52 +0100 + cubicweb (3.19.10-1) unstable; urgency=low * New upstream release. diff -r 512ba8f37bd4 -r 3231fd2fa7a5 debian/control --- a/debian/control Thu Mar 12 14:21:36 2015 +0100 +++ b/debian/control Wed Mar 18 14:41:47 2015 +0100 @@ -204,6 +204,7 @@ ${python:Depends}, cubicweb-server (= ${source:Version}), cubicweb-web (= ${source:Version}), + cubicweb-etwist (= ${source:Version}), python-pysqlite2 Suggests: w3c-dtd-xhtml, diff -r 512ba8f37bd4 -r 3231fd2fa7a5 misc/migration/3.14.4_Any.py --- a/misc/migration/3.14.4_Any.py Thu Mar 12 14:21:36 2015 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,10 +0,0 @@ -from yams import schema2sql as y2sql - -dbhelper = repo.system_source.dbhelper -rdefdef = schema['CWSource'].rdef('name') -attrtype = y2sql.type_from_constraints(dbhelper, rdefdef.object, rdefdef.constraints).split()[0] - -cursor = session.cnxset.cu -sql('UPDATE entities SET asource = source WHERE asource is NULL') -dbhelper.change_col_type(cursor, 'entities', 'asource', attrtype, False) -dbhelper.change_col_type(cursor, 'entities', 'source', attrtype, False) diff -r 512ba8f37bd4 -r 3231fd2fa7a5 misc/migration/3.18.0_Any.py --- a/misc/migration/3.18.0_Any.py Thu Mar 12 14:21:36 2015 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,151 +0,0 @@ -driver = config.system_source_config['db-driver'] -if not (driver == 'postgres' or driver.startswith('sqlserver')): - import sys - print >>sys.stderr, 'This migration is not supported for backends other than sqlserver or postgres (yet).' - sys.exit(1) - -add_relation_definition('CWAttribute', 'add_permission', 'CWGroup') -add_relation_definition('CWAttribute', 'add_permission', 'RQLExpression') - -# a bad defaultval in 3.13.8 schema was fixed in 3.13.9, but the migration was missed -rql('SET ATTR defaultval NULL WHERE ATTR from_entity E, E name "CWSource", ATTR relation_type T, T name "in_synchronization"') - -# the migration gets confused when we change rdefs out from under it. So -# explicitly remove this size constraint so it doesn't stick around and break -# things later. -rdefeid = schema['defaultval'].rdefs.values()[0].eid -rql('DELETE CWConstraint C WHERE C cstrtype T, T name "SizeConstraint", R constrained_by C, R eid %(eid)s', {'eid': rdefeid}) - -sync_schema_props_perms('defaultval') - -def convert_defaultval(cwattr, default): - from decimal import Decimal - import yams - from cubicweb import Binary - if default is None: - return - if isinstance(default, Binary): - # partially migrated instance, try to be idempotent - return default - atype = cwattr.to_entity[0].name - if atype == 'Boolean': - # boolean attributes with default=False were stored as '' - assert default in ('True', 'False', ''), repr(default) - default = default == 'True' - elif atype in ('Int', 'BigInt'): - default = int(default) - elif atype == 'Float': - default = float(default) - elif atype == 'Decimal': - default = Decimal(default) - elif atype in ('Date', 'Datetime', 'TZDatetime', 'Time'): - try: - # handle NOW and TODAY, keep them stored as strings - yams.KEYWORD_MAP[atype][default.upper()] - default = default.upper() - except KeyError: - # otherwise get an actual date or datetime - default = yams.DATE_FACTORY_MAP[atype](default) - else: - assert atype == 'String', atype - default = unicode(default) - return Binary.zpickle(default) - -dbh = repo.system_source.dbhelper - - -sql('ALTER TABLE cw_cwattribute ADD new_defaultval %s' % dbh.TYPE_MAPPING['Bytes']) - -for cwattr in rql('CWAttribute X').entities(): - olddefault = cwattr.defaultval - if olddefault is not None: - req = "UPDATE cw_cwattribute SET new_defaultval = %(val)s WHERE cw_eid = %(eid)s" - args = {'val': dbh.binary_value(convert_defaultval(cwattr, olddefault).getvalue()), 'eid': cwattr.eid} - sql(req, args, ask_confirm=False) - -sql('ALTER TABLE cw_cwattribute DROP COLUMN cw_defaultval') -if driver == 'postgres': - sql('ALTER TABLE cw_cwattribute RENAME COLUMN new_defaultval TO cw_defaultval') -else: # sqlserver - sql("sp_rename 'cw_cwattribute.new_defaultval', 'cw_defaultval', 'COLUMN'") - - -# Set object type to "Bytes" for CWAttribute's "defaultval" attribute -rql('SET X to_entity B WHERE X is CWAttribute, X from_entity Y, Y name "CWAttribute", ' - 'X relation_type Z, Z name "defaultval", B name "Bytes", NOT X to_entity B') - -oldrdef = schema['CWAttribute'].rdef('defaultval') -import yams.buildobjs as ybo -newrdef = ybo.RelationDefinition('CWAttribute', 'defaultval', 'Bytes') -newrdef.eid = oldrdef.eid -schema.add_relation_def(newrdef) -schema.del_relation_def('CWAttribute', 'defaultval', 'String') - -commit() - -sync_schema_props_perms('defaultval') - -for rschema in schema.relations(): - if rschema.symmetric: - subjects = set(repr(e.type) for e in rschema.subjects()) - objects = set(repr(e.type) for e in rschema.objects()) - assert subjects == objects - martians = set(str(eid) for eid, in sql('SELECT eid_to FROM %s_relation, entities WHERE eid_to = eid AND type NOT IN (%s)' % - (rschema.type, ','.join(subjects)))) - martians |= set(str(eid) for eid, in sql('SELECT eid_from FROM %s_relation, entities WHERE eid_from = eid AND type NOT IN (%s)' % - (rschema.type, ','.join(subjects)))) - if martians: - martians = ','.join(martians) - print 'deleting broken relations %s for eids %s' % (rschema.type, martians) - sql('DELETE FROM %s_relation WHERE eid_from IN (%s) OR eid_to IN (%s)' % (rschema.type, martians, martians)) - with session.deny_all_hooks_but(): - rql('SET X %(r)s Y WHERE Y %(r)s X, NOT X %(r)s Y' % {'r': rschema.type}) - commit() - - -# multi columns unique constraints regeneration -from cubicweb.server import schemaserial - -# syncschema hooks would try to remove indices but -# 1) we already do that below -# 2) the hook expects the CWUniqueTogetherConstraint.name attribute that hasn't -# yet been added -with session.allow_all_hooks_but('syncschema'): - rql('DELETE CWUniqueTogetherConstraint C') -commit() - -add_attribute('CWUniqueTogetherConstraint', 'name') - -# low-level wipe code for postgres & sqlserver, plain sql ... -if driver == 'postgres': - for indexname, in sql('select indexname from pg_indexes'): - if indexname.startswith('unique_'): - print 'dropping index', indexname - sql('DROP INDEX %s' % indexname) - commit() -elif driver.startswith('sqlserver'): - for viewname, in sql('select name from sys.views'): - if viewname.startswith('utv_'): - print 'dropping view (index should be cascade-deleted)', viewname - sql('DROP VIEW %s' % viewname) - commit() - -# recreate the constraints, hook will lead to low-level recreation -for eschema in sorted(schema.entities()): - if eschema._unique_together: - print 'recreate unique indexes for', eschema - rql_args = schemaserial.uniquetogether2rqls(eschema) - for rql, args in rql_args: - args['x'] = eschema.eid - session.execute(rql, args) -commit() - -# all attributes perms have to be refreshed ... -for rschema in sorted(schema.relations()): - if rschema.final: - if rschema.type in fsschema: - print 'sync perms for', rschema.type - sync_schema_props_perms(rschema.type, syncprops=False, ask_confirm=False, commit=False) - else: - print 'WARNING: attribute %s missing from fs schema' % rschema.type -commit() diff -r 512ba8f37bd4 -r 3231fd2fa7a5 misc/migration/bootstrapmigration_repository.py --- a/misc/migration/bootstrapmigration_repository.py Thu Mar 12 14:21:36 2015 +0100 +++ b/misc/migration/bootstrapmigration_repository.py Wed Mar 18 14:41:47 2015 +0100 @@ -49,6 +49,27 @@ cursor.execute(dbh.sql_restart_numrange('entities_id_seq', initial_value=lasteid)) session.commit() +if applcubicwebversion <= (3, 13, 0) and cubicwebversion >= (3, 13, 1): + sql('ALTER TABLE entities ADD asource VARCHAR(64)') + sql('UPDATE entities SET asource=cw_name ' + 'FROM cw_CWSource, cw_source_relation ' + 'WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid') + commit() + +if applcubicwebversion <= (3, 14, 4) and cubicwebversion >= (3, 14, 4): + from yams import schema2sql as y2sql + dbhelper = repo.system_source.dbhelper + rdefdef = schema['CWSource'].rdef('name') + attrtype = y2sql.type_from_constraints(dbhelper, rdefdef.object, rdefdef.constraints).split()[0] + cursor = session.cnxset.cu + sql('UPDATE entities SET asource = source WHERE asource is NULL') + dbhelper.change_col_type(cursor, 'entities', 'asource', attrtype, False) + dbhelper.change_col_type(cursor, 'entities', 'source', attrtype, False) + + # we now have a functional asource column, start using the normal eid_type_source method + if repo.system_source.eid_type_source == repo.system_source.eid_type_source_pre_131: + del repo.system_source.eid_type_source + if applcubicwebversion < (3, 19, 0) and cubicwebversion >= (3, 19, 0): try: # need explicit drop of the indexes on some database systems (sqlserver) @@ -71,6 +92,163 @@ add_entity_type('CWComputedRType') commit() +if schema['TZDatetime'].eid is None: + add_entity_type('TZDatetime', auto=False) +if schema['TZTime'].eid is None: + add_entity_type('TZTime', auto=False) + +if applcubicwebversion < (3, 18, 0) and cubicwebversion >= (3, 18, 0): + driver = config.system_source_config['db-driver'] + if not (driver == 'postgres' or driver.startswith('sqlserver')): + import sys + print >>sys.stderr, 'This migration is not supported for backends other than sqlserver or postgres (yet).' + sys.exit(1) + + add_relation_definition('CWAttribute', 'add_permission', 'CWGroup') + add_relation_definition('CWAttribute', 'add_permission', 'RQLExpression') + + # a bad defaultval in 3.13.8 schema was fixed in 3.13.9, but the migration was missed + rql('SET ATTR defaultval NULL WHERE ATTR from_entity E, E name "CWSource", ATTR relation_type T, T name "in_synchronization"') + + # the migration gets confused when we change rdefs out from under it. So + # explicitly remove this size constraint so it doesn't stick around and break + # things later. + rdefeid = schema['defaultval'].rdefs.values()[0].eid + rql('DELETE CWConstraint C WHERE C cstrtype T, T name "SizeConstraint", R constrained_by C, R eid %(eid)s', {'eid': rdefeid}) + + sync_schema_props_perms('defaultval') + + def convert_defaultval(cwattr, default): + from decimal import Decimal + import yams + from cubicweb import Binary + if default is None: + return + if isinstance(default, Binary): + # partially migrated instance, try to be idempotent + return default + atype = cwattr.to_entity[0].name + if atype == 'Boolean': + # boolean attributes with default=False were stored as '' + assert default in ('True', 'False', ''), repr(default) + default = default == 'True' + elif atype in ('Int', 'BigInt'): + default = int(default) + elif atype == 'Float': + default = float(default) + elif atype == 'Decimal': + default = Decimal(default) + elif atype in ('Date', 'Datetime', 'TZDatetime', 'Time'): + try: + # handle NOW and TODAY, keep them stored as strings + yams.KEYWORD_MAP[atype][default.upper()] + default = default.upper() + except KeyError: + # otherwise get an actual date or datetime + default = yams.DATE_FACTORY_MAP[atype](default) + else: + assert atype == 'String', atype + default = unicode(default) + return Binary.zpickle(default) + + dbh = repo.system_source.dbhelper + + + sql('ALTER TABLE cw_cwattribute ADD new_defaultval %s' % dbh.TYPE_MAPPING['Bytes']) + + for cwattr in rql('CWAttribute X').entities(): + olddefault = cwattr.defaultval + if olddefault is not None: + req = "UPDATE cw_cwattribute SET new_defaultval = %(val)s WHERE cw_eid = %(eid)s" + args = {'val': dbh.binary_value(convert_defaultval(cwattr, olddefault).getvalue()), 'eid': cwattr.eid} + sql(req, args, ask_confirm=False) + + sql('ALTER TABLE cw_cwattribute DROP COLUMN cw_defaultval') + if driver == 'postgres': + sql('ALTER TABLE cw_cwattribute RENAME COLUMN new_defaultval TO cw_defaultval') + else: # sqlserver + sql("sp_rename 'cw_cwattribute.new_defaultval', 'cw_defaultval', 'COLUMN'") + + + # Set object type to "Bytes" for CWAttribute's "defaultval" attribute + rql('SET X to_entity B WHERE X is CWAttribute, X from_entity Y, Y name "CWAttribute", ' + 'X relation_type Z, Z name "defaultval", B name "Bytes", NOT X to_entity B') + + oldrdef = schema['CWAttribute'].rdef('defaultval') + import yams.buildobjs as ybo + newrdef = ybo.RelationDefinition('CWAttribute', 'defaultval', 'Bytes') + newrdef.eid = oldrdef.eid + schema.add_relation_def(newrdef) + schema.del_relation_def('CWAttribute', 'defaultval', 'String') + + commit() + + sync_schema_props_perms('defaultval') + + for rschema in schema.relations(): + if rschema.symmetric: + subjects = set(repr(e.type) for e in rschema.subjects()) + objects = set(repr(e.type) for e in rschema.objects()) + assert subjects == objects + martians = set(str(eid) for eid, in sql('SELECT eid_to FROM %s_relation, entities WHERE eid_to = eid AND type NOT IN (%s)' % + (rschema.type, ','.join(subjects)))) + martians |= set(str(eid) for eid, in sql('SELECT eid_from FROM %s_relation, entities WHERE eid_from = eid AND type NOT IN (%s)' % + (rschema.type, ','.join(subjects)))) + if martians: + martians = ','.join(martians) + print 'deleting broken relations %s for eids %s' % (rschema.type, martians) + sql('DELETE FROM %s_relation WHERE eid_from IN (%s) OR eid_to IN (%s)' % (rschema.type, martians, martians)) + with session.deny_all_hooks_but(): + rql('SET X %(r)s Y WHERE Y %(r)s X, NOT X %(r)s Y' % {'r': rschema.type}) + commit() + + + # multi columns unique constraints regeneration + from cubicweb.server import schemaserial + + # syncschema hooks would try to remove indices but + # 1) we already do that below + # 2) the hook expects the CWUniqueTogetherConstraint.name attribute that hasn't + # yet been added + with session.allow_all_hooks_but('syncschema'): + rql('DELETE CWUniqueTogetherConstraint C') + commit() + add_attribute('CWUniqueTogetherConstraint', 'name') + + # low-level wipe code for postgres & sqlserver, plain sql ... + if driver == 'postgres': + for indexname, in sql('select indexname from pg_indexes'): + if indexname.startswith('unique_'): + print 'dropping index', indexname + sql('DROP INDEX %s' % indexname) + commit() + elif driver.startswith('sqlserver'): + for viewname, in sql('select name from sys.views'): + if viewname.startswith('utv_'): + print 'dropping view (index should be cascade-deleted)', viewname + sql('DROP VIEW %s' % viewname) + commit() + + # recreate the constraints, hook will lead to low-level recreation + for eschema in sorted(schema.entities()): + if eschema._unique_together: + print 'recreate unique indexes for', eschema + rql_args = schemaserial.uniquetogether2rqls(eschema) + for rql, args in rql_args: + args['x'] = eschema.eid + session.execute(rql, args) + commit() + + # all attributes perms have to be refreshed ... + for rschema in sorted(schema.relations()): + if rschema.final: + if rschema.type in fsschema: + print 'sync perms for', rschema.type + sync_schema_props_perms(rschema.type, syncprops=False, ask_confirm=False, commit=False) + else: + print 'WARNING: attribute %s missing from fs schema' % rschema.type + commit() + if applcubicwebversion < (3, 17, 0) and cubicwebversion >= (3, 17, 0): try: add_cube('sioc', update_database=False) @@ -91,18 +269,6 @@ 'cube, which is not installed. Continue anyway?'): raise -if applcubicwebversion <= (3, 13, 0) and cubicwebversion >= (3, 13, 1): - sql('ALTER TABLE entities ADD asource VARCHAR(64)') - sql('UPDATE entities SET asource=cw_name ' - 'FROM cw_CWSource, cw_source_relation ' - 'WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid') - commit() - -if schema['TZDatetime'].eid is None: - add_entity_type('TZDatetime', auto=False) -if schema['TZTime'].eid is None: - add_entity_type('TZTime', auto=False) - if applcubicwebversion <= (3, 14, 0) and cubicwebversion >= (3, 14, 0): if 'require_permission' in schema and not 'localperms'in repo.config.cubes(): @@ -113,14 +279,13 @@ raise ExecutionError('In cubicweb 3.14, CWPermission and related stuff ' 'has been moved to cube localperms. Install it first.') + if applcubicwebversion == (3, 6, 0) and cubicwebversion >= (3, 6, 0): CSTRMAP = dict(rql('Any T, X WHERE X is CWConstraintType, X name T', ask_confirm=False)) _add_relation_definition_no_perms('CWAttribute', 'update_permission', 'CWGroup') _add_relation_definition_no_perms('CWAttribute', 'update_permission', 'RQLExpression') rql('SET X update_permission Y WHERE X is CWAttribute, X add_permission Y') - drop_relation_definition('CWAttribute', 'add_permission', 'CWGroup') - drop_relation_definition('CWAttribute', 'add_permission', 'RQLExpression') drop_relation_definition('CWAttribute', 'delete_permission', 'CWGroup') drop_relation_definition('CWAttribute', 'delete_permission', 'RQLExpression') diff -r 512ba8f37bd4 -r 3231fd2fa7a5 server/serverconfig.py --- a/server/serverconfig.py Thu Mar 12 14:21:36 2015 +0100 +++ b/server/serverconfig.py Wed Mar 18 14:41:47 2015 +0100 @@ -237,7 +237,7 @@ }), ('zmq-address-sub', {'type' : 'csv', - 'default' : None, + 'default' : (), 'help': ('List of ZMQ addresses to subscribe to (requires pyzmq) ' '(of the form `tcp://:`)'), 'group': 'zmq', 'level': 1, diff -r 512ba8f37bd4 -r 3231fd2fa7a5 server/session.py --- a/server/session.py Thu Mar 12 14:21:36 2015 +0100 +++ b/server/session.py Wed Mar 18 14:41:47 2015 +0100 @@ -995,8 +995,8 @@ etype, extid, source = self.repo.type_and_source_from_eid(eid, self) metas = {'type': etype, 'source': source, 'extid': extid} if asdict: - metas['asource'] = meta['source'] # XXX pre 3.19 client compat - return meta + metas['asource'] = metas['source'] # XXX pre 3.19 client compat + return metas return etype, source, extid @_with_cnx_set diff -r 512ba8f37bd4 -r 3231fd2fa7a5 server/sources/native.py --- a/server/sources/native.py Thu Mar 12 14:21:36 2015 +0100 +++ b/server/sources/native.py Wed Mar 18 14:41:47 2015 +0100 @@ -862,7 +862,7 @@ res = list(res) if res[-1] is not None: res[-1] = b64decode(res[-1]) - res.append(res[1]) + res.append("system") return res def extid2eid(self, cnx, extid): diff -r 512ba8f37bd4 -r 3231fd2fa7a5 web/views/rdf.py --- a/web/views/rdf.py Thu Mar 12 14:21:36 2015 +0100 +++ b/web/views/rdf.py Wed Mar 18 14:41:47 2015 +0100 @@ -94,7 +94,7 @@ add( (cwuri, CW[rtype], URIRef(related.cwuri)) ) try: for item in xy.xeq('%s %s' % (entity.e_schema.type, rtype)): - add( (cwuri, urijoin(item), URIRef(related.cwuri)) ) + add( (cwuri, urijoin(item[1]), URIRef(related.cwuri)) ) except xy.UnsupportedVocabulary: pass else: