# HG changeset patch # User Aurélien Campeas # Date 1377263187 -7200 # Node ID c5eed908117dc7cb0d044d634abd8d5f449445eb # Parent 8a417555742665fca05b28c3f1c4901caf774f45 [schema] store default attribute values in a Bytes field, allowing python objects as default values Notes: * Binary objects grow two methods * we depend on a newer yams version * the code is quite simplified as a result of the whole change * a very old and nasty bug where bool(default values) evaluating to False not being properly migrated is killed Closes #2414591. diff -r 8a4175557426 -r c5eed908117d __init__.py --- a/__init__.py Thu Oct 17 11:34:03 2013 +0200 +++ b/__init__.py Fri Aug 23 15:06:27 2013 +0200 @@ -22,6 +22,8 @@ # ignore the pygments UserWarnings import warnings +import cPickle +import zlib warnings.filterwarnings('ignore', category=UserWarning, message='.*was already imported', module='.*pygments') @@ -120,6 +122,26 @@ binary.seek(0) return binary + def __eq__(self, other): + if not isinstance(other, Binary): + return False + return self.getvalue(), other.getvalue() + + + # Binary helpers to store/fetch python objects + + @classmethod + def zpickle(cls, obj): + """ return a Binary containing a gzipped pickle of obj """ + retval = cls() + retval.write(zlib.compress(cPickle.dumps(obj, protocol=2))) + return retval + + def unzpickle(self): + """ decompress and loads the stream before returning it """ + return cPickle.loads(zlib.decompress(self.getvalue())) + + def str_or_binary(value): if isinstance(value, Binary): return value @@ -127,7 +149,6 @@ BASE_CONVERTERS['Password'] = str_or_binary - # use this dictionary to rename entity types while keeping bw compat ETYPE_NAME_MAP = {} diff -r 8a4175557426 -r c5eed908117d __pkginfo__.py --- a/__pkginfo__.py Thu Oct 17 11:34:03 2013 +0200 +++ b/__pkginfo__.py Fri Aug 23 15:06:27 2013 +0200 @@ -43,7 +43,7 @@ 'logilab-common': '>= 0.60.0', 'logilab-mtconverter': '>= 0.8.0', 'rql': '>= 0.31.2', - 'yams': '>= 0.37.0', + 'yams': '>= 0.39.0', #gettext # for xgettext, msgcat, etc... # web dependancies 'simplejson': '>= 2.0.9', diff -r 8a4175557426 -r c5eed908117d cubicweb.spec --- a/cubicweb.spec Thu Oct 17 11:34:03 2013 +0200 +++ b/cubicweb.spec Fri Aug 23 15:06:27 2013 +0200 @@ -23,7 +23,7 @@ Requires: %{python}-logilab-common >= 0.59.0 Requires: %{python}-logilab-mtconverter >= 0.8.0 Requires: %{python}-rql >= 0.31.2 -Requires: %{python}-yams >= 0.37.0 +Requires: %{python}-yams >= 0.39.0 Requires: %{python}-logilab-database >= 1.10.0 Requires: %{python}-passlib Requires: %{python}-lxml diff -r 8a4175557426 -r c5eed908117d debian/control --- a/debian/control Thu Oct 17 11:34:03 2013 +0200 +++ b/debian/control Fri Aug 23 15:06:27 2013 +0200 @@ -15,7 +15,7 @@ python-unittest2 | python (>= 2.7), python-logilab-mtconverter, python-rql, - python-yams (>= 0.37), + python-yams (>= 0.39), python-lxml, Standards-Version: 3.9.1 Homepage: http://www.cubicweb.org @@ -152,7 +152,7 @@ gettext, python-logilab-mtconverter (>= 0.8.0), python-logilab-common (>= 0.59.0), - python-yams (>= 0.37.0), + python-yams (>= 0.39.0), python-rql (>= 0.31.2), python-lxml Recommends: diff -r 8a4175557426 -r c5eed908117d hooks/syncschema.py --- a/hooks/syncschema.py Thu Oct 17 11:34:03 2013 +0200 +++ b/hooks/syncschema.py Fri Aug 23 15:06:27 2013 +0200 @@ -28,7 +28,7 @@ from copy import copy from yams.schema import BASE_TYPES, RelationSchema, RelationDefinitionSchema -from yams import buildobjs as ybo, schema2sql as y2sql +from yams import buildobjs as ybo, schema2sql as y2sql, convert_default_value from logilab.common.decorators import clear_cache @@ -39,21 +39,6 @@ from cubicweb.server import hook, schemaserial as ss from cubicweb.server.sqlutils import SQL_PREFIX - -TYPE_CONVERTER = { # XXX - 'Boolean': bool, - 'Int': int, - 'BigInt': int, - 'Float': float, - 'Password': str, - 'String': unicode, - 'Date' : unicode, - 'Datetime' : unicode, - 'Time' : unicode, - 'TZDatetime' : unicode, - 'TZTime' : unicode, - } - # core entity and relation types which can't be removed CORE_TYPES = BASE_TYPES | SCHEMA_TYPES | META_RTYPES | set( ('CWUser', 'CWGroup','login', 'upassword', 'name', 'in_group')) @@ -437,11 +422,11 @@ def precommit_event(self): session = self.session entity = self.entity - # entity.defaultval is a string or None, but we need a correctly typed + # entity.defaultval is a Binary or None, but we need a correctly typed # value default = entity.defaultval if default is not None: - default = TYPE_CONVERTER[entity.otype.name](default) + default = default.unzpickle() props = {'default': default, 'indexed': entity.indexed, 'fulltextindexed': entity.fulltextindexed, @@ -493,20 +478,11 @@ # attribute is still set to False, so we've to ensure it's False rschema.final = True insert_rdef_on_subclasses(session, eschema, rschema, rdefdef, props) - # set default value, using sql for performance and to avoid - # modification_date update - if default: - if rdefdef.object in ('Date', 'Datetime', 'TZDatetime'): - # XXX may may want to use creation_date - if default == 'TODAY': - default = syssource.dbhelper.sql_current_date() - elif default == 'NOW': - default = syssource.dbhelper.sql_current_timestamp() - session.system_sql('UPDATE %s SET %s=%s' - % (table, column, default)) - else: - session.system_sql('UPDATE %s SET %s=%%(default)s' % (table, column), - {'default': default}) + # update existing entities with the default value of newly added attribute + if default is not None: + default = convert_default_value(self.rdefdef, default) + session.system_sql('UPDATE %s SET %s=%%(default)s' % (table, column), + {'default': default}) def revertprecommit_event(self): # revert changes on in memory schema diff -r 8a4175557426 -r c5eed908117d hooks/test/unittest_syncschema.py --- a/hooks/test/unittest_syncschema.py Thu Oct 17 11:34:03 2013 +0200 +++ b/hooks/test/unittest_syncschema.py Fri Aug 23 15:06:27 2013 +0200 @@ -19,7 +19,7 @@ from logilab.common.testlib import TestCase, unittest_main -from cubicweb import ValidationError +from cubicweb import ValidationError, Binary from cubicweb.schema import META_RTYPES from cubicweb.devtools.testlib import CubicWebTC from cubicweb.server.sqlutils import SQL_PREFIX @@ -74,9 +74,10 @@ self.commit() self.assertTrue(schema.has_entity('Societe2')) self.assertTrue(schema.has_relation('concerne2')) - attreid = self.execute('INSERT CWAttribute X: X cardinality "11", X defaultval "noname", ' + attreid = self.execute('INSERT CWAttribute X: X cardinality "11", X defaultval %(default)s, ' ' X indexed TRUE, X relation_type RT, X from_entity E, X to_entity F ' - 'WHERE RT name "name", E name "Societe2", F name "String"')[0][0] + 'WHERE RT name "name", E name "Societe2", F name "String"', + {'default': Binary.zpickle('noname')})[0][0] self._set_attr_perms(attreid) concerne2_rdef_eid = self.execute( 'INSERT CWRelation X: X cardinality "**", X relation_type RT, X from_entity E, X to_entity E ' @@ -290,8 +291,10 @@ def test_add_attribute_to_base_class(self): - attreid = self.execute('INSERT CWAttribute X: X cardinality "11", X defaultval "noname", X indexed TRUE, X relation_type RT, X from_entity E, X to_entity F ' - 'WHERE RT name "messageid", E name "BaseTransition", F name "String"')[0][0] + attreid = self.execute('INSERT CWAttribute X: X cardinality "11", X defaultval %(default)s, ' + 'X indexed TRUE, X relation_type RT, X from_entity E, X to_entity F ' + 'WHERE RT name "messageid", E name "BaseTransition", F name "String"', + {'default': Binary.zpickle('noname')})[0][0] assert self.execute('SET X read_permission Y WHERE X eid %(x)s, Y name "managers"', {'x': attreid}) self.commit() diff -r 8a4175557426 -r c5eed908117d misc/migration/3.18.0_Any.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/misc/migration/3.18.0_Any.py Fri Aug 23 15:06:27 2013 +0200 @@ -0,0 +1,60 @@ +sync_schema_props_perms('defaultval') + +def convert_defaultval(cwattr, default): + from decimal import Decimal + import yams + from cubicweb import Binary + if default is None: + return + atype = cwattr.to_entity[0].name + if atype == 'Boolean': + assert default in ('True', 'False'), default + default = default == 'True' + elif atype in ('Int', 'BigInt'): + default = int(default) + elif atype == 'Float': + default = float(default) + elif atype == 'Decimal': + default = Decimal(default) + elif atype in ('Date', 'Datetime', 'TZDatetime', 'Time'): + try: + # handle NOW and TODAY, keep them stored as strings + yams.KEYWORD_MAP[atype][default.upper()] + default = default.upper() + except KeyError: + # otherwise get an actual date or datetime + default = yams.DATE_FACTORY_MAP[atype](default) + else: + assert atype == 'String', atype + default = unicode(default) + return Binary.zpickle(default) + +dbh = repo.system_source.dbhelper +driver = config.sources()['system']['db-driver'] +if driver == 'postgres' or driver.startswith('sqlserver'): + sql('ALTER TABLE cw_cwattribute ADD new_defaultval %s' % dbh.TYPE_MAPPING['Bytes']) +else: + assert False, 'upgrade not supported on this database backend' + +for cwattr in rql('CWAttribute X').entities(): + olddefault = cwattr.defaultval + if olddefault is not None: + req = "UPDATE cw_cwattribute SET new_defaultval = %(val)s WHERE cw_eid = %(eid)s" + args = {'val': dbh.binary_value(convert_defaultval(cwattr, olddefault).getvalue()), 'eid': cwattr.eid} + sql(req, args, ask_confirm=False) + +sql('ALTER TABLE cw_cwattribute DROP COLUMN cw_defaultval') +if config.sources()['system']['db-driver'] == 'postgres': + sql('ALTER TABLE cw_cwattribute RENAME COLUMN new_defaultval TO cw_defaultval') +else: + sql("sp_rename 'cw_cwattribute.new_defaultval', 'cw_defaultval', 'COLUMN'") + +# Set object type to "Bytes" for CWAttribute's "defaultval" attribute +rql('SET X to_entity B WHERE X is CWAttribute, X from_entity Y, Y name "CWAttribute", ' + 'X relation_type Z, Z name "defaultval", B name "Bytes"') + +from yams import buildobjs as ybo +schema.add_relation_def(ybo.RelationDefinition('CWAttribute', 'defaultval', 'Bytes')) +schema.del_relation_def('CWAttribute', 'defaultval', 'String') + +commit() diff -r 8a4175557426 -r c5eed908117d schemas/bootstrap.py --- a/schemas/bootstrap.py Thu Oct 17 11:34:03 2013 +0200 +++ b/schemas/bootstrap.py Fri Aug 23 15:06:27 2013 +0200 @@ -83,7 +83,7 @@ indexed = Boolean(description=_('create an index for quick search on this attribute')) fulltextindexed = Boolean(description=_('index this attribute\'s value in the plain text index')) internationalizable = Boolean(description=_('is this attribute\'s value translatable')) - defaultval = String(maxsize=256) + defaultval = Bytes(description=_('default value as gziped pickled python object')) extra_props = Bytes(description=_('additional type specific properties')) description = RichString(internationalizable=True, diff -r 8a4175557426 -r c5eed908117d server/schemaserial.py --- a/server/schemaserial.py Thu Oct 17 11:34:03 2013 +0200 +++ b/server/schemaserial.py Fri Aug 23 15:06:27 2013 +0200 @@ -27,7 +27,7 @@ from yams import BadSchemaDefinition, schema as schemamod, buildobjs as ybo -from cubicweb import CW_SOFTWARE_ROOT, Binary +from cubicweb import CW_SOFTWARE_ROOT, Binary, typed_eid from cubicweb.schema import (KNOWN_RPROPERTIES, CONSTRAINTS, ETYPE_NAME_MAP, VIRTUAL_RTYPES, PURE_VIRTUAL_RTYPES) from cubicweb.server import sqlutils @@ -212,6 +212,11 @@ rdefeid, seid, reid, oeid, card, ord, desc, idx, ftidx, i18n, default = values typeparams = extra_props.get(rdefeid) typeparams = json.load(typeparams) if typeparams else {} + if default is not None: + if isinstance(default, Binary): + # while migrating from 3.17 to 3.18, we still have to + # handle String defaults + default = default.unzpickle() _add_rdef(rdefeid, seid, reid, oeid, cardinality=card, description=desc, order=ord, indexed=idx, fulltextindexed=ftidx, internationalizable=i18n, @@ -527,10 +532,7 @@ elif isinstance(value, str): value = unicode(value) if value is not None and prop == 'default': - if value is False: - value = u'' - if not isinstance(value, unicode): - value = unicode(value) + value = Binary.zpickle(value) values[amap.get(prop, prop)] = value if extra: values['extra_props'] = Binary(json.dumps(extra)) diff -r 8a4175557426 -r c5eed908117d server/test/data/migratedapp/schema.py --- a/server/test/data/migratedapp/schema.py Thu Oct 17 11:34:03 2013 +0200 +++ b/server/test/data/migratedapp/schema.py Fri Aug 23 15:06:27 2013 +0200 @@ -16,6 +16,7 @@ # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . """cw.server.migraction test""" +import datetime as dt from yams.buildobjs import (EntityType, RelationType, RelationDefinition, SubjectRelation, Bytes, RichString, String, Int, Boolean, Datetime, Date) @@ -62,11 +63,14 @@ 'PE require_permission P, P name "add_note", ' 'P require_group G'),)} - whatever = Int(default=2) # keep it before `date` for unittest_migraction.test_add_attribute_int + whatever = Int(default=0) # keep it before `date` for unittest_migraction.test_add_attribute_int + yesno = Boolean(default=False) date = Datetime() type = String(maxsize=1) unique_id = String(maxsize=1, required=True, unique=True) mydate = Date(default='TODAY') + oldstyledefaultdate = Date(default='2013/01/01') + newstyledefaultdate = Date(default=dt.date(2013, 1, 1)) shortpara = String(maxsize=64, default='hop') ecrit_par = SubjectRelation('Personne', constraints=[RQLConstraint('S concerne A, O concerne A')]) attachment = SubjectRelation('File') diff -r 8a4175557426 -r c5eed908117d server/test/unittest_migractions.py --- a/server/test/unittest_migractions.py Thu Oct 17 11:34:03 2013 +0200 +++ b/server/test/unittest_migractions.py Fri Aug 23 15:06:27 2013 +0200 @@ -72,6 +72,22 @@ CubicWebTC.tearDown(self) self.repo.vreg['etypes'].clear_caches() + def test_add_attribute_bool(self): + self.assertFalse('yesno' in self.schema) + self.session.create_entity('Note') + self.commit() + self.mh.cmd_add_attribute('Note', 'yesno') + self.assertTrue('yesno' in self.schema) + self.assertEqual(self.schema['yesno'].subjects(), ('Note',)) + self.assertEqual(self.schema['yesno'].objects(), ('Boolean',)) + self.assertEqual(self.schema['Note'].default('yesno'), False) + # test default value set on existing entities + note = self.session.execute('Note X').get_entity(0, 0) + self.assertEqual(note.yesno, False) + # test default value set for next entities + self.assertEqual(self.session.create_entity('Note').yesno, False) + self.mh.rollback() + def test_add_attribute_int(self): self.assertFalse('whatever' in self.schema) self.session.create_entity('Note') @@ -82,12 +98,13 @@ self.assertTrue('whatever' in self.schema) self.assertEqual(self.schema['whatever'].subjects(), ('Note',)) self.assertEqual(self.schema['whatever'].objects(), ('Int',)) - self.assertEqual(self.schema['Note'].default('whatever'), 2) + self.assertEqual(self.schema['Note'].default('whatever'), 0) # test default value set on existing entities note = self.session.execute('Note X').get_entity(0, 0) - self.assertEqual(note.whatever, 2) + self.assertIsInstance(note.whatever, int) + self.assertEqual(note.whatever, 0) # test default value set for next entities - self.assertEqual(self.session.create_entity('Note').whatever, 2) + self.assertEqual(self.session.create_entity('Note').whatever, 0) # test attribute order orderdict2 = dict(self.mh.rqlexec('Any RTN, O WHERE X name "Note", RDEF from_entity X, ' 'RDEF relation_type RT, RDEF ordernum O, RT name RTN')) @@ -127,9 +144,14 @@ def test_add_datetime_with_default_value_attribute(self): self.assertFalse('mydate' in self.schema) - self.assertFalse('shortpara' in self.schema) + self.assertFalse('oldstyledefaultdate' in self.schema) + self.assertFalse('newstyledefaultdate' in self.schema) self.mh.cmd_add_attribute('Note', 'mydate') + self.mh.cmd_add_attribute('Note', 'oldstyledefaultdate') + self.mh.cmd_add_attribute('Note', 'newstyledefaultdate') self.assertTrue('mydate' in self.schema) + self.assertTrue('oldstyledefaultdate' in self.schema) + self.assertTrue('newstyledefaultdate' in self.schema) self.assertEqual(self.schema['mydate'].subjects(), ('Note', )) self.assertEqual(self.schema['mydate'].objects(), ('Date', )) testdate = date(2005, 12, 13) @@ -137,8 +159,13 @@ eid2 = self.mh.rqlexec('INSERT Note N: N mydate %(mydate)s', {'mydate' : testdate})[0][0] d1 = self.mh.rqlexec('Any D WHERE X eid %(x)s, X mydate D', {'x': eid1})[0][0] d2 = self.mh.rqlexec('Any D WHERE X eid %(x)s, X mydate D', {'x': eid2})[0][0] + d3 = self.mh.rqlexec('Any D WHERE X eid %(x)s, X oldstyledefaultdate D', {'x': eid1})[0][0] + d4 = self.mh.rqlexec('Any D WHERE X eid %(x)s, X newstyledefaultdate D', {'x': eid1})[0][0] self.assertEqual(d1, date.today()) self.assertEqual(d2, testdate) + myfavoritedate = date(2013, 1, 1) + self.assertEqual(d3, myfavoritedate) + self.assertEqual(d4, myfavoritedate) self.mh.rollback() def test_drop_chosen_constraints_ctxmanager(self): diff -r 8a4175557426 -r c5eed908117d server/test/unittest_schemaserial.py --- a/server/test/unittest_schemaserial.py Thu Oct 17 11:34:03 2013 +0200 +++ b/server/test/unittest_schemaserial.py Fri Aug 23 15:06:27 2013 +0200 @@ -22,6 +22,7 @@ from logilab.common.testlib import TestCase, unittest_main +from cubicweb import Binary from cubicweb.schema import CubicWebSchemaLoader from cubicweb.devtools import TestServerConfiguration @@ -188,7 +189,6 @@ self.assertIn('extra_props', got[1][1]) # this extr extra_props = got[1][1]['extra_props'] - from cubicweb import Binary self.assertIsInstance(extra_props, Binary) got[1][1]['extra_props'] = got[1][1]['extra_props'].getvalue() self.assertListEqual(expected, got) @@ -197,7 +197,8 @@ self.assertListEqual([ ('INSERT CWAttribute X: X cardinality %(cardinality)s,X defaultval %(defaultval)s,X description %(description)s,X fulltextindexed %(fulltextindexed)s,X indexed %(indexed)s,X internationalizable %(internationalizable)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE eid %(se)s,ER eid %(rt)s,OE eid %(oe)s', {'se': None, 'rt': None, 'oe': None, - 'description': u'', 'internationalizable': True, 'fulltextindexed': False, 'ordernum': 3, 'defaultval': u'text/plain', 'indexed': False, 'cardinality': u'?1'}), + 'description': u'', 'internationalizable': True, 'fulltextindexed': False, + 'ordernum': 3, 'defaultval': Binary('text/plain'), 'indexed': False, 'cardinality': u'?1'}), ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT eid %(ct)s, EDEF eid %(x)s', {'x': None, 'value': u'None', 'ct': 'FormatConstraint_eid'}), ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT eid %(ct)s, EDEF eid %(x)s', diff -r 8a4175557426 -r c5eed908117d web/views/schema.py --- a/web/views/schema.py Thu Oct 17 11:34:03 2013 +0200 +++ b/web/views/schema.py Fri Aug 23 15:06:27 2013 +0200 @@ -225,6 +225,7 @@ {'x': entity.eid}) self.wview('table', rset, 'null', cellvids={0: 'rdef-name-cell', + 2: 'etype-attr-defaultval-cell', 3: 'etype-attr-cardinality-cell', 4: 'rdef-constraints-cell', 6: 'rdef-options-cell'}, @@ -271,6 +272,14 @@ self.w(self._cw._(u'no')) +class CWETypeAttributeDefaultValCell(baseviews.FinalView): + __regid__ = 'etype-attr-defaultval-cell' + + def cell_call(self, row, col): + defaultval = self.cw_rset.rows[row][col] + if defaultval is not None: + self.w(unicode(self.cw_rset.rows[row][col].unzpickle())) + class CWETypeRelationCardinalityCell(baseviews.FinalView): __regid__ = 'etype-rel-cardinality-cell'