# HG changeset patch # User Sylvain Thénault # Date 1311690832 -7200 # Node ID 1541d9e6b24236beb6e0c495c3a50fbaf880537d # Parent 8d9c732ad30ed85447a9c0f8afc90fd91bcb38b7# Parent 2f6e37661cf62f9205c7ece9947de6bd6eb6e45b backport stable diff -r 8d9c732ad30e -r 1541d9e6b242 MANIFEST.in --- a/MANIFEST.in Wed Jul 20 17:49:37 2011 +0200 +++ b/MANIFEST.in Tue Jul 26 16:33:52 2011 +0200 @@ -5,7 +5,7 @@ include bin/cubicweb-* include man/cubicweb-ctl.1 -recursive-include doc README makefile *.conf *.css *.py *.rst *.txt *.html *.png *.svg *.zargo *.dia +recursive-include doc README makefile *.conf *.js *.css *.py *.rst *.txt *.html *.png *.svg *.zargo *.dia recursive-include misc *.py *.png *.display @@ -32,5 +32,6 @@ prune doc/book/en/.static prune doc/book/fr/.static +prune doc/html/_sources/ prune misc/cwfs prune goa diff -r 8d9c732ad30e -r 1541d9e6b242 cwconfig.py --- a/cwconfig.py Wed Jul 20 17:49:37 2011 +0200 +++ b/cwconfig.py Tue Jul 26 16:33:52 2011 +0200 @@ -1235,6 +1235,7 @@ class LIMIT_SIZE(FunctionDescr): supported_backends = ('postgres', 'sqlite',) + minargs = maxargs = 3 rtype = 'String' def st_description(self, funcnode, mainindex, tr): @@ -1245,6 +1246,7 @@ class TEXT_LIMIT_SIZE(LIMIT_SIZE): supported_backends = ('mysql', 'postgres', 'sqlite',) + minargs = maxargs = 2 register_function(TEXT_LIMIT_SIZE) diff -r 8d9c732ad30e -r 1541d9e6b242 hooks/metadata.py --- a/hooks/metadata.py Wed Jul 20 17:49:37 2011 +0200 +++ b/hooks/metadata.py Tue Jul 26 16:33:52 2011 +0200 @@ -196,6 +196,16 @@ # copy entity if necessary if not oldsource.repo_source.copy_based_source: entity.complete(skip_bytes=False) + if not entity.creation_date: + entity.cw_attr_cache['creation_date'] = datetime.now() + if not entity.modification_date: + entity.cw_attr_cache['modification_date'] = datetime.now() + entity.cw_attr_cache['cwuri'] = u'%s%s' % (self._cw.base_url(), entity.eid) + for rschema, attrschema in entity.e_schema.attribute_definitions(): + if attrschema == 'Password' and \ + rschema.rdef(entity.e_schema, attrschema).cardinality[0] == '1': + from logilab.common.shellutils import generate_password + entity.cw_attr_cache[rschema.type] = generate_password() entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache) syssource.add_entity(self._cw, entity) # we don't want the moved entity to be reimported later. To @@ -205,9 +215,8 @@ # source='system'. External source will then have consider case # where `extid2eid` return a negative eid as 'this entity was known # but has been moved, ignore it'. - self._cw.system_sql('UPDATE entities SET eid=-eid,source=%(source)s ' - 'WHERE eid=%(eid)s', - {'eid': self.eidfrom, 'source': newsource.name}) + self._cw.system_sql('UPDATE entities SET eid=-eid WHERE eid=%(eid)s', + {'eid': self.eidfrom}) attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': None, 'source': 'system', 'asource': 'system', 'mtime': datetime.now()} diff -r 8d9c732ad30e -r 1541d9e6b242 misc/migration/3.13.3_Any.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/misc/migration/3.13.3_Any.py Tue Jul 26 16:33:52 2011 +0200 @@ -0,0 +1,2 @@ +drop_relation_definition('CWSourceSchemaConfig', 'cw_schema', 'CWAttribute') +sync_schema_props_perms('cw_schema') diff -r 8d9c732ad30e -r 1541d9e6b242 schemas/base.py --- a/schemas/base.py Wed Jul 20 17:49:37 2011 +0200 +++ b/schemas/base.py Tue Jul 26 16:33:52 2011 +0200 @@ -323,13 +323,28 @@ cw_for_source = SubjectRelation( 'CWSource', inlined=True, cardinality='1*', composite='object', __permissions__=RELATION_MANAGERS_PERMISSIONS) - cw_schema = SubjectRelation( - ('CWEType', 'CWRType', 'CWAttribute', 'CWRelation'), - inlined=True, cardinality='1*', composite='object', - __permissions__=RELATION_MANAGERS_PERMISSIONS) options = String(description=_('allowed options depends on the source type')) +class rtype_cw_schema(RelationDefinition): + __permissions__ = RELATION_MANAGERS_PERMISSIONS + name = 'cw_schema' + subject = 'CWSourceSchemaConfig' + object = ('CWEType', 'CWRType') + inlined = True + cardinality = '1*' + composite = 'object' + constraints = [RQLConstraint('NOT O final TRUE')] + +class rdef_cw_schema(RelationDefinition): + __permissions__ = RELATION_MANAGERS_PERMISSIONS + name = 'cw_schema' + subject = 'CWSourceSchemaConfig' + object = 'CWRelation' + inlined = True + cardinality = '1*' + composite = 'object' + # "abtract" relation types, no definition in cubicweb itself ################### class identical_to(RelationType): diff -r 8d9c732ad30e -r 1541d9e6b242 server/checkintegrity.py --- a/server/checkintegrity.py Wed Jul 20 17:49:37 2011 +0200 +++ b/server/checkintegrity.py Tue Jul 26 16:33:52 2011 +0200 @@ -188,6 +188,18 @@ if fix: session.system_sql('DELETE FROM entities WHERE eid=%s;' % eid) notify_fixed(fix) + session.system_sql('INSERT INTO cw_source_relation (eid_from, eid_to) ' + 'SELECT e.eid, s.cw_eid FROM entities as e, cw_CWSource as s ' + 'WHERE s.cw_name=e.asource AND NOT EXISTS(SELECT 1 FROM cw_source_relation as cs ' + ' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid)') + session.system_sql('INSERT INTO is_relation (eid_from, eid_to) ' + 'SELECT e.eid, s.cw_eid FROM entities as e, cw_CWEType as s ' + 'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_relation as cs ' + ' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid)') + session.system_sql('INSERT INTO is_instance_of_relation (eid_from, eid_to) ' + 'SELECT e.eid, s.cw_eid FROM entities as e, cw_CWEType as s ' + 'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_instance_of_relation as cs ' + ' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid)') print 'Checking entities tables' for eschema in schema.entities(): if eschema.final: @@ -283,10 +295,10 @@ rql = 'Any X WHERE NOT Y %s X, X is %s' % (rschema, etype) for entity in session.execute(rql).entities(): print >> sys.stderr, '%s #%s is missing mandatory %s relation %s' % ( - entity.__regid__, entity.eid, role, rschema) + entity.__regid__, entity.eid, role, rschema), if fix: #if entity.cw_describe()['source']['uri'] == 'system': XXX - entity.delete() + entity.cw_delete() notify_fixed(fix) @@ -304,9 +316,9 @@ rschema, rdef.subject) for entity in session.execute(rql).entities(): print >> sys.stderr, '%s #%s is missing mandatory attribute %s' % ( - entity.__regid__, entity.eid, rschema) + entity.__regid__, entity.eid, rschema), if fix: - entity.delete() + entity.cw_delete() notify_fixed(fix) @@ -333,22 +345,6 @@ % (table, column, eidcolumn, eid), {'v': default}) notify_fixed(fix) - cursor = session.system_sql('SELECT MIN(%s) FROM %sCWUser;' % (eidcolumn, - SQL_PREFIX)) - default_user_eid = cursor.fetchone()[0] - assert default_user_eid is not None, 'no user defined !' - for rel, default in ( ('owned_by', default_user_eid), ): - cursor = session.system_sql("SELECT eid, type FROM entities " - "WHERE source='system' AND NOT EXISTS " - "(SELECT 1 FROM %s_relation WHERE eid_from=eid);" - % rel) - for eid, etype in cursor.fetchall(): - msg = ' %s with eid %s has no %s relation' - print >> sys.stderr, msg % (etype, eid, rel), - if fix: - session.system_sql('INSERT INTO %s_relation VALUES (%s, %s) ;' - % (rel, eid, default)) - notify_fixed(fix) def check(repo, cnx, checks, reindex, fix, withpb=True): @@ -360,7 +356,7 @@ # yo, launch checks if checks: eids_cache = {} - with security_enabled(session, read=False): # ensure no read security + with security_enabled(session, read=False, write=False): # ensure no read security for check in checks: check_func = globals()['check_%s' % check] check_func(repo.schema, session, eids_cache, fix=fix) diff -r 8d9c732ad30e -r 1541d9e6b242 server/sources/storages.py --- a/server/sources/storages.py Wed Jul 20 17:49:37 2011 +0200 +++ b/server/sources/storages.py Tue Jul 26 16:33:52 2011 +0200 @@ -148,6 +148,7 @@ # We do not need to create it but we need to fetch the content of # the file as the actual content of the attribute fpath = entity.cw_edited[attr].getvalue() + assert fpath is not None binary = Binary(file(fpath, 'rb').read()) else: # We must store the content of the attributes @@ -160,17 +161,23 @@ # # fetch the current attribute value in memory binary = entity.cw_edited.pop(attr) - # Get filename for it - fpath = self.new_fs_path(entity, attr) - assert not osp.exists(fpath) - # write attribute value on disk - file(fpath, 'wb').write(binary.getvalue()) - # Mark the new file as added during the transaction. - # The file will be removed on rollback - AddFileOp.get_instance(entity._cw).add_data(fpath) + if binary is None: + fpath = None + else: + # Get filename for it + fpath = self.new_fs_path(entity, attr) + assert not osp.exists(fpath) + # write attribute value on disk + file(fpath, 'wb').write(binary.getvalue()) + # Mark the new file as added during the transaction. + # The file will be removed on rollback + AddFileOp.get_instance(entity._cw).add_data(fpath) if oldpath != fpath: # register the new location for the file. - entity.cw_edited.edited_attribute(attr, Binary(fpath)) + if fpath is None: + entity.cw_edited.edited_attribute(attr, None) + else: + entity.cw_edited.edited_attribute(attr, Binary(fpath)) # Mark the old file as useless so the file will be removed at # commit. if oldpath is not None: diff -r 8d9c732ad30e -r 1541d9e6b242 server/test/data/schema.py --- a/server/test/data/schema.py Wed Jul 20 17:49:37 2011 +0200 +++ b/server/test/data/schema.py Tue Jul 26 16:33:52 2011 +0200 @@ -18,12 +18,15 @@ from yams.buildobjs import (EntityType, RelationType, RelationDefinition, SubjectRelation, RichString, String, Int, Float, - Boolean, Datetime, TZDatetime) + Boolean, Datetime, TZDatetime, Bytes) from yams.constraints import SizeConstraint from cubicweb.schema import (WorkflowableEntityType, RQLConstraint, RQLUniqueConstraint, ERQLExpression, RRQLExpression) +class BFSSTestable(EntityType): + opt_attr = Bytes() + class Affaire(WorkflowableEntityType): __permissions__ = { 'read': ('managers', diff -r 8d9c732ad30e -r 1541d9e6b242 server/test/unittest_ldapuser.py --- a/server/test/unittest_ldapuser.py Wed Jul 20 17:49:37 2011 +0200 +++ b/server/test/unittest_ldapuser.py Tue Jul 26 16:33:52 2011 +0200 @@ -379,9 +379,11 @@ self.assertEqual(rset.rows, [[None]]) def test_copy_to_system_source(self): + source = self.repo.sources_by_uri['ldapuser'] eid = self.sexecute('CWUser X WHERE X login %(login)s', {'login': SYT})[0][0] self.sexecute('SET X cw_source S WHERE X eid %(x)s, S name "system"', {'x': eid}) self.commit() + source.reset_caches() rset = self.sexecute('CWUser X WHERE X login %(login)s', {'login': SYT}) self.assertEqual(len(rset), 1) e = rset.get_entity(0, 0) @@ -390,7 +392,9 @@ 'type': 'CWUser', 'extid': None}) self.assertEqual(e.cw_source[0].name, 'system') - source = self.repo.sources_by_uri['ldapuser'] + self.failUnless(e.creation_date) + self.failUnless(e.modification_date) + # XXX test some password has been set source.synchronize() rset = self.sexecute('CWUser X WHERE X login %(login)s', {'login': SYT}) self.assertEqual(len(rset), 1) diff -r 8d9c732ad30e -r 1541d9e6b242 server/test/unittest_storage.py --- a/server/test/unittest_storage.py Wed Jul 20 17:49:37 2011 +0200 +++ b/server/test/unittest_storage.py Tue Jul 26 16:33:52 2011 +0200 @@ -58,6 +58,7 @@ self.tempdir = tempfile.mkdtemp() bfs_storage = storages.BytesFileSystemStorage(self.tempdir) storages.set_attribute_storage(self.repo, 'File', 'data', bfs_storage) + storages.set_attribute_storage(self.repo, 'BFSSTestable', 'opt_attr', bfs_storage) def tearDown(self): super(StorageTC, self).tearDown() @@ -256,6 +257,14 @@ self.assertEqual(old_path, new_path) self.assertEqual(old_data, new_data) + @tag('update', 'NULL') + def test_bfss_update_to_None(self): + f = self.session.create_entity('BFSSTestable', opt_attr=Binary('toto')) + self.session.commit() + self.session.set_pool() + f.set_attributes(opt_attr=None) + self.session.commit() + @tag('fs_importing', 'update') def test_bfss_update_with_fs_importing(self): # use self.session to use server-side cache diff -r 8d9c732ad30e -r 1541d9e6b242 skeleton/test/pytestconf.py --- a/skeleton/test/pytestconf.py Wed Jul 20 17:49:37 2011 +0200 +++ b/skeleton/test/pytestconf.py Tue Jul 26 16:33:52 2011 +0200 @@ -19,7 +19,7 @@ """ import os -import pwd +import sys from logilab.common.pytest import PyTester @@ -28,6 +28,9 @@ (man 3 getlogin) Another solution would be to use $LOGNAME, $USER or $USERNAME """ + if sys.platform == 'win32': + return os.environ.get('USERNAME') or 'cubicweb' + import pwd return pwd.getpwuid(os.getuid())[0] diff -r 8d9c732ad30e -r 1541d9e6b242 sobjects/parsers.py --- a/sobjects/parsers.py Wed Jul 20 17:49:37 2011 +0200 +++ b/sobjects/parsers.py Tue Jul 26 16:33:52 2011 +0200 @@ -178,8 +178,10 @@ def process(self, url, raise_on_error=False, partialcommit=True): """IDataFeedParser main entry point""" - super(CWEntityXMLParser, self).process(self.complete_url(url), - raise_on_error, partialcommit) + if url.startswith('http'): # XXX similar loose test as in parse of sources.datafeed + url = self.complete_url(url) + super(CWEntityXMLParser, self).process(url, raise_on_error, partialcommit) + def parse_etree(self, parent): for node in list(parent): builder = self._cw.vreg['components'].select( @@ -226,24 +228,25 @@ be included in the resulting xml, according to source mapping. If etype is not specified, try to guess it using the last path part of - the url. + the url, i.e. the format used by default in cubicweb to map all entities + of a given type as in 'http://mysite.org/EntityType'. """ try: url, qs = url.split('?', 1) except ValueError: qs = '' + params = parse_qs(qs) + if not 'vid' in params: + params['vid'] = ['xml'] if etype is None: try: etype = url.rsplit('/', 1)[1] except ValueError: - return url + return url + '?' + self._cw.build_url_params(**params) try: etype = self._cw.vreg.case_insensitive_etypes[etype] except KeyError: - return url - params = parse_qs(qs) - if not 'vid' in params: - params['vid'] = ['xml'] + return url + '?' + self._cw.build_url_params(**params) if add_relations: relations = params.setdefault('relation', []) for rtype, role, _ in self.source.mapping.get(etype, ()): diff -r 8d9c732ad30e -r 1541d9e6b242 sobjects/test/unittest_parsers.py --- a/sobjects/test/unittest_parsers.py Wed Jul 20 17:49:37 2011 +0200 +++ b/sobjects/test/unittest_parsers.py Tue Jul 26 16:33:52 2011 +0200 @@ -144,6 +144,10 @@ 'http://www.cubicweb.org/cwuser?relation=tags-object&relation=in_group-subject&relation=in_state-subject&relation=use_email-subject&vid=xml') self.assertEqual(parser.complete_url('http://www.cubicweb.org/cwuser?vid=rdf&relation=hop'), 'http://www.cubicweb.org/cwuser?relation=hop&relation=tags-object&relation=in_group-subject&relation=in_state-subject&relation=use_email-subject&vid=rdf') + self.assertEqual(parser.complete_url('http://www.cubicweb.org/?rql=cwuser&vid=rdf&relation=hop'), + 'http://www.cubicweb.org/?rql=cwuser&relation=hop&vid=rdf') + self.assertEqual(parser.complete_url('http://www.cubicweb.org/?rql=cwuser&relation=hop'), + 'http://www.cubicweb.org/?rql=cwuser&relation=hop&vid=xml') def test_actions(self): diff -r 8d9c732ad30e -r 1541d9e6b242 web/data/cubicweb.facets.js --- a/web/data/cubicweb.facets.js Wed Jul 20 17:49:37 2011 +0200 +++ b/web/data/cubicweb.facets.js Tue Jul 26 16:33:52 2011 +0200 @@ -23,15 +23,24 @@ var values = []; $form.find('.facet').each(function() { var facetName = jQuery(this).find('.facetTitle').attr('cubicweb:facetName'); - var facetValues = jQuery(this).find('.facetValueSelected').each(function(x) { + // FacetVocabularyWidget + jQuery(this).find('.facetValueSelected').each(function(x) { names.push(facetName); values.push(this.getAttribute('cubicweb:value')); }); + // FacetStringWidget (e.g. has-text) + jQuery(this).find('input:text').each(function(){ + names.push(facetName); + values.push(this.value); + }); }); - $form.find('input').each(function() { + // pick up hidden inputs (required metadata inputs such as 'facets' + // but also RangeWidgets) + $form.find('input:hidden').each(function() { names.push(this.name); values.push(this.value); }); + // And / Or operators $form.find('select option[selected]').each(function() { names.push(this.parentNode.name); values.push(this.value); @@ -94,7 +103,7 @@ }, 'ctxcomponents', 'edit_box')); } - $node = jQuery('#breadcrumbs') + $node = jQuery('#breadcrumbs'); if ($node.length) { $node.loadxhtml('json', ajaxFuncArgs('render', { 'rql': rql diff -r 8d9c732ad30e -r 1541d9e6b242 web/facet.py --- a/web/facet.py Wed Jul 20 17:49:37 2011 +0200 +++ b/web/facet.py Tue Jul 26 16:33:52 2011 +0200 @@ -48,6 +48,7 @@ __docformat__ = "restructuredtext en" _ = unicode +from warnings import warn from copy import deepcopy from datetime import date, datetime, timedelta @@ -745,7 +746,7 @@ self._add_not_rel_restr(rel) self._and_restriction(rel, restrvar, value.pop()) while value: - restrvar, rtrel = _make_relation(self.select, filtered_variable, + restrvar, rtrel = _make_relation(self.select, self.filtered_variable, self.rtype, self.role) self._and_restriction(rel, restrvar, value.pop()) diff -r 8d9c732ad30e -r 1541d9e6b242 web/views/boxes.py --- a/web/views/boxes.py Wed Jul 20 17:49:37 2011 +0200 +++ b/web/views/boxes.py Tue Jul 26 16:33:52 2011 +0200 @@ -185,7 +185,7 @@ def render_body(self, w): for category, views in box.sort_by_category(self.views): - menu = htmlwidgets.BoxMenu(self._cw._(category)) + menu = htmlwidgets.BoxMenu(self._cw._(category), ident=category) for view in views: menu.append(self.action_link(view)) self.append(menu) diff -r 8d9c732ad30e -r 1541d9e6b242 web/views/startup.py --- a/web/views/startup.py Wed Jul 20 17:49:37 2011 +0200 +++ b/web/views/startup.py Tue Jul 26 16:33:52 2011 +0200 @@ -127,16 +127,16 @@ for etype in self.add_etype_links: eschema = self._cw.vreg.schema.eschema(etype) if eschema.has_perm(self._cw, 'add'): + url = self._cw.vreg["etypes"].etype_class(etype).cw_create_url(self._cw) self.w(u'
  • %s
    • ' % ( - self._cw.build_url('add/%s' % eschema), - self._cw.__('add a %s' % eschema).capitalize())) + url, self._cw.__('New %s' % eschema).capitalize())) self.w(u'') def add_entity_link(self, etype): """creates a [+] link for adding an entity""" url = self._cw.vreg["etypes"].etype_class(etype).cw_create_url(self._cw) return u'[+]' % ( - xml_escape(url), self._cw.__('add a %s' % etype)) + xml_escape(url), self._cw.__('New %s' % etype)) @deprecated('[3.11] display_folders method is deprecated, backport it if needed') def display_folders(self):