# HG changeset patch # User Sylvain Thénault # Date 1269523573 -3600 # Node ID b3b0b808a0edc77b1b943696514daefdadb27a3a # Parent 4cc020ee70e274910afd6dda85fea1084efe357c# Parent 55e2602545cd88affa13a16daaed3485480bb3b1 backport stable diff -r 4cc020ee70e2 -r b3b0b808a0ed cwconfig.py --- a/cwconfig.py Wed Mar 24 18:04:59 2010 +0100 +++ b/cwconfig.py Thu Mar 25 14:26:13 2010 +0100 @@ -90,7 +90,8 @@ from logilab.common.configuration import (Configuration, Method, ConfigurationMixIn, merge_options) -from cubicweb import CW_SOFTWARE_ROOT, CW_MIGRATION_MAP, ConfigurationError +from cubicweb import (CW_SOFTWARE_ROOT, CW_MIGRATION_MAP, + ConfigurationError, Binary) from cubicweb.toolsutils import env_path, create_dir CONFIGURATIONS = [] @@ -1050,7 +1051,24 @@ class FSPATH(FunctionDescr): - supported_backends = ('postgres', 'sqlite',) - rtype = 'Bytes' + """return path of some bytes attribute stored using the Bytes + File-System Storage (bfss) + """ + rtype = 'Bytes' # XXX return a String? potential pb with fs encoding + + def update_cb_stack(self, stack): + assert len(stack) == 1 + stack[0] = self.source_execute + + def as_sql(self, backend, args): + raise NotImplementedError('source only callback') + + def source_execute(self, source, value): + fpath = source.binary_to_str(value) + try: + return Binary(fpath) + except OSError, ex: + self.critical("can't open %s: %s", fpath, ex) + return None register_function(FSPATH) diff -r 4cc020ee70e2 -r b3b0b808a0ed entity.py --- a/entity.py Wed Mar 24 18:04:59 2010 +0100 +++ b/entity.py Thu Mar 25 14:26:13 2010 +0100 @@ -286,6 +286,18 @@ self.edited_attributes.add(attr) self.skip_security_attributes.add(attr) + def pop(self, attr, default=_marker): + """override pop to update self.edited_attributes on cleanup of + undesired changes introduced in the entity's dict. See `__delitem__` + """ + if default is _marker: + value = super(Entity, self).pop(attr) + else: + value = super(Entity, self).pop(attr, default) + if hasattr(self, 'edited_attributes') and attr in self.edited_attributes: + self.edited_attributes.remove(attr) + return value + def rql_set_value(self, attr, value): """call by rql execution plan when some attribute is modified diff -r 4cc020ee70e2 -r b3b0b808a0ed hooks/integrity.py --- a/hooks/integrity.py Wed Mar 24 18:04:59 2010 +0100 +++ b/hooks/integrity.py Thu Mar 25 14:26:13 2010 +0100 @@ -228,45 +228,6 @@ raise ValidationError(entity.eid, {attr: msg % val}) -class _DelayedDeleteOp(hook.Operation): - """delete the object of composite relation except if the relation - has actually been redirected to another composite - """ - - def precommit_event(self): - session = self.session - # don't do anything if the entity is being created or deleted - if not (session.deleted_in_transaction(self.eid) or - session.added_in_transaction(self.eid)): - etype = session.describe(self.eid)[0] - session.execute('DELETE %s X WHERE X eid %%(x)s, NOT %s' - % (etype, self.relation), - {'x': self.eid}, 'x') - - -class DeleteCompositeOrphanHook(IntegrityHook): - """delete the composed of a composite relation when this relation is deleted - """ - __regid__ = 'deletecomposite' - events = ('before_delete_relation',) - - def __call__(self): - # if the relation is being delete, don't delete composite's components - # automatically - pendingrdefs = self._cw.transaction_data.get('pendingrdefs', ()) - if (self._cw.describe(self.eidfrom)[0], self.rtype, - self._cw.describe(self.eidto)[0]) in pendingrdefs: - return - composite = self._cw.schema_rproperty(self.rtype, self.eidfrom, self.eidto, - 'composite') - if composite == 'subject': - _DelayedDeleteOp(self._cw, eid=self.eidto, - relation='Y %s X' % self.rtype) - elif composite == 'object': - _DelayedDeleteOp(self._cw, eid=self.eidfrom, - relation='X %s Y' % self.rtype) - - class DontRemoveOwnersGroupHook(IntegrityHook): """delete the composed of a composite relation when this relation is deleted """ @@ -314,3 +275,46 @@ user = self.entity if 'login' in user.edited_attributes and user.login: user.login = user.login.strip() + + +# 'active' integrity hooks: you usually don't want to deactivate them, they are +# not really integrity check, they maintain consistency on changes + +class _DelayedDeleteOp(hook.Operation): + """delete the object of composite relation except if the relation + has actually been redirected to another composite + """ + + def precommit_event(self): + session = self.session + # don't do anything if the entity is being created or deleted + if not (session.deleted_in_transaction(self.eid) or + session.added_in_transaction(self.eid)): + etype = session.describe(self.eid)[0] + session.execute('DELETE %s X WHERE X eid %%(x)s, NOT %s' + % (etype, self.relation), + {'x': self.eid}, 'x') + + +class DeleteCompositeOrphanHook(hook.Hook): + """delete the composed of a composite relation when this relation is deleted + """ + __regid__ = 'deletecomposite' + events = ('before_delete_relation',) + category = 'activeintegrity' + + def __call__(self): + # if the relation is being delete, don't delete composite's components + # automatically + pendingrdefs = self._cw.transaction_data.get('pendingrdefs', ()) + if (self._cw.describe(self.eidfrom)[0], self.rtype, + self._cw.describe(self.eidto)[0]) in pendingrdefs: + return + composite = self._cw.schema_rproperty(self.rtype, self.eidfrom, self.eidto, + 'composite') + if composite == 'subject': + _DelayedDeleteOp(self._cw, eid=self.eidto, + relation='Y %s X' % self.rtype) + elif composite == 'object': + _DelayedDeleteOp(self._cw, eid=self.eidfrom, + relation='X %s Y' % self.rtype) diff -r 4cc020ee70e2 -r b3b0b808a0ed misc/migration/3.7.2_Any.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/misc/migration/3.7.2_Any.py Thu Mar 25 14:26:13 2010 +0100 @@ -0,0 +1,2 @@ +sql('DROP FUNCTION IF EXISTS _fsopen(bytea)') +sql('DROP FUNCTION IF EXISTS fspath(bigint, text, text)') diff -r 4cc020ee70e2 -r b3b0b808a0ed schemas/_regproc_bss.postgres.sql --- a/schemas/_regproc_bss.postgres.sql Wed Mar 24 18:04:59 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,38 +0,0 @@ -/* -*- sql -*- - - postgres specific registered procedures for the Bytes File System storage, - require the plpythonu language installed - -*/ - - -CREATE OR REPLACE FUNCTION _fsopen(bytea) RETURNS bytea AS $$ - fpath = args[0] - if fpath: - try: - data = file(fpath, 'rb').read() - #/* XXX due to plpython bug we have to replace some characters... */ - return data.replace("\\", r"\134").replace("\000", r"\000").replace("'", r"\047") #' - except Exception, ex: - plpy.warning('failed to get content for %s: %s', fpath, ex) - return None -$$ LANGUAGE plpythonu -/* WITH(ISCACHABLE) XXX does postgres handle caching of large data nicely */ -;; - -/* fspath(eid, entity type, attribute) */ -CREATE OR REPLACE FUNCTION fspath(bigint, text, text) RETURNS bytea AS $$ - pkey = 'plan%s%s' % (args[1], args[2]) - try: - plan = SD[pkey] - except KeyError: - #/* then prepare and cache plan to get versioned file information from a - # version content eid */ - plan = plpy.prepare( - 'SELECT X.cw_%s FROM cw_%s as X WHERE X.cw_eid=$1' % (args[2], args[1]), - ['bigint']) - SD[pkey] = plan - return plpy.execute(plan, [args[0]])[0]['cw_' + args[2]] -$$ LANGUAGE plpythonu -/* WITH(ISCACHABLE) XXX does postgres handle caching of large data nicely */ -;; diff -r 4cc020ee70e2 -r b3b0b808a0ed server/session.py --- a/server/session.py Wed Mar 24 18:04:59 2010 +0100 +++ b/server/session.py Thu Mar 25 14:26:13 2010 +0100 @@ -591,10 +591,7 @@ ecache[entity.eid] = entity def entity_cache(self, eid): - try: - return self.transaction_data['ecache'][eid] - except: - raise + return self.transaction_data['ecache'][eid] def cached_entities(self): return self.transaction_data.get('ecache', {}).values() diff -r 4cc020ee70e2 -r b3b0b808a0ed server/sources/extlite.py --- a/server/sources/extlite.py Wed Mar 24 18:04:59 2010 +0100 +++ b/server/sources/extlite.py Thu Mar 25 14:26:13 2010 +0100 @@ -187,9 +187,10 @@ if self._need_sql_create: return [] assert dbg_st_search(self.uri, union, varmap, args, cachekey) - sql, query_args = self.rqlsqlgen.generate(union, args) - args = self.sqladapter.merge_args(args, query_args) - results = self.sqladapter.process_result(self.doexec(session, sql, args)) + sql, qargs, cbs = self.rqlsqlgen.generate(union, args) + args = self.sqladapter.merge_args(args, qargs) + cursor = self.doexec(session, sql, args) + results = self.sqladapter.process_result(cursor, cbs) assert dbg_results(results) return results diff -r 4cc020ee70e2 -r b3b0b808a0ed server/sources/native.py --- a/server/sources/native.py Wed Mar 24 18:04:59 2010 +0100 +++ b/server/sources/native.py Thu Mar 25 14:26:13 2010 +0100 @@ -264,8 +264,9 @@ def init(self): self.init_creating() - def map_attribute(self, etype, attr, cb): - self._rql_sqlgen.attr_map['%s.%s' % (etype, attr)] = cb + # XXX deprecates [un]map_attribute ? + def map_attribute(self, etype, attr, cb, sourcedb=True): + self._rql_sqlgen.attr_map['%s.%s' % (etype, attr)] = (cb, sourcedb) def unmap_attribute(self, etype, attr): self._rql_sqlgen.attr_map.pop('%s.%s' % (etype, attr), None) @@ -273,7 +274,8 @@ def set_storage(self, etype, attr, storage): storage_dict = self._storages.setdefault(etype, {}) storage_dict[attr] = storage - self.map_attribute(etype, attr, storage.sqlgen_callback) + self.map_attribute(etype, attr, + storage.callback, storage.is_source_callback) def unset_storage(self, etype, attr): self._storages[etype].pop(attr) @@ -348,17 +350,17 @@ if cachekey is None: self.no_cache += 1 # generate sql query if we are able to do so (not supported types...) - sql, query_args = self._rql_sqlgen.generate(union, args, varmap) + sql, qargs, cbs = self._rql_sqlgen.generate(union, args, varmap) else: # sql may be cached try: - sql, query_args = self._cache[cachekey] + sql, qargs, cbs = self._cache[cachekey] self.cache_hit += 1 except KeyError: self.cache_miss += 1 - sql, query_args = self._rql_sqlgen.generate(union, args, varmap) - self._cache[cachekey] = sql, query_args - args = self.merge_args(args, query_args) + sql, qargs, cbs = self._rql_sqlgen.generate(union, args, varmap) + self._cache[cachekey] = sql, qargs, cbs + args = self.merge_args(args, qargs) assert isinstance(sql, basestring), repr(sql) try: cursor = self.doexec(session, sql, args) @@ -367,7 +369,7 @@ self.info("request failed '%s' ... retry with a new cursor", sql) session.pool.reconnect(self) cursor = self.doexec(session, sql, args) - results = self.process_result(cursor) + results = self.process_result(cursor, cbs) assert dbg_results(results) return results @@ -381,9 +383,9 @@ self.uri, union, varmap, args, prefix='ON THE FLY temp data insertion into %s from' % table) # generate sql queries if we are able to do so - sql, query_args = self._rql_sqlgen.generate(union, args, varmap) + sql, qargs, cbs = self._rql_sqlgen.generate(union, args, varmap) query = 'INSERT INTO %s %s' % (table, sql.encode(self._dbencoding)) - self.doexec(session, query, self.merge_args(args, query_args)) + self.doexec(session, query, self.merge_args(args, qargs)) def manual_insert(self, results, table, session): """insert given result into a temporary table on the system source""" @@ -428,10 +430,14 @@ orig_values = {} etype = entity.__regid__ for attr, storage in self._storages.get(etype, {}).items(): - if attr in entity.edited_attributes: - orig_values[attr] = entity[attr] - handler = getattr(storage, 'entity_%s' % event) - handler(entity, attr) + try: + if attr in entity.edited_attributes: + orig_values[attr] = entity[attr] + handler = getattr(storage, 'entity_%s' % event) + handler(entity, attr) + except AttributeError: + assert event == 'deleted' + getattr(storage, 'entity_deleted')(entity, attr) yield # 2/ execute the source's instructions # 3/ restore original values for attr, value in orig_values.items(): diff -r 4cc020ee70e2 -r b3b0b808a0ed server/sources/rql2sql.py --- a/server/sources/rql2sql.py Wed Mar 24 18:04:59 2010 +0100 +++ b/server/sources/rql2sql.py Thu Mar 25 14:26:13 2010 +0100 @@ -33,16 +33,30 @@ import threading +from logilab.database import FunctionDescr, SQL_FUNCTIONS_REGISTRY + from rql import BadRQLQuery, CoercionError from rql.stmts import Union, Select from rql.nodes import (SortTerm, VariableRef, Constant, Function, Not, Variable, ColumnAlias, Relation, SubQuery, Exists) +from cubicweb import QueryError from cubicweb.server.sqlutils import SQL_PREFIX from cubicweb.server.utils import cleanup_solutions ColumnAlias._q_invariant = False # avoid to check for ColumnAlias / Variable +FunctionDescr.source_execute = None + +def default_update_cb_stack(self, stack): + stack.append(self.source_execute) +FunctionDescr.update_cb_stack = default_update_cb_stack + +LENGTH = SQL_FUNCTIONS_REGISTRY.get_function('LENGTH') +def length_source_execute(source, value): + return len(value.getvalue()) +LENGTH.source_execute = length_source_execute + def _new_var(select, varname): newvar = select.get_variable(varname) if not 'relations' in newvar.stinfo: @@ -252,14 +266,44 @@ selectedidx.append(vref.name) rqlst.selection.append(vref) -# IGenerator implementation for RQL->SQL ###################################### +def iter_mapped_var_sels(stmt, variable): + # variable is a Variable or ColumnAlias node mapped to a source side + # callback + if not (len(variable.stinfo['rhsrelations']) <= 1 and # < 1 on column alias + variable.stinfo['selected']): + raise QueryError("can't use %s as a restriction variable" + % variable.name) + for selectidx in variable.stinfo['selected']: + vrefs = stmt.selection[selectidx].get_nodes(VariableRef) + if len(vrefs) != 1: + raise QueryError() + yield selectidx, vrefs[0] +def update_source_cb_stack(state, stmt, node, stack): + while True: + node = node.parent + if node is stmt: + break + if not isinstance(node, Function): + raise QueryError() + func = SQL_FUNCTIONS_REGISTRY.get_function(node.name) + if func.source_execute is None: + raise QueryError('%s can not be called on mapped attribute' + % node.name) + state.source_cb_funcs.add(node) + func.update_cb_stack(stack) + + +# IGenerator implementation for RQL->SQL ####################################### class StateInfo(object): def __init__(self, existssols, unstablevars): self.existssols = existssols self.unstablevars = unstablevars self.subtables = {} + self.needs_source_cb = None + self.subquery_source_cb = None + self.source_cb_funcs = set() def reset(self, solution): """reset some visit variables""" @@ -276,6 +320,17 @@ self.restrictions = [] self._restr_stack = [] self.ignore_varmap = False + self._needs_source_cb = {} + + def merge_source_cbs(self, needs_source_cb): + if self.needs_source_cb is None: + self.needs_source_cb = needs_source_cb + elif needs_source_cb != self.needs_source_cb: + raise QueryError('query fetch some source mapped attribute, some not') + + def finalize_source_cbs(self): + if self.subquery_source_cb is not None: + self.needs_source_cb.update(self.subquery_source_cb) def add_restriction(self, restr): if restr: @@ -332,16 +387,16 @@ protected by a lock """ - def __init__(self, schema, dbms_helper, attrmap=None): + def __init__(self, schema, dbhelper, attrmap=None): self.schema = schema - self.dbms_helper = dbms_helper - self.dbencoding = dbms_helper.dbencoding - self.keyword_map = {'NOW' : self.dbms_helper.sql_current_timestamp, - 'TODAY': self.dbms_helper.sql_current_date, + self.dbhelper = dbhelper + self.dbencoding = dbhelper.dbencoding + self.keyword_map = {'NOW' : self.dbhelper.sql_current_timestamp, + 'TODAY': self.dbhelper.sql_current_date, } - if not self.dbms_helper.union_parentheses_support: + if not self.dbhelper.union_parentheses_support: self.union_sql = self.noparen_union_sql - if self.dbms_helper.fti_need_distinct: + if self.dbhelper.fti_need_distinct: self.__union_sql = self.union_sql self.union_sql = self.has_text_need_distinct_union_sql self._lock = threading.Lock() @@ -373,7 +428,7 @@ # union query for each rqlst / solution sql = self.union_sql(union) # we are done - return sql, self._query_attrs + return sql, self._query_attrs, self._state.needs_source_cb finally: self._lock.release() @@ -391,9 +446,10 @@ return '\nUNION ALL\n'.join(sqls) def noparen_union_sql(self, union, needalias=False): - # needed for sqlite backend which doesn't like parentheses around - # union query. This may cause bug in some condition (sort in one of - # the subquery) but will work in most case + # needed for sqlite backend which doesn't like parentheses around union + # query. This may cause bug in some condition (sort in one of the + # subquery) but will work in most case + # # see http://www.sqlite.org/cvstrac/tktview?tn=3074 sqls = (self.select_sql(select, needalias) for i, select in enumerate(union.children)) @@ -435,6 +491,9 @@ else: existssols, unstable = {}, () state = StateInfo(existssols, unstable) + if self._state is not None: + # state from a previous unioned select + state.merge_source_cbs(self._state.needs_source_cb) # treat subqueries self._subqueries_sql(select, state) # generate sql for this select node @@ -490,6 +549,7 @@ if fneedwrap: selection = ['T1.C%s' % i for i in xrange(len(origselection))] sql = 'SELECT %s FROM (%s) AS T1' % (','.join(selection), sql) + state.finalize_source_cbs() finally: select.selection = origselection # limit / offset @@ -504,13 +564,24 @@ def _subqueries_sql(self, select, state): for i, subquery in enumerate(select.with_): sql = self.union_sql(subquery.query, needalias=True) - tablealias = '_T%s' % i + tablealias = '_T%s' % i # XXX nested subqueries sql = '(%s) AS %s' % (sql, tablealias) state.subtables[tablealias] = (0, sql) + latest_state = self._state for vref in subquery.aliases: alias = vref.variable alias._q_sqltable = tablealias alias._q_sql = '%s.C%s' % (tablealias, alias.colnum) + try: + stack = latest_state.needs_source_cb[alias.colnum] + if state.subquery_source_cb is None: + state.subquery_source_cb = {} + for selectidx, vref in iter_mapped_var_sels(select, alias): + stack = stack[:] + update_source_cb_stack(state, select, vref, stack) + state.subquery_source_cb[selectidx] = stack + except KeyError: + continue def _solutions_sql(self, select, solutions, distinct, needalias): sqls = [] @@ -522,17 +593,18 @@ sql = [self._selection_sql(select.selection, distinct, needalias)] if self._state.restrictions: sql.append('WHERE %s' % ' AND '.join(self._state.restrictions)) + self._state.merge_source_cbs(self._state._needs_source_cb) # add required tables assert len(self._state.actual_tables) == 1, self._state.actual_tables tables = self._state.actual_tables[-1] if tables: # sort for test predictability sql.insert(1, 'FROM %s' % ', '.join(sorted(tables))) - elif self._state.restrictions and self.dbms_helper.needs_from_clause: + elif self._state.restrictions and self.dbhelper.needs_from_clause: sql.insert(1, 'FROM (SELECT 1) AS _T') sqls.append('\n'.join(sql)) if select.need_intersect: - #if distinct or not self.dbms_helper.intersect_all_support: + #if distinct or not self.dbhelper.intersect_all_support: return '\nINTERSECT\n'.join(sqls) #else: # return '\nINTERSECT ALL\n'.join(sqls) @@ -894,7 +966,13 @@ except KeyError: mapkey = '%s.%s' % (self._state.solution[lhs.name], rel.r_type) if mapkey in self.attr_map: - lhssql = self.attr_map[mapkey](self, lhs.variable, rel) + cb, sourcecb = self.attr_map[mapkey] + if sourcecb: + # callback is a source callback, we can't use this + # attribute in restriction + raise QueryError("can't use %s (%s) in restriction" + % (mapkey, rel.as_string())) + lhssql = cb(self, lhs.variable, rel) elif rel.r_type == 'eid': lhssql = lhs.variable._q_sql else: @@ -943,7 +1021,7 @@ not_ = True else: not_ = False - return self.dbms_helper.fti_restriction_sql(alias, const.eval(self._args), + return self.dbhelper.fti_restriction_sql(alias, const.eval(self._args), jointo, not_) + restriction def visit_comparison(self, cmp): @@ -956,7 +1034,7 @@ rhs = cmp.children[0] operator = cmp.operator if operator in ('IS', 'LIKE', 'ILIKE'): - if operator == 'ILIKE' and not self.dbms_helper.ilike_support: + if operator == 'ILIKE' and not self.dbhelper.ilike_support: operator = ' LIKE ' else: operator = ' %s ' % operator @@ -986,9 +1064,13 @@ def visit_function(self, func): """generate SQL name for a function""" - # func_sql_call will check function is supported by the backend - return self.dbms_helper.func_as_sql(func.name, - [c.accept(self) for c in func.children]) + args = [c.accept(self) for c in func.children] + if func in self._state.source_cb_funcs: + # function executed as a callback on the source + assert len(args) == 1 + return args[0] + # func_as_sql will check function is supported by the backend + return self.dbhelper.func_as_sql(func.name, args) def visit_constant(self, constant): """generate SQL name for a constant""" @@ -1003,7 +1085,7 @@ rel._q_needcast = value return self.keyword_map[value]() if constant.type == 'Boolean': - value = self.dbms_helper.boolean_value(value) + value = self.dbhelper.boolean_value(value) if constant.type == 'Substitute': _id = constant.value if isinstance(_id, unicode): @@ -1065,7 +1147,7 @@ self._state.add_restriction(restr) elif principal.r_type == 'has_text': sql = '%s.%s' % (self._fti_table(principal), - self.dbms_helper.fti_uid_attr) + self.dbhelper.fti_uid_attr) elif principal in variable.stinfo['rhsrelations']: if self.schema.rschema(principal.r_type).inlined: sql = self._linked_var_sql(variable) @@ -1155,12 +1237,20 @@ if isinstance(linkedvar, ColumnAlias): raise BadRQLQuery('variable %s should be selected by the subquery' % variable.name) - mapkey = '%s.%s' % (self._state.solution[linkedvar.name], rel.r_type) - if mapkey in self.attr_map: - return self.attr_map[mapkey](self, linkedvar, rel) try: sql = self._varmap['%s.%s' % (linkedvar.name, rel.r_type)] except KeyError: + mapkey = '%s.%s' % (self._state.solution[linkedvar.name], rel.r_type) + if mapkey in self.attr_map: + cb, sourcecb = self.attr_map[mapkey] + if not sourcecb: + return cb(self, linkedvar, rel) + # attribute mapped at the source level (bfss for instance) + stmt = rel.stmt + for selectidx, vref in iter_mapped_var_sels(stmt, variable): + stack = [cb] + update_source_cb_stack(self._state, stmt, vref, stack) + self._state._needs_source_cb[selectidx] = stack linkedvar.accept(self) sql = '%s.%s%s' % (linkedvar._q_sqltable, SQL_PREFIX, rel.r_type) return sql @@ -1267,7 +1357,7 @@ except AttributeError: pass self._state.done.add(relation) - alias = self.alias_and_add_table(self.dbms_helper.fti_table) + alias = self.alias_and_add_table(self.dbhelper.fti_table) relation._q_sqltable = alias return alias diff -r 4cc020ee70e2 -r b3b0b808a0ed server/sources/storages.py --- a/server/sources/storages.py Wed Mar 24 18:04:59 2010 +0100 +++ b/server/sources/storages.py Thu Mar 25 14:26:13 2010 +0100 @@ -11,10 +11,31 @@ repo.system_source.unset_storage(etype, attr) class Storage(object): - """abstract storage""" - def sqlgen_callback(self, generator, relation, linkedvar): - """sql generator callback when some attribute with a custom storage is - accessed + """abstract storage + + * If `source_callback` is true (by default), the callback will be run during + query result process of fetched attribute's valu and should have the + following prototype:: + + callback(self, source, value) + + where `value` is the value actually stored in the backend. None values + will be skipped (eg callback won't be called). + + * if `source_callback` is false, the callback will be run during sql + generation when some attribute with a custom storage is accessed and + should have the following prototype:: + + callback(self, generator, relation, linkedvar) + + where `generator` is the sql generator, `relation` the current rql syntax + tree relation and linkedvar the principal syntax tree variable holding the + attribute. + """ + is_source_callback = True + + def callback(self, *args): + """see docstring for prototype, which vary according to is_source_callback """ raise NotImplementedError() @@ -38,14 +59,16 @@ def __init__(self, defaultdir): self.default_directory = defaultdir - def sqlgen_callback(self, generator, linkedvar, relation): + def callback(self, source, value): """sql generator callback when some attribute with a custom storage is accessed """ - linkedvar.accept(generator) - return '_fsopen(%s.cw_%s)' % ( - linkedvar._q_sql.split('.', 1)[0], # table name - relation.r_type) # attribute name + fpath = source.binary_to_str(value) + try: + return Binary(file(fpath).read()) + except OSError, ex: + source.critical("can't open %s: %s", value, ex) + return None def entity_added(self, entity, attr): """an entity using this storage for attr has been added""" @@ -87,7 +110,8 @@ cu = sysource.doexec(entity._cw, 'SELECT cw_%s FROM cw_%s WHERE cw_eid=%s' % ( attr, entity.__regid__, entity.eid)) - return sysource._process_value(cu.fetchone()[0], [None, dbmod.BINARY], + BINARY = sysource.dbhelper.dbapi_module.BINARY + return sysource._process_value(cu.fetchone()[0], [None, BINARY], binarywrap=str) diff -r 4cc020ee70e2 -r b3b0b808a0ed server/sqlutils.py --- a/server/sqlutils.py Wed Mar 24 18:04:59 2010 +0100 +++ b/server/sqlutils.py Thu Mar 25 14:26:13 2010 +0100 @@ -188,9 +188,17 @@ return newargs return query_args - def process_result(self, cursor): + def process_result(self, cursor, column_callbacks=None): """return a list of CubicWeb compliant values from data in the given cursor """ + # use two different implementations to avoid paying the price of + # callback lookup for each *cell* in results when there is nothing to + # lookup + if not column_callbacks: + return self._process_result(cursor) + return self._cb_process_result(cursor, column_callbacks) + + def _process_result(self, cursor, column_callbacks=None): # begin bind to locals for optimization descr = cursor.description encoding = self._dbencoding @@ -208,6 +216,30 @@ results[i] = result return results + def _cb_process_result(self, cursor, column_callbacks): + # begin bind to locals for optimization + descr = cursor.description + encoding = self._dbencoding + process_value = self._process_value + binary = Binary + # /end + results = cursor.fetchall() + for i, line in enumerate(results): + result = [] + for col, value in enumerate(line): + if value is None: + result.append(value) + continue + cbstack = column_callbacks.get(col, None) + if cbstack is None: + value = process_value(value, descr[col], encoding, binary) + else: + for cb in cbstack: + value = cb(self, value) + result.append(value) + results[i] = result + return results + def preprocess_entity(self, entity): """return a dictionary to use as extra argument to cursor.execute to insert/update an entity into a SQL database @@ -277,28 +309,5 @@ import yams.constraints yams.constraints.patch_sqlite_decimal() - def fspath(eid, etype, attr): - try: - cu = cnx.cursor() - cu.execute('SELECT X.cw_%s FROM cw_%s as X ' - 'WHERE X.cw_eid=%%(eid)s' % (attr, etype), - {'eid': eid}) - return cu.fetchone()[0] - except: - import traceback - traceback.print_exc() - raise - cnx.create_function('fspath', 3, fspath) - - def _fsopen(fspath): - if fspath: - try: - return buffer(file(fspath).read()) - except: - import traceback - traceback.print_exc() - raise - cnx.create_function('_fsopen', 1, _fsopen) - sqlite_hooks = SQL_CONNECT_HOOKS.setdefault('sqlite', []) sqlite_hooks.append(init_sqlite_connexion) diff -r 4cc020ee70e2 -r b3b0b808a0ed server/test/unittest_rql2sql.py --- a/server/test/unittest_rql2sql.py Wed Mar 24 18:04:59 2010 +0100 +++ b/server/test/unittest_rql2sql.py Thu Mar 25 14:26:13 2010 +0100 @@ -1102,8 +1102,8 @@ #capture = True def setUp(self): RQLGeneratorTC.setUp(self) - dbms_helper = get_db_helper('postgres') - self.o = SQLGenerator(schema, dbms_helper) + dbhelper = get_db_helper('postgres') + self.o = SQLGenerator(schema, dbhelper) def _norm_sql(self, sql): return sql.strip() @@ -1113,8 +1113,8 @@ args = {'text': 'hip hop momo'} try: union = self._prepare(rql) - r, nargs = self.o.generate(union, args, - varmap=varmap) + r, nargs, cbs = self.o.generate(union, args, + varmap=varmap) args.update(nargs) self.assertLinesEquals((r % args).strip(), self._norm_sql(sql), striplines=True) except Exception, ex: @@ -1135,7 +1135,7 @@ def _checkall(self, rql, sql): try: rqlst = self._prepare(rql) - r, args = self.o.generate(rqlst) + r, args, cbs = self.o.generate(rqlst) self.assertEqual((r.strip(), args), sql) except Exception, ex: print rql @@ -1197,7 +1197,7 @@ def test_is_null_transform(self): union = self._prepare('Any X WHERE X login %(login)s') - r, args = self.o.generate(union, {'login': None}) + r, args, cbs = self.o.generate(union, {'login': None}) self.assertLinesEquals((r % args).strip(), '''SELECT _X.cw_eid FROM cw_CWUser AS _X @@ -1386,11 +1386,11 @@ '''SELECT COUNT(1) WHERE EXISTS(SELECT 1 FROM owned_by_relation AS rel_owned_by0, cw_Affaire AS _P WHERE rel_owned_by0.eid_from=_P.cw_eid AND rel_owned_by0.eid_to=1 UNION SELECT 1 FROM owned_by_relation AS rel_owned_by1, cw_Note AS _P WHERE rel_owned_by1.eid_from=_P.cw_eid AND rel_owned_by1.eid_to=1)''') - def test_attr_map(self): + def test_attr_map_sqlcb(self): def generate_ref(gen, linkedvar, rel): linkedvar.accept(gen) return 'VERSION_DATA(%s)' % linkedvar._q_sql - self.o.attr_map['Affaire.ref'] = generate_ref + self.o.attr_map['Affaire.ref'] = (generate_ref, False) try: self._check('Any R WHERE X ref R', '''SELECT VERSION_DATA(_X.cw_eid) @@ -1402,13 +1402,24 @@ finally: self.o.attr_map.clear() + def test_attr_map_sourcecb(self): + cb = lambda x,y: None + self.o.attr_map['Affaire.ref'] = (cb, True) + try: + union = self._prepare('Any R WHERE X ref R') + r, nargs, cbs = self.o.generate(union, args={}) + self.assertLinesEquals(r.strip(), 'SELECT _X.cw_ref\nFROM cw_Affaire AS _X') + self.assertEquals(cbs, {0: [cb]}) + finally: + self.o.attr_map.clear() + class SqliteSQLGeneratorTC(PostgresSQLGeneratorTC): def setUp(self): RQLGeneratorTC.setUp(self) - dbms_helper = get_db_helper('sqlite') - self.o = SQLGenerator(schema, dbms_helper) + dbhelper = get_db_helper('sqlite') + self.o = SQLGenerator(schema, dbhelper) def _norm_sql(self, sql): return sql.strip().replace(' ILIKE ', ' LIKE ').replace('\nINTERSECT ALL\n', '\nINTERSECT\n') @@ -1515,8 +1526,8 @@ def setUp(self): RQLGeneratorTC.setUp(self) - dbms_helper = get_db_helper('mysql') - self.o = SQLGenerator(schema, dbms_helper) + dbhelper = get_db_helper('mysql') + self.o = SQLGenerator(schema, dbhelper) def _norm_sql(self, sql): sql = sql.strip().replace(' ILIKE ', ' LIKE ').replace('TRUE', '1').replace('FALSE', '0') diff -r 4cc020ee70e2 -r b3b0b808a0ed server/test/unittest_storage.py --- a/server/test/unittest_storage.py Wed Mar 24 18:04:59 2010 +0100 +++ b/server/test/unittest_storage.py Thu Mar 25 14:26:13 2010 +0100 @@ -13,7 +13,7 @@ import shutil import tempfile -from cubicweb import Binary +from cubicweb import Binary, QueryError from cubicweb.selectors import implements from cubicweb.server.sources import storages from cubicweb.server.hook import Hook, Operation @@ -51,41 +51,114 @@ shutil.rmtree(self.tempdir) - def create_file(self, content): + def create_file(self, content='the-data'): req = self.request() return req.create_entity('File', data=Binary(content), data_format=u'text/plain', data_name=u'foo') - def test_bfs_storage(self): - f1 = self.create_file(content='the-data') + def test_bfss_storage(self): + f1 = self.create_file() expected_filepath = osp.join(self.tempdir, '%s_data' % f1.eid) self.failUnless(osp.isfile(expected_filepath)) self.assertEquals(file(expected_filepath).read(), 'the-data') + self.rollback() + self.failIf(osp.isfile(expected_filepath)) + f1 = self.create_file() + self.commit() + self.assertEquals(file(expected_filepath).read(), 'the-data') + f1.set_attributes(data=Binary('the new data')) + self.rollback() + self.assertEquals(file(expected_filepath).read(), 'the-data') + f1.delete() + self.failUnless(osp.isfile(expected_filepath)) + self.rollback() + self.failUnless(osp.isfile(expected_filepath)) + f1.delete() + self.commit() + self.failIf(osp.isfile(expected_filepath)) - def test_sqlite_fspath(self): - f1 = self.create_file(content='the-data') + def test_bfss_sqlite_fspath(self): + f1 = self.create_file() expected_filepath = osp.join(self.tempdir, '%s_data' % f1.eid) - fspath = self.execute('Any fspath(F, "File", "data") WHERE F eid %(f)s', + fspath = self.execute('Any fspath(D) WHERE F eid %(f)s, F data D', {'f': f1.eid})[0][0] self.assertEquals(fspath.getvalue(), expected_filepath) - def test_fs_importing_doesnt_touch_path(self): + def test_bfss_fs_importing_doesnt_touch_path(self): self.session.transaction_data['fs_importing'] = True f1 = self.session.create_entity('File', data=Binary('/the/path'), data_format=u'text/plain', data_name=u'foo') - fspath = self.execute('Any fspath(F, "File", "data") WHERE F eid %(f)s', + fspath = self.execute('Any fspath(D) WHERE F eid %(f)s, F data D', {'f': f1.eid})[0][0] self.assertEquals(fspath.getvalue(), '/the/path') - def test_storage_transparency(self): + def test_source_storage_transparency(self): self.vreg._loadedmods[__name__] = {} self.vreg.register(DummyBeforeHook) self.vreg.register(DummyAfterHook) try: - self.create_file(content='the-data') + self.create_file() finally: self.vreg.unregister(DummyBeforeHook) self.vreg.unregister(DummyAfterHook) + def test_source_mapped_attribute_error_cases(self): + ex = self.assertRaises(QueryError, self.execute, + 'Any X WHERE X data ~= "hop", X is File') + self.assertEquals(str(ex), 'can\'t use File.data (X data ILIKE "hop") in restriction') + ex = self.assertRaises(QueryError, self.execute, + 'Any X, Y WHERE X data D, Y data D, ' + 'NOT X identity Y, X is File, Y is File') + self.assertEquals(str(ex), "can't use D as a restriction variable") + # query returning mix of mapped / regular attributes (only file.data + # mapped, not image.data for instance) + ex = self.assertRaises(QueryError, self.execute, + 'Any X WITH X BEING (' + ' (Any NULL)' + ' UNION ' + ' (Any D WHERE X data D, X is File)' + ')') + self.assertEquals(str(ex), 'query fetch some source mapped attribute, some not') + ex = self.assertRaises(QueryError, self.execute, + '(Any D WHERE X data D, X is File)' + ' UNION ' + '(Any D WHERE X data D, X is Image)') + self.assertEquals(str(ex), 'query fetch some source mapped attribute, some not') + ex = self.assertRaises(QueryError, + self.execute, 'Any D WHERE X data D') + self.assertEquals(str(ex), 'query fetch some source mapped attribute, some not') + + def test_source_mapped_attribute_advanced(self): + f1 = self.create_file() + rset = self.execute('Any X,D WITH D,X BEING (' + ' (Any D, X WHERE X eid %(x)s, X data D)' + ' UNION ' + ' (Any D, X WHERE X eid %(x)s, X data D)' + ')', {'x': f1.eid}, 'x') + self.assertEquals(len(rset), 2) + self.assertEquals(rset[0][0], f1.eid) + self.assertEquals(rset[1][0], f1.eid) + self.assertEquals(rset[0][1].getvalue(), 'the-data') + self.assertEquals(rset[1][1].getvalue(), 'the-data') + rset = self.execute('Any X,LENGTH(D) WHERE X eid %(x)s, X data D', + {'x': f1.eid}, 'x') + self.assertEquals(len(rset), 1) + self.assertEquals(rset[0][0], f1.eid) + self.assertEquals(rset[0][1], len('the-data')) + rset = self.execute('Any X,LENGTH(D) WITH D,X BEING (' + ' (Any D, X WHERE X eid %(x)s, X data D)' + ' UNION ' + ' (Any D, X WHERE X eid %(x)s, X data D)' + ')', {'x': f1.eid}, 'x') + self.assertEquals(len(rset), 2) + self.assertEquals(rset[0][0], f1.eid) + self.assertEquals(rset[1][0], f1.eid) + self.assertEquals(rset[0][1], len('the-data')) + self.assertEquals(rset[1][1], len('the-data')) + ex = self.assertRaises(QueryError, self.execute, + 'Any X,UPPER(D) WHERE X eid %(x)s, X data D', + {'x': f1.eid}, 'x') + self.assertEquals(str(ex), 'UPPER can not be called on mapped attribute') + if __name__ == '__main__': unittest_main() diff -r 4cc020ee70e2 -r b3b0b808a0ed web/facet.py --- a/web/facet.py Wed Mar 24 18:04:59 2010 +0100 +++ b/web/facet.py Thu Mar 25 14:26:13 2010 +0100 @@ -341,10 +341,11 @@ class RelationFacet(VocabularyFacet): __select__ = partial_relation_possible() & match_context_prop() - # class attributes to configure the rel ation facet + # class attributes to configure the relation facet rtype = None role = 'subject' target_attr = 'eid' + target_type = None # set this to a stored procedure name if you want to sort on the result of # this function's result instead of direct value sortfunc = None @@ -368,8 +369,11 @@ sort = self.sortasc try: mainvar = self.filtered_variable - insert_attr_select_relation(rqlst, mainvar, self.rtype, self.role, - self.target_attr, self.sortfunc, sort) + var = insert_attr_select_relation( + rqlst, mainvar, self.rtype, self.role, self.target_attr, + self.sortfunc, sort) + if self.target_type is not None: + rqlst.add_type_restriction(var, self.target_type) try: rset = self.rqlexec(rqlst.as_string(), self.cw_rset.args, self.cw_rset.cachekey) except: diff -r 4cc020ee70e2 -r b3b0b808a0ed web/webconfig.py --- a/web/webconfig.py Wed Mar 24 18:04:59 2010 +0100 +++ b/web/webconfig.py Thu Mar 25 14:26:13 2010 +0100 @@ -303,11 +303,13 @@ baseurl = self['base-url'] or self.default_base_url() if baseurl and baseurl[-1] != '/': baseurl += '/' - self.global_set_option('base-url', baseurl) + if not self.repairing: + self.global_set_option('base-url', baseurl) httpsurl = self['https-url'] if httpsurl and httpsurl[-1] != '/': httpsurl += '/' - self.global_set_option('https-url', httpsurl) + if not self.repairing: + self.global_set_option('https-url', httpsurl) def _build_ext_resources(self): libresourcesfile = join(self.shared_dir(), 'data', 'external_resources')