# HG changeset patch # User Katia Saurfelt # Date 1267439174 -3600 # Node ID 083b4d45419297300de5db49380edfc9f5b314bd # Parent 9767cc516b4fa291b982b5587681ae7913957db3 server/web api for accessing to deleted_entites diff -r 9767cc516b4f -r 083b4d454192 __pkginfo__.py --- a/__pkginfo__.py Wed Mar 10 16:07:24 2010 +0100 +++ b/__pkginfo__.py Mon Mar 01 11:26:14 2010 +0100 @@ -7,7 +7,7 @@ distname = "cubicweb" modname = "cubicweb" -numversion = (3, 6, 2) +numversion = (3, 7, 0) version = '.'.join(str(num) for num in numversion) license = 'LGPL' diff -r 9767cc516b4f -r 083b4d454192 dataimport.py --- a/dataimport.py Wed Mar 10 16:07:24 2010 +0100 +++ b/dataimport.py Mon Mar 01 11:26:14 2010 +0100 @@ -559,6 +559,8 @@ self._nb_inserted_types = 0 self._nb_inserted_relations = 0 self.rql = session.unsafe_execute + # disable undoing + session.undo_actions = frozenset() def create_entity(self, etype, **kwargs): for k, v in kwargs.iteritems(): diff -r 9767cc516b4f -r 083b4d454192 dbapi.py --- a/dbapi.py Wed Mar 10 16:07:24 2010 +0100 +++ b/dbapi.py Mon Mar 01 11:26:14 2010 +0100 @@ -57,6 +57,7 @@ etypescls = cwvreg.VRegistry.REGISTRY_FACTORY['etypes'] etypescls.etype_class = etypescls.orig_etype_class + class ConnectionProperties(object): def __init__(self, cnxtype=None, lang=None, close=True, log=False): self.cnxtype = cnxtype or 'pyro' @@ -546,16 +547,16 @@ self._closed = 1 def commit(self): - """Commit any pending transaction to the database. Note that if the - database supports an auto-commit feature, this must be initially off. An - interface method may be provided to turn it back on. + """Commit pending transaction for this connection to the repository. - Database modules that do not support transactions should implement this - method with void functionality. + may raises `Unauthorized` or `ValidationError` if we attempted to do + something we're not allowed to for security or integrity reason. + + If the transaction is undoable, a transaction id will be returned. """ if not self._closed is None: raise ProgrammingError('Connection is already closed') - self._repo.commit(self.sessionid) + return self._repo.commit(self.sessionid) def rollback(self): """This method is optional since not all databases provide transaction @@ -582,6 +583,73 @@ req = self.request() return self.cursor_class(self, self._repo, req=req) + # undo support ############################################################ + + def undoable_transactions(self, ueid=None, req=None, **actionfilters): + """Return a list of undoable transaction objects by the connection's + user, ordered by descendant transaction time. + + Managers may filter according to user (eid) who has done the transaction + using the `ueid` argument. Others will only see their own transactions. + + Additional filtering capabilities is provided by using the following + named arguments: + + * `etype` to get only transactions creating/updating/deleting entities + of the given type + + * `eid` to get only transactions applied to entity of the given eid + + * `action` to get only transactions doing the given action (action in + 'C', 'U', 'D', 'A', 'R'). If `etype`, action can only be 'C', 'U' or + 'D'. + + * `public`: when additional filtering is provided, their are by default + only searched in 'public' actions, unless a `public` argument is given + and set to false. + """ + txinfos = self._repo.undoable_transactions(self.sessionid, ueid, + **actionfilters) + if req is None: + req = self.request() + for txinfo in txinfos: + txinfo.req = req + return txinfos + + def transaction_info(self, txuuid, req=None): + """Return transaction object for the given uid. + + raise `NoSuchTransaction` if not found or if session's user is not + allowed (eg not in managers group and the transaction doesn't belong to + him). + """ + txinfo = self._repo.transaction_info(self.sessionid, txuuid) + if req is None: + req = self.request() + txinfo.req = req + return txinfo + + def transaction_actions(self, txuuid, public=True): + """Return an ordered list of action effectued during that transaction. + + If public is true, return only 'public' actions, eg not ones triggered + under the cover by hooks, else return all actions. + + raise `NoSuchTransaction` if the transaction is not found or if + session's user is not allowed (eg not in managers group and the + transaction doesn't belong to him). + """ + return self._repo.transaction_actions(self.sessionid, txuuid, public) + + def undo_transaction(self, txuuid): + """Undo the given transaction. Return potential restoration errors. + + raise `NoSuchTransaction` if not found or if session's user is not + allowed (eg not in managers group and the transaction doesn't belong to + him). + """ + return self._repo.undo_transaction(self.sessionid, txuuid) + # cursor object ############################################################### diff -r 9767cc516b4f -r 083b4d454192 devtools/__init__.py --- a/devtools/__init__.py Wed Mar 10 16:07:24 2010 +0100 +++ b/devtools/__init__.py Mon Mar 01 11:26:14 2010 +0100 @@ -106,8 +106,6 @@ self.init_log(log_threshold, force=True) # need this, usually triggered by cubicweb-ctl self.load_cwctl_plugins() - self.global_set_option('anonymous-user', 'anon') - self.global_set_option('anonymous-password', 'anon') anonymous_user = TwistedConfiguration.anonymous_user.im_func @@ -123,6 +121,8 @@ super(TestServerConfiguration, self).load_configuration() self.global_set_option('anonymous-user', 'anon') self.global_set_option('anonymous-password', 'anon') + # no undo support in tests + self.global_set_option('undo-support', '') def main_config_file(self): """return instance's control configuration file""" diff -r 9767cc516b4f -r 083b4d454192 devtools/testlib.py --- a/devtools/testlib.py Wed Mar 10 16:07:24 2010 +0100 +++ b/devtools/testlib.py Mon Mar 01 11:26:14 2010 +0100 @@ -321,7 +321,10 @@ @nocoverage def commit(self): - self.cnx.commit() + try: + return self.cnx.commit() + finally: + self.session.set_pool() # ensure pool still set after commit @nocoverage def rollback(self): @@ -329,6 +332,8 @@ self.cnx.rollback() except ProgrammingError: pass + finally: + self.session.set_pool() # ensure pool still set after commit # # server side db api ####################################################### diff -r 9767cc516b4f -r 083b4d454192 entity.py --- a/entity.py Wed Mar 10 16:07:24 2010 +0100 +++ b/entity.py Mon Mar 01 11:26:14 2010 +0100 @@ -461,7 +461,7 @@ all(matching_groups(e.get_groups('read')) for e in targets): yield rschema, 'subject' - def to_complete_attributes(self, skip_bytes=True): + def to_complete_attributes(self, skip_bytes=True, skip_pwd=True): for rschema, attrschema in self.e_schema.attribute_definitions(): # skip binary data by default if skip_bytes and attrschema.type == 'Bytes': @@ -472,13 +472,13 @@ # password retreival is blocked at the repository server level rdef = rschema.rdef(self.e_schema, attrschema) if not self._cw.user.matching_groups(rdef.get_groups('read')) \ - or attrschema.type == 'Password': + or (attrschema.type == 'Password' and skip_pwd): self[attr] = None continue yield attr _cw_completed = False - def complete(self, attributes=None, skip_bytes=True): + def complete(self, attributes=None, skip_bytes=True, skip_pwd=True): """complete this entity by adding missing attributes (i.e. query the repository to fill the entity) @@ -495,7 +495,7 @@ V = varmaker.next() rql = ['WHERE %s eid %%(x)s' % V] selected = [] - for attr in (attributes or self.to_complete_attributes(skip_bytes)): + for attr in (attributes or self.to_complete_attributes(skip_bytes, skip_pwd)): # if attribute already in entity, nothing to do if self.has_key(attr): continue diff -r 9767cc516b4f -r 083b4d454192 goa/gaesource.py --- a/goa/gaesource.py Wed Mar 10 16:07:24 2010 +0100 +++ b/goa/gaesource.py Mon Mar 01 11:26:14 2010 +0100 @@ -255,10 +255,11 @@ if asession.user.eid == entity.eid: asession.user.update(dict(gaeentity)) - def delete_entity(self, session, etype, eid): + def delete_entity(self, session, entity): """delete an entity from the source""" # do not delay delete_entity as other modifications to ensure # consistency + eid = entity.eid key = Key(eid) Delete(key) session.clear_datastore_cache(key) diff -r 9767cc516b4f -r 083b4d454192 hooks/__init__.py --- a/hooks/__init__.py Wed Mar 10 16:07:24 2010 +0100 +++ b/hooks/__init__.py Mon Mar 01 11:26:14 2010 +0100 @@ -1,1 +1,36 @@ -"""core hooks""" +"""core hooks + +:organization: Logilab +:copyright: 2009-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. +:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr +:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses +""" +__docformat__ = "restructuredtext en" + +from datetime import timedelta, datetime +from cubicweb.server import hook + +class ServerStartupHook(hook.Hook): + """task to cleanup expirated auth cookie entities""" + __regid__ = 'cw_cleanup_transactions' + events = ('server_startup',) + + def __call__(self): + # XXX use named args and inner functions to avoid referencing globals + # which may cause reloading pb + lifetime = timedelta(days=self.repo.config['keep-transaction-lifetime']) + def cleanup_old_transactions(repo=self.repo, lifetime=lifetime): + mindate = datetime.now() - lifetime + session = repo.internal_session() + try: + session.system_sql( + 'DELETE FROM transaction WHERE tx_time < %(time)s', + {'time': mindate}) + # cleanup deleted entities + session.system_sql( + 'DELETE FROM deleted_entities WHERE dtime < %(time)s', + {'time': mindate}) + session.commit() + finally: + session.close() + self.repo.looping_task(60*60*24, cleanup_old_transactions, self.repo) diff -r 9767cc516b4f -r 083b4d454192 misc/migration/3.7.0_Any.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/misc/migration/3.7.0_Any.py Mon Mar 01 11:26:14 2010 +0100 @@ -0,0 +1,40 @@ +typemap = repo.system_source.dbhelper.TYPE_MAPPING +sqls = """ +CREATE TABLE transactions ( + tx_uuid CHAR(32) PRIMARY KEY NOT NULL, + tx_user INTEGER NOT NULL, + tx_time %s NOT NULL +);; +CREATE INDEX transactions_tx_user_idx ON transactions(tx_user);; + +CREATE TABLE tx_entity_actions ( + tx_uuid CHAR(32) REFERENCES transactions(tx_uuid) ON DELETE CASCADE, + txa_action CHAR(1) NOT NULL, + txa_public %s NOT NULL, + txa_order INTEGER, + eid INTEGER NOT NULL, + etype VARCHAR(64) NOT NULL, + changes %s +);; +CREATE INDEX tx_entity_actions_txa_action_idx ON tx_entity_actions(txa_action);; +CREATE INDEX tx_entity_actions_txa_public_idx ON tx_entity_actions(txa_public);; +CREATE INDEX tx_entity_actions_eid_idx ON tx_entity_actions(eid);; +CREATE INDEX tx_entity_actions_etype_idx ON tx_entity_actions(etype);; + +CREATE TABLE tx_relation_actions ( + tx_uuid CHAR(32) REFERENCES transactions(tx_uuid) ON DELETE CASCADE, + txa_action CHAR(1) NOT NULL, + txa_public %s NOT NULL, + txa_order INTEGER, + eid_from INTEGER NOT NULL, + eid_to INTEGER NOT NULL, + rtype VARCHAR(256) NOT NULL +);; +CREATE INDEX tx_relation_actions_txa_action_idx ON tx_relation_actions(txa_action);; +CREATE INDEX tx_relation_actions_txa_public_idx ON tx_relation_actions(txa_public);; +CREATE INDEX tx_relation_actions_eid_from_idx ON tx_relation_actions(eid_from);; +CREATE INDEX tx_relation_actions_eid_to_idx ON tx_relation_actions(eid_to) +""" % (typemap['Datetime'], + typemap['Boolean'], typemap['Bytes'], typemap['Boolean']) +for statement in sqls.split(';;'): + sql(statement) diff -r 9767cc516b4f -r 083b4d454192 schema.py --- a/schema.py Wed Mar 10 16:07:24 2010 +0100 +++ b/schema.py Mon Mar 01 11:26:14 2010 +0100 @@ -34,14 +34,15 @@ PURE_VIRTUAL_RTYPES = set(('identity', 'has_text',)) VIRTUAL_RTYPES = set(('eid', 'identity', 'has_text',)) -# set of meta-relations available for every entity types +# set of meta-relations available for every entity types META_RTYPES = set(( 'owned_by', 'created_by', 'is', 'is_instance_of', 'identity', 'eid', 'creation_date', 'modification_date', 'has_text', 'cwuri', )) -SYSTEM_RTYPES = set(('require_permission', 'custom_workflow', 'in_state', 'wf_info_for')) +SYSTEM_RTYPES = set(('require_permission', 'custom_workflow', 'in_state', + 'wf_info_for')) -# set of entity and relation types used to build the schema +# set of entity and relation types used to build the schema SCHEMA_TYPES = set(( 'CWEType', 'CWRType', 'CWAttribute', 'CWRelation', 'CWConstraint', 'CWConstraintType', 'RQLExpression', diff -r 9767cc516b4f -r 083b4d454192 server/__init__.py --- a/server/__init__.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/__init__.py Mon Mar 01 11:26:14 2010 +0100 @@ -150,7 +150,7 @@ schemasql = sqlschema(schema, driver) #skip_entities=[str(e) for e in schema.entities() # if not repo.system_source.support_entity(str(e))]) - sqlexec(schemasql, execute, pbtitle=_title) + sqlexec(schemasql, execute, pbtitle=_title, delimiter=';;') sqlcursor.close() sqlcnx.commit() sqlcnx.close() diff -r 9767cc516b4f -r 083b4d454192 server/repository.py --- a/server/repository.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/repository.py Mon Mar 01 11:26:14 2010 +0100 @@ -24,9 +24,11 @@ from os.path import join from datetime import datetime from time import time, localtime, strftime +#from pickle import dumps from logilab.common.decorators import cached from logilab.common.compat import any +from logilab.common import flatten from yams import BadSchemaDefinition from rql import RQLSyntaxError @@ -630,7 +632,7 @@ """commit transaction for the session with the given id""" self.debug('begin commit for session %s', sessionid) try: - self._get_session(sessionid).commit() + return self._get_session(sessionid).commit() except (ValidationError, Unauthorized): raise except: @@ -679,10 +681,42 @@ custom properties) """ session = self._get_session(sessionid, setpool=False) - # update session properties for prop, value in props.items(): session.change_property(prop, value) + def undoable_transactions(self, sessionid, ueid=None, **actionfilters): + """See :class:`cubicweb.dbapi.Connection.undoable_transactions`""" + session = self._get_session(sessionid, setpool=True) + try: + return self.system_source.undoable_transactions(session, ueid, + **actionfilters) + finally: + session.reset_pool() + + def transaction_info(self, sessionid, txuuid): + """See :class:`cubicweb.dbapi.Connection.transaction_info`""" + session = self._get_session(sessionid, setpool=True) + try: + return self.system_source.tx_info(session, txuuid) + finally: + session.reset_pool() + + def transaction_actions(self, sessionid, txuuid, public=True): + """See :class:`cubicweb.dbapi.Connection.transaction_actions`""" + session = self._get_session(sessionid, setpool=True) + try: + return self.system_source.tx_actions(session, txuuid, public) + finally: + session.reset_pool() + + def undo_transaction(self, sessionid, txuuid): + """See :class:`cubicweb.dbapi.Connection.undo_transaction`""" + session = self._get_session(sessionid, setpool=True) + try: + return self.system_source.undo_transaction(session, txuuid) + finally: + session.reset_pool() + # public (inter-repository) interface ##################################### def entities_modified_since(self, etypes, mtime): @@ -886,60 +920,58 @@ self.system_source.add_info(session, entity, source, extid, complete) CleanupEidTypeCacheOp(session) - def delete_info(self, session, eid): - self._prepare_delete_info(session, eid) - self._delete_info(session, eid) + def delete_info(self, session, entity, sourceuri, extid): + """called by external source when some entity known by the system source + has been deleted in the external source + """ + self._prepare_delete_info(session, entity, sourceuri) + self._delete_info(session, entity, sourceuri, extid) - def _prepare_delete_info(self, session, eid): + def _prepare_delete_info(self, session, entity, sourceuri): """prepare the repository for deletion of an entity: * update the fti * mark eid as being deleted in session info * setup cache update operation + * if undoable, get back all entity's attributes and relation """ + eid = entity.eid self.system_source.fti_unindex_entity(session, eid) pending = session.transaction_data.setdefault('pendingeids', set()) pending.add(eid) CleanupEidTypeCacheOp(session) - def _delete_info(self, session, eid): + def _delete_info(self, session, entity, sourceuri, extid): + # attributes=None, relations=None): """delete system information on deletion of an entity: - * delete all relations on this entity - * transfer record from the entities table to the deleted_entities table + * delete all remaining relations from/to this entity + * call delete info on the system source which will transfer record from + the entities table to the deleted_entities table """ - etype, uri, extid = self.type_and_source_from_eid(eid, session) - self._clear_eid_relations(session, etype, eid) - self.system_source.delete_info(session, eid, etype, uri, extid) - - def _clear_eid_relations(self, session, etype, eid): - """when a entity is deleted, build and execute rql query to delete all - its relations - """ - rql = [] - eschema = self.schema.eschema(etype) pendingrtypes = session.transaction_data.get('pendingrtypes', ()) + # delete remaining relations: if user can delete the entity, he can + # delete all its relations without security checking with security_enabled(session, read=False, write=False): - for rschema, targetschemas, x in eschema.relation_definitions(): + eid = entity.eid + for rschema, _, role in entity.e_schema.relation_definitions(): rtype = rschema.type if rtype in schema.VIRTUAL_RTYPES or rtype in pendingrtypes: continue - var = '%s%s' % (rtype.upper(), x.upper()) - if x == 'subject': + if role == 'subject': # don't skip inlined relation so they are regularly # deleted and so hooks are correctly called - selection = 'X %s %s' % (rtype, var) + selection = 'X %s Y' % rtype else: - selection = '%s %s X' % (var, rtype) + selection = 'Y %s X' % rtype rql = 'DELETE %s WHERE X eid %%(x)s' % selection - # if user can delete the entity, he can delete all its relations - # without security checking session.execute(rql, {'x': eid}, 'x', build_descr=False) + self.system_source.delete_info(session, entity, sourceuri, extid) def locate_relation_source(self, session, subject, rtype, object): subjsource = self.source_from_eid(subject, session) objsource = self.source_from_eid(object, session) if not subjsource is objsource: source = self.system_source - if not (subjsource.may_cross_relation(rtype) + if not (subjsource.may_cross_relation(rtype) and objsource.may_cross_relation(rtype)): raise MultiSourcesError( "relation %s can't be crossed among sources" @@ -983,7 +1015,7 @@ self.hm.call_hooks('before_add_entity', session, entity=entity) # XXX use entity.keys here since edited_attributes is not updated for # inline relations - for attr in entity.keys(): + for attr in entity.iterkeys(): rschema = eschema.subjrels[attr] if not rschema.final: # inlined relation relations.append((attr, entity[attr])) @@ -1094,19 +1126,16 @@ def glob_delete_entity(self, session, eid): """delete an entity and all related entities from the repository""" - # call delete_info before hooks - self._prepare_delete_info(session, eid) - etype, uri, extid = self.type_and_source_from_eid(eid, session) + entity = session.entity_from_eid(eid) + etype, sourceuri, extid = self.type_and_source_from_eid(eid, session) + self._prepare_delete_info(session, entity, sourceuri) if server.DEBUG & server.DBG_REPO: print 'DELETE entity', etype, eid - if eid == 937: - server.DEBUG |= (server.DBG_SQL | server.DBG_RQL | server.DBG_MORE) - source = self.sources_by_uri[uri] + source = self.sources_by_uri[sourceuri] if source.should_call_hooks: - entity = session.entity_from_eid(eid) self.hm.call_hooks('before_delete_entity', session, entity=entity) - self._delete_info(session, eid) - source.delete_entity(session, etype, eid) + self._delete_info(session, entity, sourceuri, extid) + source.delete_entity(session, entity) if source.should_call_hooks: self.hm.call_hooks('after_delete_entity', session, entity=entity) # don't clear cache here this is done in a hook on commit diff -r 9767cc516b4f -r 083b4d454192 server/serverconfig.py --- a/server/serverconfig.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/serverconfig.py Mon Mar 01 11:26:14 2010 +0100 @@ -127,6 +127,20 @@ 'help': 'size of the parsed rql cache size.', 'group': 'main', 'inputlevel': 1, }), + ('undo-support', + {'type' : 'string', 'default': '', + 'help': 'string defining actions that will have undo support: \ +[C]reate [U]pdate [D]elete entities / [A]dd [R]emove relation. Leave it empty \ +for no undo support, set it to CUDAR for full undo support, or to DR for \ +support undoing of deletion only.', + 'group': 'main', 'inputlevel': 1, + }), + ('keep-transaction-lifetime', + {'type' : 'int', 'default': 7, + 'help': 'number of days during which transaction records should be \ +kept (hence undoable).', + 'group': 'main', 'inputlevel': 1, + }), ('delay-full-text-indexation', {'type' : 'yn', 'default': False, 'help': 'When full text indexation of entity has a too important cost' diff -r 9767cc516b4f -r 083b4d454192 server/session.py --- a/server/session.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/session.py Mon Mar 01 11:26:14 2010 +0100 @@ -12,12 +12,13 @@ import sys import threading from time import time +from uuid import uuid4 from logilab.common.deprecation import deprecated from rql.nodes import VariableRef, Function, ETYPE_PYOBJ_MAP, etype_from_pyobj from yams import BASE_TYPES -from cubicweb import Binary, UnknownEid +from cubicweb import Binary, UnknownEid, schema from cubicweb.req import RequestSessionBase from cubicweb.dbapi import ConnectionProperties from cubicweb.utils import make_uid @@ -25,6 +26,10 @@ ETYPE_PYOBJ_MAP[Binary] = 'Bytes' +NO_UNDO_TYPES = schema.SCHEMA_TYPES.copy() +NO_UNDO_TYPES.add('CWCache') +# XXX rememberme,forgotpwd,apycot,vcsfile + def is_final(rqlst, variable, args): # try to find if this is a final var or not for select in rqlst.children: @@ -110,6 +115,7 @@ """tie session id, user, connections pool and other session data all together """ + is_internal_session = False def __init__(self, user, repo, cnxprops=None, _id=None): super(Session, self).__init__(repo.vreg) @@ -120,8 +126,14 @@ self.cnxtype = cnxprops.cnxtype self.creation = time() self.timestamp = self.creation - self.is_internal_session = False self.default_mode = 'read' + # support undo for Create Update Delete entity / Add Remove relation + if repo.config.creating or repo.config.repairing or self.is_internal_session: + self.undo_actions = () + else: + self.undo_actions = set(repo.config['undo-support'].upper()) + if self.undo_actions - set('CUDAR'): + raise Exception('bad undo-support string in configuration') # short cut to querier .execute method self._execute = repo.querier.execute # shared data, used to communicate extra information between the client @@ -334,7 +346,10 @@ # so we can't rely on simply checking session.read_security, but # recalling the first transition from DEFAULT_SECURITY to something # else (False actually) is not perfect but should be enough - self._threaddata.dbapi_query = oldmode is self.DEFAULT_SECURITY + # + # also reset dbapi_query to true when we go back to DEFAULT_SECURITY + self._threaddata.dbapi_query = (oldmode is self.DEFAULT_SECURITY + or activated is self.DEFAULT_SECURITY) return oldmode @property @@ -689,6 +704,7 @@ self.critical('error while %sing', trstate, exc_info=sys.exc_info()) self.info('%s session %s done', trstate, self.id) + return self.transaction_uuid(set=False) finally: self._clear_thread_data() self._touch() @@ -769,6 +785,27 @@ else: self.pending_operations.insert(index, operation) + # undo support ############################################################ + + def undoable_action(self, action, ertype): + return action in self.undo_actions and not ertype in NO_UNDO_TYPES + # XXX elif transaction on mark it partial + + def transaction_uuid(self, set=True): + try: + return self.transaction_data['tx_uuid'] + except KeyError: + if not set: + return + self.transaction_data['tx_uuid'] = uuid = uuid4().hex + self.repo.system_source.start_undoable_transaction(self, uuid) + return uuid + + def transaction_inc_action_counter(self): + num = self.transaction_data.setdefault('tx_action_count', 0) + 1 + self.transaction_data['tx_action_count'] = num + return num + # querier helpers ######################################################### @property @@ -890,13 +927,13 @@ class InternalSession(Session): """special session created internaly by the repository""" + is_internal_session = True def __init__(self, repo, cnxprops=None): super(InternalSession, self).__init__(InternalManager(), repo, cnxprops, _id='internal') self.user.req = self # XXX remove when "vreg = user.req.vreg" hack in entity.py is gone self.cnxtype = 'inmemory' - self.is_internal_session = True self.disable_hook_categories('integrity') diff -r 9767cc516b4f -r 083b4d454192 server/sources/__init__.py --- a/server/sources/__init__.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/sources/__init__.py Mon Mar 01 11:26:14 2010 +0100 @@ -351,7 +351,7 @@ """update an entity in the source""" raise NotImplementedError() - def delete_entity(self, session, etype, eid): + def delete_entity(self, session, entity): """delete an entity from the source""" raise NotImplementedError() @@ -372,11 +372,15 @@ def create_eid(self, session): raise NotImplementedError() - def add_info(self, session, entity, source, extid=None): + def add_info(self, session, entity, source, extid): """add type and source info for an eid into the system table""" raise NotImplementedError() - def delete_info(self, session, eid, etype, uri, extid): + def update_info(self, session, entity, need_fti_update): + """mark entity as being modified, fulltext reindex if needed""" + raise NotImplementedError() + + def delete_info(self, session, entity, uri, extid, attributes, relations): """delete system information on deletion of an entity by transfering record from the entities table to the deleted_entities table """ diff -r 9767cc516b4f -r 083b4d454192 server/sources/extlite.py --- a/server/sources/extlite.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/sources/extlite.py Mon Mar 01 11:26:14 2010 +0100 @@ -225,15 +225,15 @@ """update an entity in the source""" raise NotImplementedError() - def delete_entity(self, session, etype, eid): + def delete_entity(self, session, entity): """delete an entity from the source this is not deleting a file in the svn but deleting entities from the source. Main usage is to delete repository content when a Repository entity is deleted. """ - attrs = {SQL_PREFIX + 'eid': eid} - sql = self.sqladapter.sqlgen.delete(SQL_PREFIX + etype, attrs) + attrs = {'cw_eid': entity.eid} + sql = self.sqladapter.sqlgen.delete(SQL_PREFIX + entity.__regid__, attrs) self.doexec(session, sql, attrs) def local_add_relation(self, session, subject, rtype, object): diff -r 9767cc516b4f -r 083b4d454192 server/sources/ldapuser.py --- a/server/sources/ldapuser.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/sources/ldapuser.py Mon Mar 01 11:26:14 2010 +0100 @@ -476,7 +476,8 @@ if eid: self.warning('deleting ldap user with eid %s and dn %s', eid, base) - self.repo.delete_info(session, eid) + entity = session.entity_from_eid(eid, 'CWUser') + self.repo.delete_info(session, entity, self.uri, base) self._cache.pop(base, None) return [] ## except ldap.REFERRAL, e: @@ -554,7 +555,7 @@ """replace an entity in the source""" raise RepositoryError('this source is read only') - def delete_entity(self, session, etype, eid): + def delete_entity(self, session, entity): """delete an entity from the source""" raise RepositoryError('this source is read only') diff -r 9767cc516b4f -r 083b4d454192 server/sources/native.py --- a/server/sources/native.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/sources/native.py Mon Mar 01 11:26:14 2010 +0100 @@ -11,8 +11,11 @@ :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses """ +from __future__ import with_statement + __docformat__ = "restructuredtext en" +from pickle import loads, dumps from threading import Lock from datetime import datetime from base64 import b64decode, b64encode @@ -24,12 +27,15 @@ from logilab.common.shellutils import getlogin from logilab.database import get_db_helper -from cubicweb import UnknownEid, AuthenticationError, Binary, server +from cubicweb import UnknownEid, AuthenticationError, Binary, server, neg_role +from cubicweb import transaction as tx +from cubicweb.schema import VIRTUAL_RTYPES from cubicweb.cwconfig import CubicWebNoAppConfiguration from cubicweb.server import hook from cubicweb.server.utils import crypt_password from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn from cubicweb.server.rqlannotation import set_qdata +from cubicweb.server.session import hooks_control from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results from cubicweb.server.sources.rql2sql import SQLGenerator @@ -93,6 +99,35 @@ table, restr, attr) +def sql_or_clauses(sql, clauses): + select, restr = sql.split(' WHERE ', 1) + restrclauses = restr.split(' AND ') + for clause in clauses: + restrclauses.remove(clause) + if restrclauses: + restr = '%s AND (%s)' % (' AND '.join(restrclauses), + ' OR '.join(clauses)) + else: + restr = '(%s)' % ' OR '.join(clauses) + return '%s WHERE %s' % (select, restr) + + +class UndoException(Exception): + """something went wrong during undoing""" + + +def _undo_check_relation_target(tentity, rdef, role): + """check linked entity has not been redirected for this relation""" + card = rdef.role_cardinality(role) + if card in '?1' and tentity.related(rdef.rtype, role): + raise UndoException(tentity._cw._( + "Can't restore %(role)s relation %(rtype)s to entity %(eid)s which " + "is already linked using this relation.") + % {'role': neg_role(role), + 'rtype': rdef.rtype, + 'eid': tentity.eid}) + + class NativeSQLSource(SQLAdapterMixIn, AbstractSource): """adapter for source using the native cubicweb schema (see below) """ @@ -370,34 +405,57 @@ def add_entity(self, session, entity): """add a new entity to the source""" attrs = self.preprocess_entity(entity) - sql = self.sqlgen.insert(SQL_PREFIX + str(entity.e_schema), attrs) + sql = self.sqlgen.insert(SQL_PREFIX + entity.__regid__, attrs) self.doexec(session, sql, attrs) + if session.undoable_action('C', entity.__regid__): + self._record_tx_action(session, 'tx_entity_actions', 'C', + etype=entity.__regid__, eid=entity.eid) def update_entity(self, session, entity): """replace an entity in the source""" attrs = self.preprocess_entity(entity) - sql = self.sqlgen.update(SQL_PREFIX + str(entity.e_schema), attrs, - [SQL_PREFIX + 'eid']) + if session.undoable_action('U', entity.__regid__): + changes = self._save_attrs(session, entity, attrs) + self._record_tx_action(session, 'tx_entity_actions', 'U', + etype=entity.__regid__, eid=entity.eid, + changes=self._binary(dumps(changes))) + sql = self.sqlgen.update(SQL_PREFIX + entity.__regid__, attrs, + ['cw_eid']) self.doexec(session, sql, attrs) - def delete_entity(self, session, etype, eid): + def delete_entity(self, session, entity): """delete an entity from the source""" - attrs = {SQL_PREFIX + 'eid': eid} - sql = self.sqlgen.delete(SQL_PREFIX + etype, attrs) + if session.undoable_action('D', entity.__regid__): + attrs = [SQL_PREFIX + r.type + for r in entity.e_schema.subject_relations() + if (r.final or r.inlined) and not r in VIRTUAL_RTYPES] + changes = self._save_attrs(session, entity, attrs) + self._record_tx_action(session, 'tx_entity_actions', 'D', + etype=entity.__regid__, eid=entity.eid, + changes=self._binary(dumps(changes))) + attrs = {'cw_eid': entity.eid} + sql = self.sqlgen.delete(SQL_PREFIX + entity.__regid__, attrs) self.doexec(session, sql, attrs) - def add_relation(self, session, subject, rtype, object, inlined=False): + def _add_relation(self, session, subject, rtype, object, inlined=False): """add a relation to the source""" if inlined is False: attrs = {'eid_from': subject, 'eid_to': object} sql = self.sqlgen.insert('%s_relation' % rtype, attrs) else: # used by data import etype = session.describe(subject)[0] - attrs = {SQL_PREFIX + 'eid': subject, SQL_PREFIX + rtype: object} + attrs = {'cw_eid': subject, SQL_PREFIX + rtype: object} sql = self.sqlgen.update(SQL_PREFIX + etype, attrs, - [SQL_PREFIX + 'eid']) + ['cw_eid']) self.doexec(session, sql, attrs) + def add_relation(self, session, subject, rtype, object, inlined=False): + """add a relation to the source""" + self._add_relation(session, subject, rtype, object, inlined) + if session.undoable_action('A', rtype): + self._record_tx_action(session, 'tx_relation_actions', 'A', + eid_from=subject, rtype=rtype, eid_to=object) + def delete_relation(self, session, subject, rtype, object): """delete a relation from the source""" rschema = self.schema.rschema(rtype) @@ -411,6 +469,9 @@ attrs = {'eid_from': subject, 'eid_to': object} sql = self.sqlgen.delete('%s_relation' % rtype, attrs) self.doexec(session, sql, attrs) + if session.undoable_action('R', rtype): + self._record_tx_action(session, 'tx_relation_actions', 'R', + eid_from=subject, rtype=rtype, eid_to=object) def doexec(self, session, query, args=None, rollback=True): """Execute a query. @@ -467,6 +528,9 @@ # short cut to method requiring advanced db helper usage ################## + def binary_to_str(self, value): + return self.dbhelper.dbapi_module.binary_to_str(value) + def create_index(self, session, table, column, unique=False): cursor = LogCursor(session.pool[self.uri]) self.dbhelper.create_index(cursor, table, column, unique) @@ -481,7 +545,7 @@ """return a tuple (type, source, extid) for the entity with id """ sql = 'SELECT type, source, extid FROM entities WHERE eid=%s' % eid try: - res = session.system_sql(sql).fetchone() + res = self.doexec(session, sql).fetchone() except: assert session.pool, 'session has no pool set' raise UnknownEid(eid) @@ -496,9 +560,10 @@ def extid2eid(self, session, source, extid): """get eid from an external id. Return None if no record found.""" assert isinstance(extid, str) - cursor = session.system_sql('SELECT eid FROM entities WHERE ' - 'extid=%(x)s AND source=%(s)s', - {'x': b64encode(extid), 's': source.uri}) + cursor = self.doexec(session, + 'SELECT eid FROM entities ' + 'WHERE extid=%(x)s AND source=%(s)s', + {'x': b64encode(extid), 's': source.uri}) # XXX testing rowcount cause strange bug with sqlite, results are there # but rowcount is 0 #if cursor.rowcount > 0: @@ -529,7 +594,7 @@ finally: self._eid_creation_lock.release() - def add_info(self, session, entity, source, extid=None, complete=True): + def add_info(self, session, entity, source, extid, complete): """add type and source info for an eid into the system table""" # begin by inserting eid/type/source/extid into the entities table if extid is not None: @@ -537,7 +602,7 @@ extid = b64encode(extid) attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': extid, 'source': source.uri, 'mtime': datetime.now()} - session.system_sql(self.sqlgen.insert('entities', attrs), attrs) + self.doexec(session, self.sqlgen.insert('entities', attrs), attrs) # now we can update the full text index if self.do_fti and self.need_fti_indexation(entity.__regid__): if complete: @@ -545,26 +610,28 @@ FTIndexEntityOp(session, entity=entity) def update_info(self, session, entity, need_fti_update): + """mark entity as being modified, fulltext reindex if needed""" if self.do_fti and need_fti_update: # reindex the entity only if this query is updating at least # one indexable attribute FTIndexEntityOp(session, entity=entity) # update entities.mtime attrs = {'eid': entity.eid, 'mtime': datetime.now()} - session.system_sql(self.sqlgen.update('entities', attrs, ['eid']), attrs) + self.doexec(session, self.sqlgen.update('entities', attrs, ['eid']), attrs) - def delete_info(self, session, eid, etype, uri, extid): + def delete_info(self, session, entity, uri, extid): """delete system information on deletion of an entity by transfering record from the entities table to the deleted_entities table """ - attrs = {'eid': eid} - session.system_sql(self.sqlgen.delete('entities', attrs), attrs) + attrs = {'eid': entity.eid} + self.doexec(session, self.sqlgen.delete('entities', attrs), attrs) if extid is not None: assert isinstance(extid, str), type(extid) extid = b64encode(extid) - attrs = {'type': etype, 'eid': eid, 'extid': extid, - 'source': uri, 'dtime': datetime.now()} - session.system_sql(self.sqlgen.insert('deleted_entities', attrs), attrs) + attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': extid, + 'source': uri, 'dtime': datetime.now(), + } + self.doexec(session, self.sqlgen.insert('deleted_entities', attrs), attrs) def modified_entities(self, session, etypes, mtime): """return a 2-uple: @@ -575,13 +642,315 @@ deleted since the given timestamp """ modsql = _modified_sql('entities', etypes) - cursor = session.system_sql(modsql, {'time': mtime}) + cursor = self.doexec(session, modsql, {'time': mtime}) modentities = cursor.fetchall() delsql = _modified_sql('deleted_entities', etypes) - cursor = session.system_sql(delsql, {'time': mtime}) + cursor = self.doexec(session, delsql, {'time': mtime}) delentities = cursor.fetchall() return modentities, delentities + # undo support ############################################################# + + def undoable_transactions(self, session, ueid=None, **actionfilters): + """See :class:`cubicweb.dbapi.Connection.undoable_transactions`""" + # force filtering to session's user if not a manager + if not session.user.is_in_group('managers'): + ueid = session.user.eid + restr = {} + if ueid is not None: + restr['tx_user'] = ueid + sql = self.sqlgen.select('transactions', restr, ('tx_uuid', 'tx_time', 'tx_user')) + if actionfilters: + # we will need subqueries to filter transactions according to + # actions done + tearestr = {} # filters on the tx_entity_actions table + trarestr = {} # filters on the tx_relation_actions table + genrestr = {} # generic filters, appliyable to both table + # unless public explicitly set to false, we only consider public + # actions + if actionfilters.pop('public', True): + genrestr['txa_public'] = True + # put additional filters in trarestr and/or tearestr + for key, val in actionfilters.iteritems(): + if key == 'etype': + # filtering on etype implies filtering on entity actions + # only, and with no eid specified + assert actionfilters.get('action', 'C') in 'CUD' + assert not 'eid' in actionfilters + tearestr['etype'] = val + elif key == 'eid': + # eid filter may apply to 'eid' of tx_entity_actions or to + # 'eid_from' OR 'eid_to' of tx_relation_actions + if actionfilters.get('action', 'C') in 'CUD': + tearestr['eid'] = val + if actionfilters.get('action', 'A') in 'AR': + trarestr['eid_from'] = val + trarestr['eid_to'] = val + elif key == 'action': + if val in 'CUD': + tearestr['txa_action'] = val + else: + assert val in 'AR' + trarestr['txa_action'] = val + else: + raise AssertionError('unknow filter %s' % key) + assert trarestr or tearestr, "can't only filter on 'public'" + subqsqls = [] + # append subqueries to the original query, using EXISTS() + if trarestr or (genrestr and not tearestr): + trarestr.update(genrestr) + trasql = self.sqlgen.select('tx_relation_actions', trarestr, ('1',)) + if 'eid_from' in trarestr: + # replace AND by OR between eid_from/eid_to restriction + trasql = sql_or_clauses(trasql, ['eid_from = %(eid_from)s', + 'eid_to = %(eid_to)s']) + trasql += ' AND transactions.tx_uuid=tx_relation_actions.tx_uuid' + subqsqls.append('EXISTS(%s)' % trasql) + if tearestr or (genrestr and not trarestr): + tearestr.update(genrestr) + teasql = self.sqlgen.select('tx_entity_actions', tearestr, ('1',)) + teasql += ' AND transactions.tx_uuid=tx_entity_actions.tx_uuid' + subqsqls.append('EXISTS(%s)' % teasql) + if restr: + sql += ' AND %s' % ' OR '.join(subqsqls) + else: + sql += ' WHERE %s' % ' OR '.join(subqsqls) + restr.update(trarestr) + restr.update(tearestr) + # we want results ordered by transaction's time descendant + sql += ' ORDER BY tx_time DESC' + cu = self.doexec(session, sql, restr) + # turn results into transaction objects + return [tx.Transaction(*args) for args in cu.fetchall()] + + def tx_info(self, session, txuuid): + """See :class:`cubicweb.dbapi.Connection.transaction_info`""" + return tx.Transaction(txuuid, *self._tx_info(session, txuuid)) + + def tx_actions(self, session, txuuid, public): + """See :class:`cubicweb.dbapi.Connection.transaction_actions`""" + self._tx_info(session, txuuid) + restr = {'tx_uuid': txuuid} + if public: + restr['txa_public'] = True + sql = self.sqlgen.select('tx_entity_actions', restr, + ('txa_action', 'txa_public', 'txa_order', + 'etype', 'eid', 'changes')) + cu = self.doexec(session, sql, restr) + actions = [tx.EntityAction(a,p,o,et,e,c and loads(self.binary_to_str(c))) + for a,p,o,et,e,c in cu.fetchall()] + sql = self.sqlgen.select('tx_relation_actions', restr, + ('txa_action', 'txa_public', 'txa_order', + 'rtype', 'eid_from', 'eid_to')) + cu = self.doexec(session, sql, restr) + actions += [tx.RelationAction(*args) for args in cu.fetchall()] + return sorted(actions, key=lambda x: x.order) + + def undo_transaction(self, session, txuuid): + """See :class:`cubicweb.dbapi.Connection.undo_transaction`""" + # set mode so pool isn't released subsquently until commit/rollback + session.mode = 'write' + errors = [] + with hooks_control(session, session.HOOKS_DENY_ALL, 'integrity'): + for action in reversed(self.tx_actions(session, txuuid, False)): + undomethod = getattr(self, '_undo_%s' % action.action.lower()) + errors += undomethod(session, action) + # remove the transactions record + self.doexec(session, + "DELETE FROM transactions WHERE tx_uuid='%s'" % txuuid) + return errors + + def start_undoable_transaction(self, session, uuid): + """session callback to insert a transaction record in the transactions + table when some undoable transaction is started + """ + ueid = session.user.eid + attrs = {'tx_uuid': uuid, 'tx_user': ueid, 'tx_time': datetime.now()} + self.doexec(session, self.sqlgen.insert('transactions', attrs), attrs) + + def _save_attrs(self, session, entity, attrs): + """return a pickleable dictionary containing current values for given + attributes of the entity + """ + restr = {'cw_eid': entity.eid} + sql = self.sqlgen.select(SQL_PREFIX + entity.__regid__, restr, attrs) + cu = self.doexec(session, sql, restr) + values = dict(zip(attrs, cu.fetchone())) + # ensure backend specific binary are converted back to string + eschema = entity.e_schema + for column in attrs: + # [3:] remove 'cw_' prefix + attr = column[3:] + if not eschema.subjrels[attr].final: + continue + if eschema.destination(attr) in ('Password', 'Bytes'): + value = values[column] + if value is not None: + values[column] = self.binary_to_str(value) + return values + + def _record_tx_action(self, session, table, action, **kwargs): + """record a transaction action in the given table (either + 'tx_entity_actions' or 'tx_relation_action') + """ + kwargs['tx_uuid'] = session.transaction_uuid() + kwargs['txa_action'] = action + kwargs['txa_order'] = session.transaction_inc_action_counter() + kwargs['txa_public'] = session.running_dbapi_query + self.doexec(session, self.sqlgen.insert(table, kwargs), kwargs) + + def _tx_info(self, session, txuuid): + """return transaction's time and user of the transaction with the given uuid. + + raise `NoSuchTransaction` if there is no such transaction of if the + session's user isn't allowed to see it. + """ + restr = {'tx_uuid': txuuid} + sql = self.sqlgen.select('transactions', restr, ('tx_time', 'tx_user')) + cu = self.doexec(session, sql, restr) + try: + time, ueid = cu.fetchone() + except TypeError: + raise tx.NoSuchTransaction() + if not (session.user.is_in_group('managers') + or session.user.eid == ueid): + raise tx.NoSuchTransaction() + return time, ueid + + def _undo_d(self, session, action): + """undo an entity deletion""" + errors = [] + err = errors.append + eid = action.eid + etype = action.etype + _ = session._ + # get an entity instance + try: + entity = self.repo.vreg['etypes'].etype_class(etype)(session) + except Exception: + err("can't restore entity %s of type %s, type no more supported" + % (eid, etype)) + return errors + # check for schema changes, entities linked through inlined relation + # still exists, rewrap binary values + eschema = entity.e_schema + getrschema = eschema.subjrels + for column, value in action.changes.items(): + rtype = column[3:] # remove cw_ prefix + try: + rschema = getrschema[rtype] + except KeyError: + err(_("Can't restore relation %(rtype)s of entity %(eid)s, " + "this relation does not exists anymore in the schema.") + % {'rtype': rtype, 'eid': eid}) + if not rschema.final: + assert value is None + # try: + # tentity = session.entity_from_eid(eid) + # except UnknownEid: + # err(_("Can't restore %(role)s relation %(rtype)s to " + # "entity %(eid)s which doesn't exist anymore.") + # % {'role': _('subject'), + # 'rtype': _(rtype), + # 'eid': eid}) + # continue + # rdef = rdefs[(eschema, tentity.__regid__)] + # try: + # _undo_check_relation_target(tentity, rdef, 'object') + # except UndoException, ex: + # err(unicode(ex)) + # continue + # if rschema.inlined: + # entity[rtype] = value + # else: + # # restore relation where inlined changed since the deletion + # del action.changes[column] + # self._add_relation(session, subject, rtype, object) + # # set related cache + # session.update_rel_cache_add(eid, rtype, value, + # rschema.symmetric) + elif eschema.destination(rtype) in ('Bytes', 'Password'): + action.changes[column] = self._binary(value) + entity[rtype] = Binary(value) + elif isinstance(value, str): + entity[rtype] = unicode(value, session.encoding, 'replace') + else: + entity[rtype] = value + entity.set_eid(eid) + entity.edited_attributes = set(entity) + entity.check() + self.repo.hm.call_hooks('before_add_entity', session, entity=entity) + # restore the entity + action.changes['cw_eid'] = eid + sql = self.sqlgen.insert(SQL_PREFIX + etype, action.changes) + self.doexec(session, sql, action.changes) + # restore record in entities (will update fti if needed) + self.add_info(session, entity, self, None, True) + # remove record from deleted_entities + self.doexec(session, 'DELETE FROM deleted_entities WHERE eid=%s' % eid) + self.repo.hm.call_hooks('after_add_entity', session, entity=entity) + return errors + + def _undo_r(self, session, action): + """undo a relation removal""" + errors = [] + err = errors.append + _ = session._ + subj, rtype, obj = action.eid_from, action.rtype, action.eid_to + entities = [] + for role, eid in (('subject', subj), ('object', obj)): + try: + entities.append(session.entity_from_eid(eid)) + except UnknownEid: + err(_("Can't restore relation %(rtype)s, %(role)s entity %(eid)s" + " doesn't exist anymore.") + % {'role': _(role), + 'rtype': _(rtype), + 'eid': eid}) + if not len(entities) == 2: + return errors + sentity, oentity = entities + try: + rschema = self.schema.rschema(rtype) + rdef = rschema.rdefs[(sentity.__regid__, oentity.__regid__)] + except KeyError: + err(_("Can't restore relation %(rtype)s between %(subj)s and " + "%(obj)s, that relation does not exists anymore in the " + "schema.") + % {'rtype': rtype, + 'subj': subj, + 'obj': obj}) + else: + for role, entity in (('subject', sentity), + ('object', oentity)): + try: + _undo_check_relation_target(entity, rdef, role) + except UndoException, ex: + err(unicode(ex)) + continue + if not errors: + self.repo.hm.call_hooks('before_add_relation', session, + eidfrom=subj, rtype=rtype, eidto=obj) + # add relation in the database + self._add_relation(session, subj, rtype, obj, rschema.inlined) + # set related cache + session.update_rel_cache_add(subj, rtype, obj, rschema.symmetric) + self.repo.hm.call_hooks('after_add_relation', session, + eidfrom=subj, rtype=rtype, eidto=obj) + return errors + + def _undo_c(self, session, action): + """undo an entity creation""" + return ['undoing of entity creation not yet supported.'] + + def _undo_u(self, session, action): + """undo an entity update""" + return ['undoing of entity updating not yet supported.'] + + def _undo_a(self, session, action): + """undo a relation addition""" + return ['undoing of relation addition not yet supported.'] + # full text index handling ################################################# @cached @@ -650,7 +1019,7 @@ def sql_schema(driver): helper = get_db_helper(driver) - tstamp_col_type = helper.TYPE_MAPPING['Datetime'] + typemap = helper.TYPE_MAPPING schema = """ /* Create the repository's system database */ @@ -662,10 +1031,10 @@ source VARCHAR(64) NOT NULL, mtime %s NOT NULL, extid VARCHAR(256) -); -CREATE INDEX entities_type_idx ON entities(type); -CREATE INDEX entities_mtime_idx ON entities(mtime); -CREATE INDEX entities_extid_idx ON entities(extid); +);; +CREATE INDEX entities_type_idx ON entities(type);; +CREATE INDEX entities_mtime_idx ON entities(mtime);; +CREATE INDEX entities_extid_idx ON entities(extid);; CREATE TABLE deleted_entities ( eid INTEGER PRIMARY KEY NOT NULL, @@ -673,11 +1042,58 @@ source VARCHAR(64) NOT NULL, dtime %s NOT NULL, extid VARCHAR(256) -); -CREATE INDEX deleted_entities_type_idx ON deleted_entities(type); -CREATE INDEX deleted_entities_dtime_idx ON deleted_entities(dtime); -CREATE INDEX deleted_entities_extid_idx ON deleted_entities(extid); -""" % (helper.sql_create_sequence('entities_id_seq'), tstamp_col_type, tstamp_col_type) +);; +CREATE INDEX deleted_entities_type_idx ON deleted_entities(type);; +CREATE INDEX deleted_entities_dtime_idx ON deleted_entities(dtime);; +CREATE INDEX deleted_entities_extid_idx ON deleted_entities(extid);; + +CREATE TABLE transactions ( + tx_uuid CHAR(32) PRIMARY KEY NOT NULL, + tx_user INTEGER NOT NULL, + tx_time %s NOT NULL +);; +CREATE INDEX transactions_tx_user_idx ON transactions(tx_user);; + +CREATE TABLE tx_entity_actions ( + tx_uuid CHAR(32) REFERENCES transactions(tx_uuid) ON DELETE CASCADE, + txa_action CHAR(1) NOT NULL, + txa_public %s NOT NULL, + txa_order INTEGER, + eid INTEGER NOT NULL, + etype VARCHAR(64) NOT NULL, + changes %s +);; +CREATE INDEX tx_entity_actions_txa_action_idx ON tx_entity_actions(txa_action);; +CREATE INDEX tx_entity_actions_txa_public_idx ON tx_entity_actions(txa_public);; +CREATE INDEX tx_entity_actions_eid_idx ON tx_entity_actions(eid);; +CREATE INDEX tx_entity_actions_etype_idx ON tx_entity_actions(etype);; + +CREATE TABLE tx_relation_actions ( + tx_uuid CHAR(32) REFERENCES transactions(tx_uuid) ON DELETE CASCADE, + txa_action CHAR(1) NOT NULL, + txa_public %s NOT NULL, + txa_order INTEGER, + eid_from INTEGER NOT NULL, + eid_to INTEGER NOT NULL, + rtype VARCHAR(256) NOT NULL +);; +CREATE INDEX tx_relation_actions_txa_action_idx ON tx_relation_actions(txa_action);; +CREATE INDEX tx_relation_actions_txa_public_idx ON tx_relation_actions(txa_public);; +CREATE INDEX tx_relation_actions_eid_from_idx ON tx_relation_actions(eid_from);; +CREATE INDEX tx_relation_actions_eid_to_idx ON tx_relation_actions(eid_to);; +""" % (helper.sql_create_sequence('entities_id_seq').replace(';', ';;'), + typemap['Datetime'], typemap['Datetime'], typemap['Datetime'], + typemap['Boolean'], typemap['Bytes'], typemap['Boolean']) + if helper.backend_name == 'sqlite': + # sqlite support the ON DELETE CASCADE syntax but do nothing + schema += ''' +CREATE TRIGGER fkd_transactions +BEFORE DELETE ON transactions +FOR EACH ROW BEGIN + DELETE FROM tx_entity_actions WHERE tx_uuid=OLD.tx_uuid; + DELETE FROM tx_relation_actions WHERE tx_uuid=OLD.tx_uuid; +END;; +''' return schema @@ -687,18 +1103,19 @@ %s DROP TABLE entities; DROP TABLE deleted_entities; +DROP TABLE transactions; +DROP TABLE tx_entity_actions; +DROP TABLE tx_relation_actions; """ % helper.sql_drop_sequence('entities_id_seq') def grant_schema(user, set_owner=True): result = '' - if set_owner: - result = 'ALTER TABLE entities OWNER TO %s;\n' % user - result += 'ALTER TABLE deleted_entities OWNER TO %s;\n' % user - result += 'ALTER TABLE entities_id_seq OWNER TO %s;\n' % user - result += 'GRANT ALL ON entities TO %s;\n' % user - result += 'GRANT ALL ON deleted_entities TO %s;\n' % user - result += 'GRANT ALL ON entities_id_seq TO %s;\n' % user + for table in ('entities', 'deleted_entities', 'entities_id_seq', + 'transactions', 'tx_entity_actions', 'tx_relation_actions'): + if set_owner: + result = 'ALTER TABLE %s OWNER TO %s;\n' % (table, user) + result += 'GRANT ALL ON %s TO %s;\n' % (table, user) return result diff -r 9767cc516b4f -r 083b4d454192 server/sources/pyrorql.py --- a/server/sources/pyrorql.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/sources/pyrorql.py Mon Mar 01 11:26:14 2010 +0100 @@ -203,7 +203,8 @@ insert=False) # entity has been deleted from external repository but is not known here if eid is not None: - repo.delete_info(session, eid) + entity = session.entity_from_eid(eid, etype) + repo.delete_info(session, entity, self.uri, extid) except: self.exception('while updating %s with external id %s of source %s', etype, extid, self.uri) @@ -350,11 +351,11 @@ self._query_cache.clear() entity.clear_all_caches() - def delete_entity(self, session, etype, eid): + def delete_entity(self, session, entity): """delete an entity from the source""" cu = session.pool[self.uri] - cu.execute('DELETE %s X WHERE X eid %%(x)s' % etype, - {'x': self.eid2extid(eid, session)}, 'x') + cu.execute('DELETE %s X WHERE X eid %%(x)s' % entity.__regid__, + {'x': self.eid2extid(entity.eid, session)}, 'x') self._query_cache.clear() def add_relation(self, session, subject, rtype, object): diff -r 9767cc516b4f -r 083b4d454192 server/sqlutils.py --- a/server/sqlutils.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/sqlutils.py Mon Mar 01 11:26:14 2010 +0100 @@ -94,14 +94,15 @@ w('') dbhelper = db.get_db_helper(driver) if text_index: - w(dbhelper.sql_init_fti()) + w(dbhelper.sql_init_fti().replace(';', ';;')) w('') w(schema2sql(dbhelper, schema, prefix=SQL_PREFIX, - skip_entities=skip_entities, skip_relations=skip_relations)) + skip_entities=skip_entities, + skip_relations=skip_relations).replace(';', ';;')) if dbhelper.users_support and user: w('') w(sqlgrants(schema, driver, user, text_index, set_owner, - skip_relations, skip_entities)) + skip_relations, skip_entities).replace(';', ';;')) return '\n'.join(output) diff -r 9767cc516b4f -r 083b4d454192 server/test/unittest_repository.py --- a/server/test/unittest_repository.py Wed Mar 10 16:07:24 2010 +0100 +++ b/server/test/unittest_repository.py Mon Mar 01 11:26:14 2010 +0100 @@ -385,14 +385,14 @@ entity.eid = -1 entity.complete = lambda x: None self.session.set_pool() - self.repo.add_info(self.session, entity, self.repo.sources_by_uri['system']) + self.repo.add_info(self.session, entity, self.repo.system_source) cu = self.session.system_sql('SELECT * FROM entities WHERE eid = -1') data = cu.fetchall() self.assertIsInstance(data[0][3], datetime) data[0] = list(data[0]) data[0][3] = None self.assertEquals(tuplify(data), [(-1, 'Personne', 'system', None, None)]) - self.repo.delete_info(self.session, -1) + self.repo.delete_info(self.session, entity, 'system', None) #self.repo.commit() cu = self.session.system_sql('SELECT * FROM entities WHERE eid = -1') data = cu.fetchall() diff -r 9767cc516b4f -r 083b4d454192 server/test/unittest_undo.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/test/unittest_undo.py Mon Mar 01 11:26:14 2010 +0100 @@ -0,0 +1,206 @@ +""" + +:organization: Logilab +:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. +:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr +:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses +""" +from __future__ import with_statement + +from cubicweb import ValidationError +from cubicweb.devtools.testlib import CubicWebTC +from cubicweb.transaction import * + +class UndoableTransactionTC(CubicWebTC): + + def setup_database(self): + self.session.undo_actions = set('CUDAR') + self.toto = self.create_user('toto', password='toto', groups=('users',), + commit=False) + self.txuuid = self.commit() + + def tearDown(self): + self.restore_connection() + self.session.undo_support = set() + super(UndoableTransactionTC, self).tearDown() + + def test_undo_api(self): + self.failUnless(self.txuuid) + # test transaction api + self.assertRaises(NoSuchTransaction, + self.cnx.transaction_info, 'hop') + self.assertRaises(NoSuchTransaction, + self.cnx.transaction_actions, 'hop') + self.assertRaises(NoSuchTransaction, + self.cnx.undo_transaction, 'hop') + txinfo = self.cnx.transaction_info(self.txuuid) + self.failUnless(txinfo.datetime) + self.assertEquals(txinfo.user_eid, self.session.user.eid) + self.assertEquals(txinfo.user().login, 'admin') + actions = txinfo.actions_list() + self.assertEquals(len(actions), 2) + actions = txinfo.actions_list(public=False) + self.assertEquals(len(actions), 6) + a1 = actions[0] + self.assertEquals(a1.action, 'C') + self.assertEquals(a1.eid, self.toto.eid) + self.assertEquals(a1.etype,'CWUser') + self.assertEquals(a1.changes, None) + self.assertEquals(a1.public, True) + self.assertEquals(a1.order, 1) + a4 = actions[3] + self.assertEquals(a4.action, 'A') + self.assertEquals(a4.rtype, 'in_group') + self.assertEquals(a4.eid_from, self.toto.eid) + self.assertEquals(a4.eid_to, self.toto.in_group[0].eid) + self.assertEquals(a4.order, 4) + for i, rtype in ((1, 'owned_by'), (2, 'owned_by'), + (4, 'created_by'), (5, 'in_state')): + a = actions[i] + self.assertEquals(a.action, 'A') + self.assertEquals(a.eid_from, self.toto.eid) + self.assertEquals(a.rtype, rtype) + self.assertEquals(a.order, i+1) + # test undoable_transactions + txs = self.cnx.undoable_transactions() + self.assertEquals(len(txs), 1) + self.assertEquals(txs[0].uuid, self.txuuid) + # test transaction_info / undoable_transactions security + cnx = self.login('anon') + self.assertRaises(NoSuchTransaction, + cnx.transaction_info, self.txuuid) + self.assertRaises(NoSuchTransaction, + cnx.transaction_actions, self.txuuid) + self.assertRaises(NoSuchTransaction, + cnx.undo_transaction, self.txuuid) + txs = cnx.undoable_transactions() + self.assertEquals(len(txs), 0) + + def test_undoable_transactions(self): + toto = self.toto + e = self.session.create_entity('EmailAddress', + address=u'toto@logilab.org', + reverse_use_email=toto) + txuuid1 = self.commit() + toto.delete() + txuuid2 = self.commit() + undoable_transactions = self.cnx.undoable_transactions + txs = undoable_transactions(action='D') + self.assertEquals(len(txs), 1, txs) + self.assertEquals(txs[0].uuid, txuuid2) + txs = undoable_transactions(action='C') + self.assertEquals(len(txs), 2, txs) + self.assertEquals(txs[0].uuid, txuuid1) + self.assertEquals(txs[1].uuid, self.txuuid) + txs = undoable_transactions(eid=toto.eid) + self.assertEquals(len(txs), 3) + self.assertEquals(txs[0].uuid, txuuid2) + self.assertEquals(txs[1].uuid, txuuid1) + self.assertEquals(txs[2].uuid, self.txuuid) + txs = undoable_transactions(etype='CWUser') + self.assertEquals(len(txs), 2) + txs = undoable_transactions(etype='CWUser', action='C') + self.assertEquals(len(txs), 1) + self.assertEquals(txs[0].uuid, self.txuuid) + txs = undoable_transactions(etype='EmailAddress', action='D') + self.assertEquals(len(txs), 0) + txs = undoable_transactions(etype='EmailAddress', action='D', + public=False) + self.assertEquals(len(txs), 1) + self.assertEquals(txs[0].uuid, txuuid2) + txs = undoable_transactions(eid=toto.eid, action='R', public=False) + self.assertEquals(len(txs), 1) + self.assertEquals(txs[0].uuid, txuuid2) + + def test_undo_deletion_base(self): + toto = self.toto + e = self.session.create_entity('EmailAddress', + address=u'toto@logilab.org', + reverse_use_email=toto) + # entity with inlined relation + p = self.session.create_entity('CWProperty', + pkey=u'ui.default-text-format', + value=u'text/rest', + for_user=toto) + self.commit() + txs = self.cnx.undoable_transactions() + self.assertEquals(len(txs), 2) + toto.delete() + txuuid = self.commit() + actions = self.cnx.transaction_info(txuuid).actions_list() + self.assertEquals(len(actions), 1) + toto.clear_all_caches() + e.clear_all_caches() + errors = self.cnx.undo_transaction(txuuid) + undotxuuid = self.commit() + self.assertEquals(undotxuuid, None) # undo not undoable + self.assertEquals(errors, []) + self.failUnless(self.execute('Any X WHERE X eid %(x)s', {'x': toto.eid}, 'x')) + self.failUnless(self.execute('Any X WHERE X eid %(x)s', {'x': e.eid}, 'x')) + self.failUnless(self.execute('Any X WHERE X has_text "toto@logilab"')) + self.assertEquals(toto.state, 'activated') + self.assertEquals(toto.get_email(), 'toto@logilab.org') + self.assertEquals([(p.pkey, p.value) for p in toto.reverse_for_user], + [('ui.default-text-format', 'text/rest')]) + self.assertEquals([g.name for g in toto.in_group], + ['users']) + self.assertEquals([et.name for et in toto.related('is', entities=True)], + ['CWUser']) + self.assertEquals([et.name for et in toto.is_instance_of], + ['CWUser']) + # undoing shouldn't be visble in undoable transaction, and the undoed + # transaction should be removed + txs = self.cnx.undoable_transactions() + self.assertEquals(len(txs), 2) + self.assertRaises(NoSuchTransaction, + self.cnx.transaction_info, txuuid) + # also check transaction actions have been properly deleted + cu = self.session.system_sql( + "SELECT * from tx_entity_actions WHERE tx_uuid='%s'" % txuuid) + self.failIf(cu.fetchall()) + cu = self.session.system_sql( + "SELECT * from tx_relation_actions WHERE tx_uuid='%s'" % txuuid) + self.failIf(cu.fetchall()) + # the final test: check we can login with the previously deleted user + self.login('toto') + + def test_undo_deletion_integrity_1(self): + session = self.session + # 'Personne fiche Card with' '??' cardinality + c = session.create_entity('Card', title=u'hop', content=u'hop') + p = session.create_entity('Personne', nom=u'louis', fiche=c) + self.commit() + c.delete() + txuuid = self.commit() + c2 = session.create_entity('Card', title=u'hip', content=u'hip') + p.set_relations(fiche=c2) + self.commit() + errors = self.cnx.undo_transaction(txuuid) + self.commit() + p.clear_all_caches() + self.assertEquals(p.fiche[0].eid, c2.eid) + self.assertEquals(len(errors), 1) + self.assertEquals(errors[0], + "Can't restore object relation fiche to entity " + "%s which is already linked using this relation." % p.eid) + + def test_undo_deletion_integrity_2(self): + # test validation error raised if we can't restore a required relation + session = self.session + g = session.create_entity('CWGroup', name=u'staff') + session.execute('DELETE U in_group G WHERE U eid %(x)s', {'x': self.toto.eid}) + self.toto.set_relations(in_group=g) + self.commit() + self.toto.delete() + txuuid = self.commit() + g.delete() + self.commit() + errors = self.cnx.undo_transaction(txuuid) + self.assertRaises(ValidationError, self.commit) + + def test_undo_creation(self): + # XXX what about relation / composite entities which have been created + # afterwhile and linked to the undoed addition ? + self.skip('not implemented') + + # test implicit 'replacement' of an inlined relation diff -r 9767cc516b4f -r 083b4d454192 test/unittest_dbapi.py --- a/test/unittest_dbapi.py Wed Mar 10 16:07:24 2010 +0100 +++ b/test/unittest_dbapi.py Mon Mar 01 11:26:14 2010 +0100 @@ -5,11 +5,13 @@ :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses """ +from __future__ import with_statement +from copy import copy + from cubicweb import ConnectionError from cubicweb.dbapi import ProgrammingError from cubicweb.devtools.testlib import CubicWebTC - class DBAPITC(CubicWebTC): def test_public_repo_api(self): @@ -68,6 +70,7 @@ self.assertRaises(ProgrammingError, cnx.set_shared_data, 'data', 0) self.assertRaises(ProgrammingError, cnx.get_shared_data, 'data') + if __name__ == '__main__': from logilab.common.testlib import unittest_main unittest_main() diff -r 9767cc516b4f -r 083b4d454192 transaction.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/transaction.py Mon Mar 01 11:26:14 2010 +0100 @@ -0,0 +1,96 @@ +"""undoable transaction objects. + + +This module is in the cubicweb package and not in cubicweb.server because those +objects should be accessible to client through pyro, where the cubicweb.server +package may not be installed. + +:organization: Logilab +:copyright: 2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. +:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr +:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses +""" +__docformat__ = "restructuredtext en" +_ = unicode + +from cubicweb import RepositoryError + + +ACTION_LABELS = { + 'C': _('entity creation'), + 'U': _('entity update'), + 'D': _('entity deletion'), + 'A': _('relation add'), + 'R': _('relation removal'), + } + + +class NoSuchTransaction(RepositoryError): + pass + + +class Transaction(object): + """an undoable transaction""" + + def __init__(self, uuid, time, ueid): + self.uuid = uuid + self.datetime = time + self.user_eid = ueid + # should be set by the dbapi connection + self.req = None + + def __repr__(self): + return '' % ( + self.uuid, self.user_eid, self.datetime) + + def user(self): + """return the user entity which has done the transaction, + none if not found. + """ + return self.req.execute('Any X WHERE X eid %(x)s', + {'x': self.user_eid}, 'x').get_entity(0, 0) + + def actions_list(self, public=True): + """return an ordered list of action effectued during that transaction + + if public is true, return only 'public' action, eg not ones triggered + under the cover by hooks. + """ + return self.req.cnx.transaction_actions(self.uuid, public) + + +class AbstractAction(object): + def __init__(self, action, public, order): + self.action = action + self.public = public + self.order = order + + @property + def label(self): + return ACTION_LABELS[self.action] + + +class EntityAction(AbstractAction): + def __init__(self, action, public, order, etype, eid, changes): + AbstractAction.__init__(self, action, public, order) + self.etype = etype + self.eid = eid + self.changes = changes + + def __repr__(self): + return '<%s: %s %s (%s)>' % ( + self.label, self.eid, self.changes, + self.public and 'dbapi' or 'hook') + + +class RelationAction(AbstractAction): + def __init__(self, action, public, order, rtype, eidfrom, eidto): + AbstractAction.__init__(self, action, public, order) + self.rtype = rtype + self.eid_from = eidfrom + self.eid_to = eidto + + def __repr__(self): + return '<%s: %s %s %s (%s)>' % ( + self.label, self.eid_from, self.rtype, self.eid_to, + self.public and 'dbapi' or 'hook') diff -r 9767cc516b4f -r 083b4d454192 web/application.py --- a/web/application.py Wed Mar 10 16:07:24 2010 +0100 +++ b/web/application.py Mon Mar 01 11:26:14 2010 +0100 @@ -342,7 +342,11 @@ # redirect is raised by edit controller when everything went fine, # so try to commit try: - req.cnx.commit() + txuuid = req.cnx.commit() + if txuuid is not None: + msg = u'[%s]' %( + req.build_url('undo', txuuid=txuuid), req._('undo')) + req.append_to_redirect_message(msg) except ValidationError, ex: self.validation_error_handler(req, ex) except Unauthorized, ex: diff -r 9767cc516b4f -r 083b4d454192 web/views/basecontrollers.py --- a/web/views/basecontrollers.py Wed Mar 10 16:07:24 2010 +0100 +++ b/web/views/basecontrollers.py Mon Mar 01 11:26:14 2010 +0100 @@ -605,3 +605,27 @@ url = self._cw.build_url(__message=self._cw._('bug report sent')) raise Redirect(url) + +class UndoController(SendMailController): + __regid__ = 'undo' + __select__ = authenticated_user() & match_form_params('txuuid') + + def publish(self, rset=None): + txuuid = self._cw.form['txuuid'] + errors = self._cw.cnx.undo_transaction(txuuid) + if errors: + self.w(self._cw._('some errors occured:')) + self.wview('pyvalist', pyvalue=errors) + else: + self.redirect() + + def redirect(self): + req = self._cw + breadcrumbs = req.get_session_data('breadcrumbs', None) + if breadcrumbs is not None and len(breadcrumbs) > 1: + url = req.rebuild_url(breadcrumbs[-2], + __message=req._('transaction undoed')) + else: + url = req.build_url(__message=req._('transaction undoed')) + raise Redirect(url) +