# HG changeset patch # User Alexandre Fayolle # Date 1301660755 -7200 # Node ID 62561ea082d23d2e19ec1ab2b8e5d37324074a58 # Parent e3f69df8dac75e1e0c033fded02d0eb630baf074# Parent 923013173031df03cefb9947e2e3ffacd4daf848 merge stable to default diff -r e3f69df8dac7 -r 62561ea082d2 dataimport.py --- a/dataimport.py Fri Apr 01 08:20:25 2011 +0200 +++ b/dataimport.py Fri Apr 01 14:25:55 2011 +0200 @@ -19,12 +19,11 @@ """This module provides tools to import tabular data. - Example of use (run this with `cubicweb-ctl shell instance import-script.py`): .. sourcecode:: python - from cubicweb.devtools.dataimport import * + from cubicweb.dataimport import * # define data generators GENERATORS = [] @@ -36,12 +35,11 @@ def gen_users(ctl): for row in ctl.iter_and_commit('utilisateurs'): entity = mk_entity(row, USERS) - entity['upassword'] = u'motdepasse' + entity['upassword'] = 'motdepasse' ctl.check('login', entity['login'], None) - ctl.store.add('CWUser', entity) - email = {'address': row['email']} - ctl.store.add('EmailAddress', email) - ctl.store.relate(entity['eid'], 'use_email', email['eid']) + entity = ctl.store.create_entity('CWUser', **entity) + email = ctl.store.create_entity('EmailAddress', address=row['email']) + ctl.store.relate(entity.eid, 'use_email', email.eid) ctl.store.rql('SET U in_group G WHERE G name "users", U eid %(x)s', {'x':entity['eid']}) CHK = [('login', check_doubles, 'Utilisateurs Login', @@ -74,15 +72,18 @@ import os.path as osp from StringIO import StringIO from copy import copy +from datetime import datetime -from logilab.common import shellutils +from logilab.common import shellutils, attrdict from logilab.common.date import strptime from logilab.common.decorators import cached from logilab.common.deprecation import deprecated +from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES from cubicweb.server.utils import eschema_eid from cubicweb.server.edition import EditedEntity + def count_lines(stream_or_filename): if isinstance(stream_or_filename, basestring): f = open(stream_or_filename) @@ -145,6 +146,19 @@ for row in reader: yield dict(zip(header, row)) +def lazydbtable(cu, table, headers): + """return an iterator on rows of a sql table. On each row, fetch columns + defined in headers and return values as a dictionary. + + >>> data = lazydbtable(cu, 'experimentation', ('id', 'nickname', 'gps')) + """ + cu.execute('SELECT %s FROM %s' % (','.join(headers), table,)) + while True: + row = cu.fetchone() + if row is None: + break + yield dict(zip(headers, row)) + def mk_entity(row, map): """Return a dict made from sanitized mapped values. @@ -174,7 +188,6 @@ raise ValueError('error with %r field: %s' % (src, err)) return res - # user interactions ############################################################ def tell(msg): @@ -287,11 +300,9 @@ But it will not enforce the constraints of the schema and hence will miss some problems >>> store = ObjectStore() - >>> user = {'login': 'johndoe'} - >>> store.add('CWUser', user) - >>> group = {'name': 'unknown'} - >>> store.add('CWUser', group) - >>> store.relate(user['eid'], 'in_group', group['eid']) + >>> user = store.create_entity('CWUser', login=u'johndoe') + >>> group = store.create_entity('CWUser', name=u'unknown') + >>> store.relate(user.eid, 'in_group', group.eid) """ def __init__(self): self.items = [] @@ -307,7 +318,8 @@ return len(self.items) - 1 def create_entity(self, etype, **data): - data['eid'] = eid = self._put(etype, data) + data = attrdict(data) + data['eid'] = eid = self._put(etype, data) self.eids[eid] = data self.types.setdefault(etype, []).append(eid) return data @@ -594,11 +606,6 @@ self.get_data(datakey)) - -from datetime import datetime -from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES - - class NoHookRQLObjectStore(RQLObjectStore): """ObjectStore that works with an actual RQL repository (production mode)""" _rql = None # bw compat diff -r e3f69df8dac7 -r 62561ea082d2 server/hook.py --- a/server/hook.py Fri Apr 01 08:20:25 2011 +0200 +++ b/server/hook.py Fri Apr 01 14:25:55 2011 +0200 @@ -93,7 +93,7 @@ unset), specific events are issued and the Hooks matching these events are called. -You can get the event that triggered a hook by accessing its :attr:event +You can get the event that triggered a hook by accessing its `event` attribute. .. _`dataflow`: http://en.wikipedia.org/wiki/Dataflow @@ -105,68 +105,64 @@ When called for one of these events, hook will have an `entity` attribute containing the entity instance. -* `before_add_entity`, `before_update_entity`: +- `before_add_entity`, `before_update_entity`: - - on those events, you can check what attributes of the entity are modified in - `entity.cw_edited` (by definition the database is not yet updated in a before - event) + On those events, you can access the modified attributes of the entity using + the `entity.cw_edited` dictionnary. The values can be modified and the old + values can be retrieved. + + If you modify the `entity.cw_edited` dictionnary in the hook, that is before + the database operations take place, you will avoid the need to process a whole + new rql query and the underlying backend query (eg usually sql) will contain + the modified data. For example: - - you are allowed to further modify the entity before database - operations, using the dictionary notation on `cw_edited`. By doing - this, you'll avoid the need for a whole new rql query processing, - the only difference is that the underlying backend query (eg - usually sql) will contains the additional data. For example: + .. sourcecode:: python - .. sourcecode:: python + self.entity.cw_edited['age'] = 42 - self.entity.set_attributes(age=42) + will modify the age before it is written to the backend storage. - will set the `age` attribute of the entity to 42. But to do so, it will - generate a rql query that will have to be processed, then trigger some - hooks, and so one (potentially leading to infinite hook loops or such - awkward situations..) You can avoid this by doing the modification that way: + Similarly, removing an attribute from `cw_edited` will cancel its + modification: - .. sourcecode:: python + .. sourcecode:: python - self.entity.cw_edited['age'] = 42 + del self.entity.cw_edited['age'] - Here the attribute will simply be edited in the same query that the - one that triggered the hook. + On a `before_update_entity` event, you can access the old and new values: - Similarly, removing an attribute from `cw_edited` will cancel its - modification. + .. sourcecode:: python - - on `before_update_entity` event, you can access to old and new values in - this hook, by using `entity.cw_edited.oldnewvalue(attr)` + old, new = entity.cw_edited.oldnewvalue('age') + +- `after_add_entity`, `after_update_entity` + On those events, you can get the list of attributes that were modified using + the `entity.cw_edited` dictionnary, but you can not modify it or get the old + value of an attribute. -* `after_add_entity`, `after_update_entity` +- `before_delete_entity`, `after_delete_entity` - - on those events, you can still check what attributes of the entity are - modified in `entity.cw_edited` but you can't get anymore the old value, nor - modify it. + On those events, the entity has no `cw_edited` dictionnary. -* `before_delete_entity`, `after_delete_entity` - - - on those events, the entity has no `cw_edited` set. - +.. note:: `self.entity.set_attributes(age=42)` will set the `age` attribute to + 42. But to do so, it will generate a rql query that will have to be processed, + hence may trigger some hooks, etc. This could lead to infinitely looping hooks. Relation modification related events ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When called for one of these events, hook will have `eidfrom`, `rtype`, `eidto` -attributes containing respectivly the eid of the subject entity, the relation +attributes containing respectively the eid of the subject entity, the relation type and the eid of the object entity. * `before_add_relation`, `before_delete_relation` - - on those events, you can still get original relation by issuing a rql query + On those events, you can still get the original relation by issuing a rql query. * `after_add_relation`, `after_delete_relation` -This is an occasion to remind us that relations support the add / delete -operation, but no update. - +Take note that relations can be added or deleted, but not updated. Non data events ~~~~~~~~~~~~~~~