[migration] Lesser memory consumption of cwuri fix + actually commit every 100 processed entities oldstable
authorSylvain Thénault <sylvain.thenault@logilab.fr>
Thu, 24 Mar 2011 13:30:14 +0100
brancholdstable
changeset 7106 63f0bc280354
parent 7105 aefd1776122f
child 7107 5ea4bf53eff2
child 7178 a62f24e1497e
[migration] Lesser memory consumption of cwuri fix + actually commit every 100 processed entities Don't use the ORM as added value is poor while all intentiated object are keept by rset cache.
misc/migration/3.10.9_Any.py
--- a/misc/migration/3.10.9_Any.py	Thu Mar 24 13:25:57 2011 +0100
+++ b/misc/migration/3.10.9_Any.py	Thu Mar 24 13:30:14 2011 +0100
@@ -16,9 +16,10 @@
     enabled = interactive_mode
     with progress(title=title, nbops=nbops, size=30, enabled=enabled) as pb:
         with hooks_control(session, session.HOOKS_DENY_ALL, 'integrity'):
-            for i,  e in enumerate(rset.entities()):
-                e.set_attributes(cwuri=e.cwuri.replace('/eid', ''))
-                if i % 100: # commit every 100 entities to limit memory consumption
+            for i,  row in enumerate(rset):
+                data = {'eid': row[0], 'cwuri': row[1].replace(u'/eid', u'')}
+                rql('SET X cwuri %(cwuri)s WHERE X eid %(eid)s', data)
+                if not i % 100: # commit every 100 entities to limit memory consumption
                     pb.text = "%i committed" % i
                     commit(ask_confirm=False)
                 pb.update()