dataimport.py
branchstable
changeset 6136 79da6f969b15
parent 6133 6f3eabbbdf2e
child 6149 57e956441ca4
child 6169 55378e1bab1b
--- a/dataimport.py	Mon Aug 23 12:49:28 2010 +0200
+++ b/dataimport.py	Tue Aug 24 11:57:45 2010 +0200
@@ -124,13 +124,15 @@
     for row in it:
         yield [item.decode(encoding) for item in row]
 
-def commit_every(nbit, store, it):
-    for i, x in enumerate(it):
-        yield x
-        if nbit is not None and i % nbit:
-            store.commit()
-    if nbit is not None:
-        store.commit()
+def callfunc_every(func, number, iterable):
+    """yield items of `iterable` one by one and call function `func`
+    every `number` iterations. Always call function `func` at the end.
+    """
+    for idx, item in enumerate(iterable):
+        yield item
+        if idx % number:
+            func()
+    func()
 
 def lazytable(reader):
     """The first row is taken to be the header of the table and
@@ -229,7 +231,7 @@
     return None
 
 def required(value):
-    """raise ValueError is value is empty
+    """raise ValueError if value is empty
 
     This check should be often found in last position in the chain.
     """
@@ -570,7 +572,10 @@
 
     def iter_and_commit(self, datakey):
         """iter rows, triggering commit every self.commitevery iterations"""
-        return commit_every(self.commitevery, self.store, self.get_data(datakey))
+        if self.commitevery is None:
+            return self.get_data(datakey)
+        else:
+            return callfunc_every(self.commitevery, self.store.commit, self.get_data(datakey))