add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)
process_result is reimplemented using the new method, and the two helper
methods are turned into generators.
These generators use cursor.fetchmany instead of cursor.fetchall after setting
cursor.arraysize to 100. This means that the whole result set should never
loaded in memory when using the iter_process_result method. This is used in the
"portable" database dump implementation when we typically to 'SELECT * FROM
table', but could probably be used too in other parts of cubicweb.
--- a/server/sqlutils.py Mon May 09 14:35:10 2011 +0200
+++ b/server/sqlutils.py Thu Apr 21 16:33:55 2011 +0200
@@ -204,6 +204,12 @@
def process_result(self, cursor, column_callbacks=None, session=None):
"""return a list of CubicWeb compliant values from data in the given cursor
"""
+ return list(self.iter_process_result(cursor, column_callbacks, session))
+
+ def iter_process_result(self, cursor, column_callbacks=None, session=None):
+ """return a iterator on tuples of CubicWeb compliant values from data
+ in the given cursor
+ """
# use two different implementations to avoid paying the price of
# callback lookup for each *cell* in results when there is nothing to
# lookup
@@ -219,16 +225,19 @@
process_value = self._process_value
binary = Binary
# /end
- results = cursor.fetchall()
- for i, line in enumerate(results):
- result = []
- for col, value in enumerate(line):
- if value is None:
- result.append(value)
- continue
- result.append(process_value(value, descr[col], encoding, binary))
- results[i] = result
- return results
+ cursor.arraysize = 100
+ while True:
+ results = cursor.fetchmany()
+ if not results:
+ break
+ for line in results:
+ result = []
+ for col, value in enumerate(line):
+ if value is None:
+ result.append(value)
+ continue
+ result.append(process_value(value, descr[col], encoding, binary))
+ yield result
def _cb_process_result(self, cursor, column_callbacks, session):
# begin bind to locals for optimization
@@ -237,22 +246,25 @@
process_value = self._process_value
binary = Binary
# /end
- results = cursor.fetchall()
- for i, line in enumerate(results):
- result = []
- for col, value in enumerate(line):
- if value is None:
+ cursor.arraysize = 100
+ while True:
+ results = cursor.fetchmany()
+ if not results:
+ break
+ for line in results:
+ result = []
+ for col, value in enumerate(line):
+ if value is None:
+ result.append(value)
+ continue
+ cbstack = column_callbacks.get(col, None)
+ if cbstack is None:
+ value = process_value(value, descr[col], encoding, binary)
+ else:
+ for cb in cbstack:
+ value = cb(self, session, value)
result.append(value)
- continue
- cbstack = column_callbacks.get(col, None)
- if cbstack is None:
- value = process_value(value, descr[col], encoding, binary)
- else:
- for cb in cbstack:
- value = cb(self, session, value)
- result.append(value)
- results[i] = result
- return results
+ yield result
def preprocess_entity(self, entity):
"""return a dictionary to use as extra argument to cursor.execute