# HG changeset patch
# User Alexandre Fayolle <alexandre.fayolle@logilab.fr>
# Date 1303396435 -7200
# Node ID c419c2d0d13e2c74bd7dfd49e26c9d00c3673a85
# Parent  9303fd71c2eeef7e83d87a149c8ec71db42d4322
add a new method iter_process_result which does the same as proces_result but is a generator (closes #1625374)

process_result is reimplemented using the new method, and the two helper
methods are turned into generators.

These generators use cursor.fetchmany instead of cursor.fetchall after setting
cursor.arraysize to 100. This means that the whole result set should never
loaded in memory when using the iter_process_result method. This is used in the
"portable" database dump implementation when we typically to 'SELECT * FROM
table', but could probably be used too in other parts of cubicweb.

diff -r 9303fd71c2ee -r c419c2d0d13e server/sqlutils.py
--- a/server/sqlutils.py	Mon May 09 14:35:10 2011 +0200
+++ b/server/sqlutils.py	Thu Apr 21 16:33:55 2011 +0200
@@ -204,6 +204,12 @@
     def process_result(self, cursor, column_callbacks=None, session=None):
         """return a list of CubicWeb compliant values from data in the given cursor
         """
+        return list(self.iter_process_result(cursor, column_callbacks, session))
+
+    def iter_process_result(self, cursor, column_callbacks=None, session=None):
+        """return a iterator on tuples of CubicWeb compliant values from data
+        in the given cursor
+        """
         # use two different implementations to avoid paying the price of
         # callback lookup for each *cell* in results when there is nothing to
         # lookup
@@ -219,16 +225,19 @@
         process_value = self._process_value
         binary = Binary
         # /end
-        results = cursor.fetchall()
-        for i, line in enumerate(results):
-            result = []
-            for col, value in enumerate(line):
-                if value is None:
-                    result.append(value)
-                    continue
-                result.append(process_value(value, descr[col], encoding, binary))
-            results[i] = result
-        return results
+        cursor.arraysize = 100
+        while True:
+            results = cursor.fetchmany()
+            if not results:
+                break
+            for line in results:
+                result = []
+                for col, value in enumerate(line):
+                    if value is None:
+                        result.append(value)
+                        continue
+                    result.append(process_value(value, descr[col], encoding, binary))
+                yield result
 
     def _cb_process_result(self, cursor, column_callbacks, session):
         # begin bind to locals for optimization
@@ -237,22 +246,25 @@
         process_value = self._process_value
         binary = Binary
         # /end
-        results = cursor.fetchall()
-        for i, line in enumerate(results):
-            result = []
-            for col, value in enumerate(line):
-                if value is None:
+        cursor.arraysize = 100
+        while True:
+            results = cursor.fetchmany()
+            if not results:
+                break
+            for line in results:
+                result = []
+                for col, value in enumerate(line):
+                    if value is None:
+                        result.append(value)
+                        continue
+                    cbstack = column_callbacks.get(col, None)
+                    if cbstack is None:
+                        value = process_value(value, descr[col], encoding, binary)
+                    else:
+                        for cb in cbstack:
+                            value = cb(self, session, value)
                     result.append(value)
-                    continue
-                cbstack = column_callbacks.get(col, None)
-                if cbstack is None:
-                    value = process_value(value, descr[col], encoding, binary)
-                else:
-                    for cb in cbstack:
-                        value = cb(self, session, value)
-                result.append(value)
-            results[i] = result
-        return results
+                yield result
 
     def preprocess_entity(self, entity):
         """return a dictionary to use as extra argument to cursor.execute