[server] use lgd.process_cursor to optimize processing of large resultsets
update lgdb dependency to ensure we've a fix necessary for proper behaviour of the previous changeset
--- a/__pkginfo__.py Wed Dec 21 18:04:07 2011 +0100
+++ b/__pkginfo__.py Wed Dec 21 18:26:35 2011 +0100
@@ -52,7 +52,7 @@
'Twisted': '',
# XXX graphviz
# server dependencies
- 'logilab-database': '>= 1.8.1',
+ 'logilab-database': '>= 1.8.2',
'pysqlite': '>= 2.5.5', # XXX install pysqlite2
}
--- a/debian/control Wed Dec 21 18:04:07 2011 +0100
+++ b/debian/control Wed Dec 21 18:26:35 2011 +0100
@@ -35,7 +35,7 @@
Conflicts: cubicweb-multisources
Replaces: cubicweb-multisources
Provides: cubicweb-multisources
-Depends: ${misc:Depends}, ${python:Depends}, cubicweb-common (= ${source:Version}), cubicweb-ctl (= ${source:Version}), python-logilab-database (>= 1.8.1), cubicweb-postgresql-support | cubicweb-mysql-support | python-pysqlite2
+Depends: ${misc:Depends}, ${python:Depends}, cubicweb-common (= ${source:Version}), cubicweb-ctl (= ${source:Version}), python-logilab-database (>= 1.8.2), cubicweb-postgresql-support | cubicweb-mysql-support | python-pysqlite2
Recommends: pyro (<< 4.0.0), cubicweb-documentation (= ${source:Version})
Description: server part of the CubicWeb framework
CubicWeb is a semantic web application framework.
--- a/server/sqlutils.py Wed Dec 21 18:04:07 2011 +0100
+++ b/server/sqlutils.py Wed Dec 21 18:26:35 2011 +0100
@@ -214,31 +214,11 @@
# callback lookup for each *cell* in results when there is nothing to
# lookup
if not column_callbacks:
- return self._process_result(cursor)
+ return self.dbhelper.dbapi_module.process_cursor(cursor, self._dbencoding,
+ Binary)
assert session
return self._cb_process_result(cursor, column_callbacks, session)
- def _process_result(self, cursor):
- # begin bind to locals for optimization
- descr = cursor.description
- encoding = self._dbencoding
- process_value = self._process_value
- binary = Binary
- # /end
- cursor.arraysize = 100
- while True:
- results = cursor.fetchmany()
- if not results:
- break
- for line in results:
- result = []
- for col, value in enumerate(line):
- if value is None:
- result.append(value)
- continue
- result.append(process_value(value, descr[col], encoding, binary))
- yield result
-
def _cb_process_result(self, cursor, column_callbacks, session):
# begin bind to locals for optimization
descr = cursor.description
--- a/server/test/unittest_postgres.py Wed Dec 21 18:04:07 2011 +0100
+++ b/server/test/unittest_postgres.py Wed Dec 21 18:26:35 2011 +0100
@@ -32,7 +32,7 @@
content=u'cubicweb cubicweb')
self.commit()
self.assertEqual(req.execute('Card X ORDERBY FTIRANK(X) DESC WHERE X has_text "cubicweb"').rows,
- [[c1.eid], [c3.eid], [c2.eid]])
+ [(c1.eid,), (c3.eid,), (c2.eid,)])
def test_attr_weight(self):
@@ -49,7 +49,7 @@
content=u'autre chose')
self.commit()
self.assertEqual(req.execute('Card X ORDERBY FTIRANK(X) DESC WHERE X has_text "cubicweb"').rows,
- [[c3.eid], [c1.eid], [c2.eid]])
+ [(c3.eid,), (c1.eid,), (c2.eid,)])
def test_entity_weight(self):
class PersonneIFTIndexableAdapter(IFTIndexableAdapter):
@@ -62,7 +62,7 @@
c3 = req.create_entity('Comment', content=u'cubicweb cubicweb cubicweb', comments=c1)
self.commit()
self.assertEqual(req.execute('Any X ORDERBY FTIRANK(X) DESC WHERE X has_text "cubicweb"').rows,
- [[c1.eid], [c3.eid], [c2.eid]])
+ [(c1.eid,), (c3.eid,), (c2.eid,)])
def test_tz_datetime(self):