1709 rows = list(rows_iterator) |
1709 rows = list(rows_iterator) |
1710 serialized = self._serialize(seq, columns, rows) |
1710 serialized = self._serialize(seq, columns, rows) |
1711 archive.writestr('sequences/%s' % seq, serialized) |
1711 archive.writestr('sequences/%s' % seq, serialized) |
1712 |
1712 |
1713 def write_table(self, archive, table): |
1713 def write_table(self, archive, table): |
|
1714 nb_lines_sql = 'SELECT COUNT(*) FROM %s' % table |
|
1715 self.cursor.execute(nb_lines_sql) |
|
1716 rowcount = self.cursor.fetchone()[0] |
1714 sql = 'SELECT * FROM %s' % table |
1717 sql = 'SELECT * FROM %s' % table |
1715 columns, rows_iterator = self._get_cols_and_rows(sql) |
1718 columns, rows_iterator = self._get_cols_and_rows(sql) |
1716 self.logger.info('number of rows: %d', self.cursor.rowcount) |
1719 self.logger.info('number of rows: %d', rowcount) |
1717 if table.startswith('cw_'): # entities |
1720 if table.startswith('cw_'): # entities |
1718 blocksize = 2000 |
1721 blocksize = 2000 |
1719 else: # relations and metadata |
1722 else: # relations and metadata |
1720 blocksize = 10000 |
1723 blocksize = 10000 |
1721 if self.cursor.rowcount > 0: |
1724 if rowcount > 0: |
1722 for i, start in enumerate(xrange(0, self.cursor.rowcount, blocksize)): |
1725 for i, start in enumerate(xrange(0, rowcount, blocksize)): |
1723 rows = list(itertools.islice(rows_iterator, blocksize)) |
1726 rows = list(itertools.islice(rows_iterator, blocksize)) |
1724 serialized = self._serialize(table, columns, rows) |
1727 serialized = self._serialize(table, columns, rows) |
1725 archive.writestr('tables/%s.%04d' % (table, i), serialized) |
1728 archive.writestr('tables/%s.%04d' % (table, i), serialized) |
1726 self.logger.debug('wrote rows %d to %d (out of %d) to %s.%04d', |
1729 self.logger.debug('wrote rows %d to %d (out of %d) to %s.%04d', |
1727 start, start+len(rows)-1, |
1730 start, start+len(rows)-1, |
1728 self.cursor.rowcount, |
1731 rowcount, |
1729 table, i) |
1732 table, i) |
1730 else: |
1733 else: |
1731 rows = [] |
1734 rows = [] |
1732 serialized = self._serialize(table, columns, rows) |
1735 serialized = self._serialize(table, columns, rows) |
1733 archive.writestr('tables/%s.%04d' % (table, 0), serialized) |
1736 archive.writestr('tables/%s.%04d' % (table, 0), serialized) |