1684 rows = list(rows_iterator) |
1684 rows = list(rows_iterator) |
1685 serialized = self._serialize(seq, columns, rows) |
1685 serialized = self._serialize(seq, columns, rows) |
1686 archive.writestr('sequences/%s' % seq, serialized) |
1686 archive.writestr('sequences/%s' % seq, serialized) |
1687 |
1687 |
1688 def write_table(self, archive, table): |
1688 def write_table(self, archive, table): |
|
1689 nb_lines_sql = 'SELECT COUNT(*) FROM %s' % table |
|
1690 self.cursor.execute(nb_lines_sql) |
|
1691 rowcount = self.cursor.fetchone()[0] |
1689 sql = 'SELECT * FROM %s' % table |
1692 sql = 'SELECT * FROM %s' % table |
1690 columns, rows_iterator = self._get_cols_and_rows(sql) |
1693 columns, rows_iterator = self._get_cols_and_rows(sql) |
1691 self.logger.info('number of rows: %d', self.cursor.rowcount) |
1694 self.logger.info('number of rows: %d', rowcount) |
1692 if table.startswith('cw_'): # entities |
1695 if table.startswith('cw_'): # entities |
1693 blocksize = 2000 |
1696 blocksize = 2000 |
1694 else: # relations and metadata |
1697 else: # relations and metadata |
1695 blocksize = 10000 |
1698 blocksize = 10000 |
1696 if self.cursor.rowcount > 0: |
1699 if rowcount > 0: |
1697 for i, start in enumerate(xrange(0, self.cursor.rowcount, blocksize)): |
1700 for i, start in enumerate(xrange(0, rowcount, blocksize)): |
1698 rows = list(itertools.islice(rows_iterator, blocksize)) |
1701 rows = list(itertools.islice(rows_iterator, blocksize)) |
1699 serialized = self._serialize(table, columns, rows) |
1702 serialized = self._serialize(table, columns, rows) |
1700 archive.writestr('tables/%s.%04d' % (table, i), serialized) |
1703 archive.writestr('tables/%s.%04d' % (table, i), serialized) |
1701 self.logger.debug('wrote rows %d to %d (out of %d) to %s.%04d', |
1704 self.logger.debug('wrote rows %d to %d (out of %d) to %s.%04d', |
1702 start, start+len(rows)-1, |
1705 start, start+len(rows)-1, |
1703 self.cursor.rowcount, |
1706 rowcount, |
1704 table, i) |
1707 table, i) |
1705 else: |
1708 else: |
1706 rows = [] |
1709 rows = [] |
1707 serialized = self._serialize(table, columns, rows) |
1710 serialized = self._serialize(table, columns, rows) |
1708 archive.writestr('tables/%s.%04d' % (table, 0), serialized) |
1711 archive.writestr('tables/%s.%04d' % (table, 0), serialized) |