760 mo = re.search("unique_[a-z0-9]{32}", arg) |
760 mo = re.search("unique_[a-z0-9]{32}", arg) |
761 if mo is not None: |
761 if mo is not None: |
762 raise UniqueTogetherError(session, cstrname=mo.group(0)) |
762 raise UniqueTogetherError(session, cstrname=mo.group(0)) |
763 # sqlite |
763 # sqlite |
764 mo = re.search('columns (.*) are not unique', arg) |
764 mo = re.search('columns (.*) are not unique', arg) |
|
765 if mo is None: |
|
766 # sqlite > 3.7 |
|
767 mo = re.search('UNIQUE constraint failed: (.*)', arg) |
765 if mo is not None: # sqlite in use |
768 if mo is not None: # sqlite in use |
766 # we left chop the 'cw_' prefix of attribute names |
769 # we left chop the 'cw_' prefix of attribute names |
767 rtypes = [c.strip()[3:] |
770 rtypes = [c.strip()[3:] |
768 for c in mo.group(1).split(',')] |
771 for c in mo.group(1).split(',')] |
769 raise UniqueTogetherError(session, rtypes=rtypes) |
772 raise UniqueTogetherError(session, rtypes=rtypes) |
1696 * a list of rows (as tuples with one element per column) |
1699 * a list of rows (as tuples with one element per column) |
1697 |
1700 |
1698 Tables are saved in chunks in different files in order to prevent |
1701 Tables are saved in chunks in different files in order to prevent |
1699 a too high memory consumption. |
1702 a too high memory consumption. |
1700 """ |
1703 """ |
|
1704 blocksize = 100 |
|
1705 |
1701 def __init__(self, source): |
1706 def __init__(self, source): |
1702 """ |
1707 """ |
1703 :param: source an instance of the system source |
1708 :param: source an instance of the system source |
1704 """ |
1709 """ |
1705 self._source = source |
1710 self._source = source |
1780 self.cursor.execute(nb_lines_sql) |
1785 self.cursor.execute(nb_lines_sql) |
1781 rowcount = self.cursor.fetchone()[0] |
1786 rowcount = self.cursor.fetchone()[0] |
1782 sql = 'SELECT * FROM %s' % table |
1787 sql = 'SELECT * FROM %s' % table |
1783 columns, rows_iterator = self._get_cols_and_rows(sql) |
1788 columns, rows_iterator = self._get_cols_and_rows(sql) |
1784 self.logger.info('number of rows: %d', rowcount) |
1789 self.logger.info('number of rows: %d', rowcount) |
1785 if table.startswith('cw_'): # entities |
1790 blocksize = self.blocksize |
1786 blocksize = 2000 |
|
1787 else: # relations and metadata |
|
1788 blocksize = 10000 |
|
1789 if rowcount > 0: |
1791 if rowcount > 0: |
1790 for i, start in enumerate(xrange(0, rowcount, blocksize)): |
1792 for i, start in enumerate(xrange(0, rowcount, blocksize)): |
1791 rows = list(itertools.islice(rows_iterator, blocksize)) |
1793 rows = list(itertools.islice(rows_iterator, blocksize)) |
1792 serialized = self._serialize(table, columns, rows) |
1794 serialized = self._serialize(table, columns, rows) |
1793 archive.writestr('tables/%s.%04d' % (table, i), serialized) |
1795 archive.writestr('tables/%s.%04d' % (table, i), serialized) |