1551 |
1551 |
1552 The backup and restore methods are used to dump / restore the |
1552 The backup and restore methods are used to dump / restore the |
1553 system database in a database independent format. The file is a |
1553 system database in a database independent format. The file is a |
1554 Zip archive containing the following files: |
1554 Zip archive containing the following files: |
1555 |
1555 |
1556 * format.txt: the format of the archive. Currently '1.0' |
1556 * format.txt: the format of the archive. Currently '1.1' |
1557 * tables.txt: list of filenames in the archive tables/ directory |
1557 * tables.txt: list of filenames in the archive tables/ directory |
1558 * sequences.txt: list of filenames in the archive sequences/ directory |
1558 * sequences.txt: list of filenames in the archive sequences/ directory |
|
1559 * numranges.txt: list of filenames in the archive numrange/ directory |
1559 * versions.txt: the list of cube versions from CWProperty |
1560 * versions.txt: the list of cube versions from CWProperty |
1560 * tables/<tablename>.<chunkno>: pickled data |
1561 * tables/<tablename>.<chunkno>: pickled data |
1561 * sequences/<sequencename>: pickled data |
1562 * sequences/<sequencename>: pickled data |
1562 |
1563 |
1563 The pickled data format for tables and sequences is a tuple of 3 elements: |
1564 The pickled data format for tables, numranges and sequences is a tuple of 3 elements: |
1564 * the table name |
1565 * the table name |
1565 * a tuple of column names |
1566 * a tuple of column names |
1566 * a list of rows (as tuples with one element per column) |
1567 * a list of rows (as tuples with one element per column) |
1567 |
1568 |
1568 Tables are saved in chunks in different files in order to prevent |
1569 Tables are saved in chunks in different files in order to prevent |
1596 self.logger.info('writing metadata') |
1597 self.logger.info('writing metadata') |
1597 self.write_metadata(archive) |
1598 self.write_metadata(archive) |
1598 for seq in self.get_sequences(): |
1599 for seq in self.get_sequences(): |
1599 self.logger.info('processing sequence %s', seq) |
1600 self.logger.info('processing sequence %s', seq) |
1600 self.write_sequence(archive, seq) |
1601 self.write_sequence(archive, seq) |
|
1602 for numrange in self.get_numranges(): |
|
1603 self.logger.info('processing numrange %s', numrange) |
|
1604 self.write_numrange(archive, numrange) |
1601 for table in self.get_tables(): |
1605 for table in self.get_tables(): |
1602 self.logger.info('processing table %s', table) |
1606 self.logger.info('processing table %s', table) |
1603 self.write_table(archive, table) |
1607 self.write_table(archive, table) |
1604 finally: |
1608 finally: |
1605 archive.close() |
1609 archive.close() |
1626 continue |
1630 continue |
1627 relation_tables.append('%s_relation' % rtype) |
1631 relation_tables.append('%s_relation' % rtype) |
1628 return non_entity_tables + etype_tables + relation_tables |
1632 return non_entity_tables + etype_tables + relation_tables |
1629 |
1633 |
1630 def get_sequences(self): |
1634 def get_sequences(self): |
|
1635 return [] |
|
1636 |
|
1637 def get_numranges(self): |
1631 return ['entities_id_seq'] |
1638 return ['entities_id_seq'] |
1632 |
1639 |
1633 def write_metadata(self, archive): |
1640 def write_metadata(self, archive): |
1634 archive.writestr('format.txt', '1.0') |
1641 archive.writestr('format.txt', '1.1') |
1635 archive.writestr('tables.txt', '\n'.join(self.get_tables())) |
1642 archive.writestr('tables.txt', '\n'.join(self.get_tables())) |
1636 archive.writestr('sequences.txt', '\n'.join(self.get_sequences())) |
1643 archive.writestr('sequences.txt', '\n'.join(self.get_sequences())) |
|
1644 archive.writestr('numranges.txt', '\n'.join(self.get_numranges())) |
1637 versions = self._get_versions() |
1645 versions = self._get_versions() |
1638 versions_str = '\n'.join('%s %s' % (k, v) |
1646 versions_str = '\n'.join('%s %s' % (k, v) |
1639 for k, v in versions) |
1647 for k, v in versions) |
1640 archive.writestr('versions.txt', versions_str) |
1648 archive.writestr('versions.txt', versions_str) |
1641 |
1649 |
1643 sql = self.dbhelper.sql_sequence_current_state(seq) |
1651 sql = self.dbhelper.sql_sequence_current_state(seq) |
1644 columns, rows_iterator = self._get_cols_and_rows(sql) |
1652 columns, rows_iterator = self._get_cols_and_rows(sql) |
1645 rows = list(rows_iterator) |
1653 rows = list(rows_iterator) |
1646 serialized = self._serialize(seq, columns, rows) |
1654 serialized = self._serialize(seq, columns, rows) |
1647 archive.writestr('sequences/%s' % seq, serialized) |
1655 archive.writestr('sequences/%s' % seq, serialized) |
|
1656 |
|
1657 def write_numrange(self, archive, numrange): |
|
1658 sql = self.dbhelper.sql_numrange_current_state(numrange) |
|
1659 columns, rows_iterator = self._get_cols_and_rows(sql) |
|
1660 rows = list(rows_iterator) |
|
1661 serialized = self._serialize(numrange, columns, rows) |
|
1662 archive.writestr('numrange/%s' % numrange, serialized) |
1648 |
1663 |
1649 def write_table(self, archive, table): |
1664 def write_table(self, archive, table): |
1650 nb_lines_sql = 'SELECT COUNT(*) FROM %s' % table |
1665 nb_lines_sql = 'SELECT COUNT(*) FROM %s' % table |
1651 self.cursor.execute(nb_lines_sql) |
1666 self.cursor.execute(nb_lines_sql) |
1652 rowcount = self.cursor.fetchone()[0] |
1667 rowcount = self.cursor.fetchone()[0] |
1680 |
1695 |
1681 def restore(self, backupfile): |
1696 def restore(self, backupfile): |
1682 archive = zipfile.ZipFile(backupfile, 'r', allowZip64=True) |
1697 archive = zipfile.ZipFile(backupfile, 'r', allowZip64=True) |
1683 self.cnx = self.get_connection() |
1698 self.cnx = self.get_connection() |
1684 self.cursor = self.cnx.cursor() |
1699 self.cursor = self.cnx.cursor() |
1685 sequences, tables, table_chunks = self.read_metadata(archive, backupfile) |
1700 sequences, numranges, tables, table_chunks = self.read_metadata(archive, backupfile) |
1686 for seq in sequences: |
1701 for seq in sequences: |
1687 self.logger.info('restoring sequence %s', seq) |
1702 self.logger.info('restoring sequence %s', seq) |
1688 self.read_sequence(archive, seq) |
1703 self.read_sequence(archive, seq) |
|
1704 for numrange in numranges: |
|
1705 self.logger.info('restoring numrange %s', seq) |
|
1706 self.read_numrange(archive, numrange) |
1689 for table in tables: |
1707 for table in tables: |
1690 self.logger.info('restoring table %s', table) |
1708 self.logger.info('restoring table %s', table) |
1691 self.read_table(archive, table, sorted(table_chunks[table])) |
1709 self.read_table(archive, table, sorted(table_chunks[table])) |
1692 self.cnx.close() |
1710 self.cnx.close() |
1693 archive.close() |
1711 archive.close() |
1694 self.logger.info('done') |
1712 self.logger.info('done') |
1695 |
1713 |
1696 def read_metadata(self, archive, backupfile): |
1714 def read_metadata(self, archive, backupfile): |
1697 formatinfo = archive.read('format.txt') |
1715 formatinfo = archive.read('format.txt') |
1698 self.logger.info('checking metadata') |
1716 self.logger.info('checking metadata') |
1699 if formatinfo.strip() != "1.0": |
1717 if formatinfo.strip() != "1.1": |
1700 self.logger.critical('Unsupported format in archive: %s', formatinfo) |
1718 self.logger.critical('Unsupported format in archive: %s', formatinfo) |
1701 raise ValueError('Unknown format in %s: %s' % (backupfile, formatinfo)) |
1719 raise ValueError('Unknown format in %s: %s' % (backupfile, formatinfo)) |
1702 tables = archive.read('tables.txt').splitlines() |
1720 tables = archive.read('tables.txt').splitlines() |
1703 sequences = archive.read('sequences.txt').splitlines() |
1721 sequences = archive.read('sequences.txt').splitlines() |
|
1722 numranges = archive.read('numranges.txt').splitlines() |
1704 file_versions = self._parse_versions(archive.read('versions.txt')) |
1723 file_versions = self._parse_versions(archive.read('versions.txt')) |
1705 versions = set(self._get_versions()) |
1724 versions = set(self._get_versions()) |
1706 if file_versions != versions: |
1725 if file_versions != versions: |
1707 self.logger.critical('Unable to restore : versions do not match') |
1726 self.logger.critical('Unable to restore : versions do not match') |
1708 self.logger.critical('Expected:\n%s', '\n'.join('%s : %s' % (cube, ver) |
1727 self.logger.critical('Expected:\n%s', '\n'.join('%s : %s' % (cube, ver) |
1715 if not name.startswith('tables/'): |
1734 if not name.startswith('tables/'): |
1716 continue |
1735 continue |
1717 filename = basename(name) |
1736 filename = basename(name) |
1718 tablename, _ext = filename.rsplit('.', 1) |
1737 tablename, _ext = filename.rsplit('.', 1) |
1719 table_chunks.setdefault(tablename, []).append(name) |
1738 table_chunks.setdefault(tablename, []).append(name) |
1720 return sequences, tables, table_chunks |
1739 return sequences, numranges, tables, table_chunks |
1721 |
1740 |
1722 def read_sequence(self, archive, seq): |
1741 def read_sequence(self, archive, seq): |
1723 seqname, columns, rows = loads(archive.read('sequences/%s' % seq)) |
1742 seqname, columns, rows = loads(archive.read('sequences/%s' % seq)) |
1724 assert seqname == seq |
1743 assert seqname == seq |
1725 assert len(rows) == 1 |
1744 assert len(rows) == 1 |
1726 assert len(rows[0]) == 1 |
1745 assert len(rows[0]) == 1 |
1727 value = rows[0][0] |
1746 value = rows[0][0] |
1728 sql = self.dbhelper.sql_restart_sequence(seq, value) |
1747 sql = self.dbhelper.sql_restart_sequence(seq, value) |
|
1748 self.cursor.execute(sql) |
|
1749 self.cnx.commit() |
|
1750 |
|
1751 def read_numrange(self, archive, numrange): |
|
1752 rangename, columns, rows = loads(archive.read('numrange/%s' % numrange)) |
|
1753 assert rangename == numrange |
|
1754 assert len(rows) == 1 |
|
1755 assert len(rows[0]) == 1 |
|
1756 value = rows[0][0] |
|
1757 sql = self.dbhelper.sql_restart_numrange(numrange, value) |
1729 self.cursor.execute(sql) |
1758 self.cursor.execute(sql) |
1730 self.cnx.commit() |
1759 self.cnx.commit() |
1731 |
1760 |
1732 def read_table(self, archive, table, filenames): |
1761 def read_table(self, archive, table, filenames): |
1733 merge_args = self._source.merge_args |
1762 merge_args = self._source.merge_args |