1539 |
1539 |
1540 The backup and restore methods are used to dump / restore the |
1540 The backup and restore methods are used to dump / restore the |
1541 system database in a database independent format. The file is a |
1541 system database in a database independent format. The file is a |
1542 Zip archive containing the following files: |
1542 Zip archive containing the following files: |
1543 |
1543 |
1544 * format.txt: the format of the archive. Currently '1.0' |
1544 * format.txt: the format of the archive. Currently '1.1' |
1545 * tables.txt: list of filenames in the archive tables/ directory |
1545 * tables.txt: list of filenames in the archive tables/ directory |
1546 * sequences.txt: list of filenames in the archive sequences/ directory |
1546 * sequences.txt: list of filenames in the archive sequences/ directory |
|
1547 * numranges.txt: list of filenames in the archive numrange/ directory |
1547 * versions.txt: the list of cube versions from CWProperty |
1548 * versions.txt: the list of cube versions from CWProperty |
1548 * tables/<tablename>.<chunkno>: pickled data |
1549 * tables/<tablename>.<chunkno>: pickled data |
1549 * sequences/<sequencename>: pickled data |
1550 * sequences/<sequencename>: pickled data |
1550 |
1551 |
1551 The pickled data format for tables and sequences is a tuple of 3 elements: |
1552 The pickled data format for tables, numranges and sequences is a tuple of 3 elements: |
1552 * the table name |
1553 * the table name |
1553 * a tuple of column names |
1554 * a tuple of column names |
1554 * a list of rows (as tuples with one element per column) |
1555 * a list of rows (as tuples with one element per column) |
1555 |
1556 |
1556 Tables are saved in chunks in different files in order to prevent |
1557 Tables are saved in chunks in different files in order to prevent |
1682 |
1683 |
1683 def restore(self, backupfile): |
1684 def restore(self, backupfile): |
1684 archive = zipfile.ZipFile(backupfile, 'r', allowZip64=True) |
1685 archive = zipfile.ZipFile(backupfile, 'r', allowZip64=True) |
1685 self.cnx = self.get_connection() |
1686 self.cnx = self.get_connection() |
1686 self.cursor = self.cnx.cursor() |
1687 self.cursor = self.cnx.cursor() |
1687 sequences, tables, table_chunks = self.read_metadata(archive, backupfile) |
1688 sequences, numranges, tables, table_chunks = self.read_metadata(archive, backupfile) |
1688 for seq in sequences: |
1689 for seq in sequences: |
1689 self.logger.info('restoring sequence %s', seq) |
1690 self.logger.info('restoring sequence %s', seq) |
1690 self.read_sequence(archive, seq) |
1691 self.read_sequence(archive, seq) |
|
1692 for numrange in numranges: |
|
1693 self.logger.info('restoring numrange %s', seq) |
|
1694 self.read_numrange(archive, numrange) |
1691 for table in tables: |
1695 for table in tables: |
1692 self.logger.info('restoring table %s', table) |
1696 self.logger.info('restoring table %s', table) |
1693 self.read_table(archive, table, sorted(table_chunks[table])) |
1697 self.read_table(archive, table, sorted(table_chunks[table])) |
1694 self.cnx.close() |
1698 self.cnx.close() |
1695 archive.close() |
1699 archive.close() |
1696 self.logger.info('done') |
1700 self.logger.info('done') |
1697 |
1701 |
1698 def read_metadata(self, archive, backupfile): |
1702 def read_metadata(self, archive, backupfile): |
1699 formatinfo = archive.read('format.txt') |
1703 formatinfo = archive.read('format.txt') |
1700 self.logger.info('checking metadata') |
1704 self.logger.info('checking metadata') |
1701 if formatinfo.strip() != "1.0": |
1705 if formatinfo.strip() != "1.1": |
1702 self.logger.critical('Unsupported format in archive: %s', formatinfo) |
1706 self.logger.critical('Unsupported format in archive: %s', formatinfo) |
1703 raise ValueError('Unknown format in %s: %s' % (backupfile, formatinfo)) |
1707 raise ValueError('Unknown format in %s: %s' % (backupfile, formatinfo)) |
1704 tables = archive.read('tables.txt').splitlines() |
1708 tables = archive.read('tables.txt').splitlines() |
1705 sequences = archive.read('sequences.txt').splitlines() |
1709 sequences = archive.read('sequences.txt').splitlines() |
|
1710 numranges = archive.read('numranges.txt').splitlines() |
1706 file_versions = self._parse_versions(archive.read('versions.txt')) |
1711 file_versions = self._parse_versions(archive.read('versions.txt')) |
1707 versions = set(self._get_versions()) |
1712 versions = set(self._get_versions()) |
1708 if file_versions != versions: |
1713 if file_versions != versions: |
1709 self.logger.critical('Unable to restore : versions do not match') |
1714 self.logger.critical('Unable to restore : versions do not match') |
1710 self.logger.critical('Expected:\n%s', '\n'.join('%s : %s' % (cube, ver) |
1715 self.logger.critical('Expected:\n%s', '\n'.join('%s : %s' % (cube, ver) |
1717 if not name.startswith('tables/'): |
1722 if not name.startswith('tables/'): |
1718 continue |
1723 continue |
1719 filename = basename(name) |
1724 filename = basename(name) |
1720 tablename, _ext = filename.rsplit('.', 1) |
1725 tablename, _ext = filename.rsplit('.', 1) |
1721 table_chunks.setdefault(tablename, []).append(name) |
1726 table_chunks.setdefault(tablename, []).append(name) |
1722 return sequences, tables, table_chunks |
1727 return sequences, numranges, tables, table_chunks |
1723 |
1728 |
1724 def read_sequence(self, archive, seq): |
1729 def read_sequence(self, archive, seq): |
1725 seqname, columns, rows = loads(archive.read('sequences/%s' % seq)) |
1730 seqname, columns, rows = loads(archive.read('sequences/%s' % seq)) |
1726 assert seqname == seq |
1731 assert seqname == seq |
1727 assert len(rows) == 1 |
1732 assert len(rows) == 1 |
1728 assert len(rows[0]) == 1 |
1733 assert len(rows[0]) == 1 |
1729 value = rows[0][0] |
1734 value = rows[0][0] |
1730 sql = self.dbhelper.sql_restart_sequence(seq, value) |
1735 sql = self.dbhelper.sql_restart_sequence(seq, value) |
|
1736 self.cursor.execute(sql) |
|
1737 self.cnx.commit() |
|
1738 |
|
1739 def read_numrange(self, archive, numrange): |
|
1740 rangename, columns, rows = loads(archive.read('numrange/%s' % numrange)) |
|
1741 assert rangename == numrange |
|
1742 assert len(rows) == 1 |
|
1743 assert len(rows[0]) == 1 |
|
1744 value = rows[0][0] |
|
1745 sql = self.dbhelper.sql_restart_numrange(numrange, value) |
1731 self.cursor.execute(sql) |
1746 self.cursor.execute(sql) |
1732 self.cnx.commit() |
1747 self.cnx.commit() |
1733 |
1748 |
1734 def read_table(self, archive, table, filenames): |
1749 def read_table(self, archive, table, filenames): |
1735 merge_args = self._source.merge_args |
1750 merge_args = self._source.merge_args |