--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/cubicweb/server/migractions.py Sat Jan 16 13:48:51 2016 +0100
@@ -0,0 +1,1603 @@
+# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+"""a class implementing basic actions used in migration scripts.
+
+The following schema actions are supported for now:
+* add/drop/rename attribute
+* add/drop entity/relation type
+* rename entity type
+
+The following data actions are supported for now:
+* add an entity
+* execute raw RQL queries
+"""
+from __future__ import print_function
+
+__docformat__ = "restructuredtext en"
+
+import sys
+import os
+import tarfile
+import tempfile
+import shutil
+import os.path as osp
+from datetime import datetime
+from glob import glob
+from copy import copy
+from warnings import warn
+from contextlib import contextmanager
+
+from six import PY2, text_type
+
+from logilab.common.deprecation import deprecated
+from logilab.common.decorators import cached, clear_cache
+
+from yams.buildobjs import EntityType
+from yams.constraints import SizeConstraint
+from yams.schema import RelationDefinitionSchema
+
+from cubicweb import CW_SOFTWARE_ROOT, AuthenticationError, ExecutionError
+from cubicweb.predicates import is_instance
+from cubicweb.schema import (ETYPE_NAME_MAP, META_RTYPES, VIRTUAL_RTYPES,
+ PURE_VIRTUAL_RTYPES,
+ CubicWebRelationSchema, order_eschemas)
+from cubicweb.cwvreg import CW_EVENT_MANAGER
+from cubicweb import repoapi
+from cubicweb.migration import MigrationHelper, yes
+from cubicweb.server import hook, schemaserial as ss
+from cubicweb.server.schema2sql import eschema2sql, rschema2sql, unique_index_name, sql_type
+from cubicweb.server.utils import manager_userpasswd
+from cubicweb.server.sqlutils import sqlexec, SQL_PREFIX
+
+
+class ClearGroupMap(hook.Hook):
+ __regid__ = 'cw.migration.clear_group_mapping'
+ __select__ = hook.Hook.__select__ & is_instance('CWGroup')
+ events = ('after_add_entity', 'after_update_entity',)
+ def __call__(self):
+ clear_cache(self.mih, 'group_mapping')
+ self.mih._synchronized.clear()
+
+ @classmethod
+ def mih_register(cls, repo):
+ # may be already registered in tests (e.g. unittest_migractions at
+ # least)
+ if not cls.__regid__ in repo.vreg['after_add_entity_hooks']:
+ repo.vreg.register(ClearGroupMap)
+
+
+class ServerMigrationHelper(MigrationHelper):
+ """specific migration helper for server side migration scripts,
+ providing actions related to schema/data migration
+ """
+
+ def __init__(self, config, schema, interactive=True,
+ repo=None, cnx=None, verbosity=1, connect=True):
+ MigrationHelper.__init__(self, config, interactive, verbosity)
+ if not interactive:
+ assert cnx
+ assert repo
+ if cnx is not None:
+ assert repo
+ self.cnx = cnx
+ self.repo = repo
+ self.session = cnx.session
+ elif connect:
+ self.repo = config.repository()
+ self.set_cnx()
+ else:
+ self.session = None
+ # no config on shell to a remote instance
+ if config is not None and (cnx or connect):
+ repo = self.repo
+ # register a hook to clear our group_mapping cache and the
+ # self._synchronized set when some group is added or updated
+ ClearGroupMap.mih = self
+ ClearGroupMap.mih_register(repo)
+ CW_EVENT_MANAGER.bind('after-registry-reload',
+ ClearGroupMap.mih_register, repo)
+ # notify we're starting maintenance (called instead of server_start
+ # which is called on regular start
+ repo.hm.call_hooks('server_maintenance', repo=repo)
+ if not schema and not config.quick_start:
+ insert_lperms = self.repo.get_versions()['cubicweb'] < (3, 14, 0) and 'localperms' in config.available_cubes()
+ if insert_lperms:
+ cubes = config._cubes
+ config._cubes += ('localperms',)
+ try:
+ schema = config.load_schema(expand_cubes=True)
+ finally:
+ if insert_lperms:
+ config._cubes = cubes
+ self.fs_schema = schema
+ self._synchronized = set()
+
+ # overriden from base MigrationHelper ######################################
+
+ def set_cnx(self):
+ try:
+ login = self.repo.config.default_admin_config['login']
+ pwd = self.repo.config.default_admin_config['password']
+ except KeyError:
+ login, pwd = manager_userpasswd()
+ while True:
+ try:
+ self.cnx = repoapi.connect(self.repo, login, password=pwd)
+ if not 'managers' in self.cnx.user.groups:
+ print('migration need an account in the managers group')
+ else:
+ break
+ except AuthenticationError:
+ print('wrong user/password')
+ except (KeyboardInterrupt, EOFError):
+ print('aborting...')
+ sys.exit(0)
+ try:
+ login, pwd = manager_userpasswd()
+ except (KeyboardInterrupt, EOFError):
+ print('aborting...')
+ sys.exit(0)
+ self.session = self.repo._get_session(self.cnx.sessionid)
+
+ def cube_upgraded(self, cube, version):
+ self.cmd_set_property('system.version.%s' % cube.lower(),
+ text_type(version))
+ self.commit()
+
+ def shutdown(self):
+ if self.repo is not None:
+ self.repo.shutdown()
+
+ def migrate(self, vcconf, toupgrade, options):
+ if not options.fs_only:
+ if options.backup_db is None:
+ self.backup_database()
+ elif options.backup_db:
+ self.backup_database(askconfirm=False)
+ # disable notification during migration
+ with self.cnx.allow_all_hooks_but('notification'):
+ super(ServerMigrationHelper, self).migrate(vcconf, toupgrade, options)
+
+ def cmd_process_script(self, migrscript, funcname=None, *args, **kwargs):
+ try:
+ return super(ServerMigrationHelper, self).cmd_process_script(
+ migrscript, funcname, *args, **kwargs)
+ except ExecutionError as err:
+ sys.stderr.write("-> %s\n" % err)
+ except BaseException:
+ self.rollback()
+ raise
+
+ # Adjust docstring
+ cmd_process_script.__doc__ = MigrationHelper.cmd_process_script.__doc__
+
+ # server specific migration methods ########################################
+
+ def backup_database(self, backupfile=None, askconfirm=True, format='native'):
+ config = self.config
+ repo = self.repo
+ # paths
+ timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
+ instbkdir = osp.join(config.appdatahome, 'backup')
+ if not osp.exists(instbkdir):
+ os.makedirs(instbkdir)
+ backupfile = backupfile or osp.join(instbkdir, '%s-%s.tar.gz'
+ % (config.appid, timestamp))
+ # check backup has to be done
+ if osp.exists(backupfile) and not \
+ self.confirm('Backup file %s exists, overwrite it?' % backupfile):
+ print('-> no backup done.')
+ return
+ elif askconfirm and not self.confirm('Backup %s database?' % config.appid):
+ print('-> no backup done.')
+ return
+ open(backupfile,'w').close() # kinda lock
+ os.chmod(backupfile, 0o600)
+ # backup
+ source = repo.system_source
+ tmpdir = tempfile.mkdtemp()
+ try:
+ failed = False
+ try:
+ source.backup(osp.join(tmpdir, source.uri), self.confirm, format=format)
+ except Exception as ex:
+ print('-> error trying to backup %s [%s]' % (source.uri, ex))
+ if not self.confirm('Continue anyway?', default='n'):
+ raise SystemExit(1)
+ else:
+ failed = True
+ with open(osp.join(tmpdir, 'format.txt'), 'w') as format_file:
+ format_file.write('%s\n' % format)
+ with open(osp.join(tmpdir, 'versions.txt'), 'w') as version_file:
+ versions = repo.get_versions()
+ for cube, version in versions.items():
+ version_file.write('%s %s\n' % (cube, version))
+ if not failed:
+ bkup = tarfile.open(backupfile, 'w|gz')
+ for filename in os.listdir(tmpdir):
+ bkup.add(osp.join(tmpdir, filename), filename)
+ bkup.close()
+ # call hooks
+ repo.hm.call_hooks('server_backup', repo=repo, timestamp=timestamp)
+ # done
+ print('-> backup file', backupfile)
+ finally:
+ shutil.rmtree(tmpdir)
+
+ def restore_database(self, backupfile, drop=True, askconfirm=True, format='native'):
+ # check
+ if not osp.exists(backupfile):
+ raise ExecutionError("Backup file %s doesn't exist" % backupfile)
+ if askconfirm and not self.confirm('Restore %s database from %s ?'
+ % (self.config.appid, backupfile)):
+ return
+ # unpack backup
+ tmpdir = tempfile.mkdtemp()
+ try:
+ bkup = tarfile.open(backupfile, 'r|gz')
+ except tarfile.ReadError:
+ # assume restoring old backup
+ shutil.copy(backupfile, osp.join(tmpdir, 'system'))
+ else:
+ for name in bkup.getnames():
+ if name[0] in '/.':
+ raise ExecutionError('Security check failed, path starts with "/" or "."')
+ bkup.close() # XXX seek error if not close+open !?!
+ bkup = tarfile.open(backupfile, 'r|gz')
+ bkup.extractall(path=tmpdir)
+ bkup.close()
+ if osp.isfile(osp.join(tmpdir, 'format.txt')):
+ with open(osp.join(tmpdir, 'format.txt')) as format_file:
+ written_format = format_file.readline().strip()
+ if written_format in ('portable', 'native'):
+ format = written_format
+ self.config.init_cnxset_pool = False
+ repo = self.repo = self.config.repository()
+ source = repo.system_source
+ try:
+ source.restore(osp.join(tmpdir, source.uri), self.confirm, drop, format)
+ except Exception as exc:
+ print('-> error trying to restore %s [%s]' % (source.uri, exc))
+ if not self.confirm('Continue anyway?', default='n'):
+ raise SystemExit(1)
+ shutil.rmtree(tmpdir)
+ # call hooks
+ repo.init_cnxset_pool()
+ repo.hm.call_hooks('server_restore', repo=repo, timestamp=backupfile)
+ print('-> database restored.')
+
+ def commit(self):
+ self.cnx.commit()
+
+ def rollback(self):
+ self.cnx.rollback()
+
+ def rqlexecall(self, rqliter, ask_confirm=False):
+ for rql, kwargs in rqliter:
+ self.rqlexec(rql, kwargs, ask_confirm=ask_confirm)
+
+ @cached
+ def _create_context(self):
+ """return a dictionary to use as migration script execution context"""
+ context = super(ServerMigrationHelper, self)._create_context()
+ context.update({'commit': self.checkpoint,
+ 'rollback': self.rollback,
+ 'sql': self.sqlexec,
+ 'rql': self.rqlexec,
+ 'rqliter': self.rqliter,
+ 'schema': self.repo.get_schema(),
+ 'cnx': self.cnx,
+ 'fsschema': self.fs_schema,
+ 'session' : self.cnx,
+ 'repo' : self.repo,
+ })
+ return context
+
+ @cached
+ def group_mapping(self):
+ """cached group mapping"""
+ return ss.group_mapping(self.cnx)
+
+ def cstrtype_mapping(self):
+ """cached constraint types mapping"""
+ return ss.cstrtype_mapping(self.cnx)
+
+ def cmd_exec_event_script(self, event, cube=None, funcname=None,
+ *args, **kwargs):
+ """execute a cube event scripts `migration/<event>.py` where event
+ is one of 'precreate', 'postcreate', 'preremove' and 'postremove'.
+ """
+ assert event in ('precreate', 'postcreate', 'preremove', 'postremove')
+ if cube:
+ cubepath = self.config.cube_dir(cube)
+ apc = osp.join(cubepath, 'migration', '%s.py' % event)
+ elif kwargs.pop('apphome', False):
+ apc = osp.join(self.config.apphome, 'migration', '%s.py' % event)
+ else:
+ apc = osp.join(self.config.migration_scripts_dir(), '%s.py' % event)
+ if osp.exists(apc):
+ if self.config.free_wheel:
+ self.cmd_deactivate_verification_hooks()
+ self.info('executing %s', apc)
+ confirm = self.confirm
+ execscript_confirm = self.execscript_confirm
+ self.confirm = yes
+ self.execscript_confirm = yes
+ try:
+ if event == 'postcreate':
+ with self.cnx.allow_all_hooks_but():
+ return self.cmd_process_script(apc, funcname, *args, **kwargs)
+ return self.cmd_process_script(apc, funcname, *args, **kwargs)
+ finally:
+ self.confirm = confirm
+ self.execscript_confirm = execscript_confirm
+ if self.config.free_wheel:
+ self.cmd_reactivate_verification_hooks()
+
+ def cmd_install_custom_sql_scripts(self, cube=None):
+ """install a cube custom sql scripts `schema/*.<driver>.sql` where
+ <driver> depends on the instance main database backend (eg 'postgres',
+ 'mysql'...)
+ """
+ driver = self.repo.system_source.dbdriver
+ if cube is None:
+ directory = osp.join(CW_SOFTWARE_ROOT, 'schemas')
+ else:
+ directory = osp.join(self.config.cube_dir(cube), 'schema')
+ sql_scripts = glob(osp.join(directory, '*.%s.sql' % driver))
+ for fpath in sql_scripts:
+ print('-> installing', fpath)
+ failed = sqlexec(open(fpath).read(), self.cnx.system_sql, False,
+ delimiter=';;')
+ if failed:
+ print('-> ERROR, skipping', fpath)
+
+ # schema synchronization internals ########################################
+
+ def _synchronize_permissions(self, erschema, teid):
+ """permission synchronization for an entity or relation type"""
+ assert teid, erschema
+ if 'update' in erschema.ACTIONS or erschema.final:
+ # entity type
+ exprtype = u'ERQLExpression'
+ else:
+ # relation type
+ exprtype = u'RRQLExpression'
+ gm = self.group_mapping()
+ confirm = self.verbosity >= 2
+ # * remove possibly deprecated permission (eg in the persistent schema
+ # but not in the new schema)
+ # * synchronize existing expressions
+ # * add new groups/expressions
+ for action in erschema.ACTIONS:
+ perm = '%s_permission' % action
+ # handle groups
+ newgroups = list(erschema.get_groups(action))
+ for geid, gname in self.rqlexec('Any G, GN WHERE T %s G, G name GN, '
+ 'T eid %%(x)s' % perm, {'x': teid},
+ ask_confirm=False):
+ if not gname in newgroups:
+ if not confirm or self.confirm('Remove %s permission of %s to %s?'
+ % (action, erschema, gname)):
+ self.rqlexec('DELETE T %s G WHERE G eid %%(x)s, T eid %s'
+ % (perm, teid),
+ {'x': geid}, ask_confirm=False)
+ else:
+ newgroups.remove(gname)
+ for gname in newgroups:
+ if not confirm or self.confirm('Grant %s permission of %s to %s?'
+ % (action, erschema, gname)):
+ try:
+ self.rqlexec('SET T %s G WHERE G eid %%(x)s, T eid %s'
+ % (perm, teid),
+ {'x': gm[gname]}, ask_confirm=False)
+ except KeyError:
+ self.error('can grant %s perm to unexistant group %s',
+ action, gname)
+ # handle rql expressions
+ newexprs = dict((expr.expression, expr) for expr in erschema.get_rqlexprs(action))
+ for expreid, expression in self.rqlexec('Any E, EX WHERE T %s E, E expression EX, '
+ 'T eid %s' % (perm, teid),
+ ask_confirm=False):
+ if not expression in newexprs:
+ if not confirm or self.confirm('Remove %s expression for %s permission of %s?'
+ % (expression, action, erschema)):
+ # deleting the relation will delete the expression entity
+ self.rqlexec('DELETE T %s E WHERE E eid %%(x)s, T eid %s'
+ % (perm, teid),
+ {'x': expreid}, ask_confirm=False)
+ else:
+ newexprs.pop(expression)
+ for expression in newexprs.values():
+ expr = expression.expression
+ if not confirm or self.confirm('Add %s expression for %s permission of %s?'
+ % (expr, action, erschema)):
+ self.rqlexec('INSERT RQLExpression X: X exprtype %%(exprtype)s, '
+ 'X expression %%(expr)s, X mainvars %%(vars)s, T %s X '
+ 'WHERE T eid %%(x)s' % perm,
+ {'expr': expr, 'exprtype': exprtype,
+ 'vars': u','.join(sorted(expression.mainvars)),
+ 'x': teid},
+ ask_confirm=False)
+
+ def _synchronize_rschema(self, rtype, syncrdefs=True,
+ syncperms=True, syncprops=True):
+ """synchronize properties of the persistent relation schema against its
+ current definition:
+
+ * description
+ * symmetric, meta
+ * inlined
+ * relation definitions if `syncrdefs`
+ * permissions if `syncperms`
+
+ physical schema changes should be handled by repository's schema hooks
+ """
+ rtype = str(rtype)
+ if rtype in self._synchronized:
+ return
+ if syncrdefs and syncperms and syncprops:
+ self._synchronized.add(rtype)
+ rschema = self.fs_schema.rschema(rtype)
+ reporschema = self.repo.schema.rschema(rtype)
+ if syncprops:
+ assert reporschema.eid, reporschema
+ self.rqlexecall(ss.updaterschema2rql(rschema, reporschema.eid),
+ ask_confirm=self.verbosity>=2)
+ if rschema.rule:
+ if syncperms:
+ self._synchronize_permissions(rschema, reporschema.eid)
+ elif syncrdefs:
+ for subj, obj in rschema.rdefs:
+ if (subj, obj) not in reporschema.rdefs:
+ continue
+ if rschema in VIRTUAL_RTYPES:
+ continue
+ self._synchronize_rdef_schema(subj, rschema, obj,
+ syncprops=syncprops,
+ syncperms=syncperms)
+
+ def _synchronize_eschema(self, etype, syncrdefs=True,
+ syncperms=True, syncprops=True):
+ """synchronize properties of the persistent entity schema against
+ its current definition:
+
+ * description
+ * internationalizable, fulltextindexed, indexed, meta
+ * relations from/to this entity
+ * __unique_together__
+ * permissions if `syncperms`
+ """
+ etype = str(etype)
+ if etype in self._synchronized:
+ return
+ if syncrdefs and syncperms and syncprops:
+ self._synchronized.add(etype)
+ repoeschema = self.repo.schema.eschema(etype)
+ try:
+ eschema = self.fs_schema.eschema(etype)
+ except KeyError:
+ return # XXX somewhat unexpected, no?...
+ if syncprops:
+ repospschema = repoeschema.specializes()
+ espschema = eschema.specializes()
+ if repospschema and not espschema:
+ self.rqlexec('DELETE X specializes Y WHERE X is CWEType, X name %(x)s',
+ {'x': str(repoeschema)}, ask_confirm=False)
+ elif not repospschema and espschema:
+ self.rqlexec('SET X specializes Y WHERE X is CWEType, X name %(x)s, '
+ 'Y is CWEType, Y name %(y)s',
+ {'x': str(repoeschema), 'y': str(espschema)},
+ ask_confirm=False)
+ self.rqlexecall(ss.updateeschema2rql(eschema, repoeschema.eid),
+ ask_confirm=self.verbosity >= 2)
+ if syncperms:
+ self._synchronize_permissions(eschema, repoeschema.eid)
+ if syncrdefs:
+ for rschema, targettypes, role in eschema.relation_definitions(True):
+ if rschema in VIRTUAL_RTYPES:
+ continue
+ if role == 'subject':
+ if not rschema in repoeschema.subject_relations():
+ continue
+ subjtypes, objtypes = [etype], targettypes
+ else: # role == 'object'
+ if not rschema in repoeschema.object_relations():
+ continue
+ subjtypes, objtypes = targettypes, [etype]
+ self._synchronize_rschema(rschema, syncrdefs=False,
+ syncprops=syncprops, syncperms=syncperms)
+ if rschema.rule: # rdef for computed rtype are infered hence should not be
+ # synchronized
+ continue
+ reporschema = self.repo.schema.rschema(rschema)
+ for subj in subjtypes:
+ for obj in objtypes:
+ if (subj, obj) not in reporschema.rdefs:
+ continue
+ self._synchronize_rdef_schema(subj, rschema, obj,
+ syncprops=syncprops, syncperms=syncperms)
+ if syncprops: # need to process __unique_together__ after rdefs were processed
+ # mappings from constraint name to columns
+ # filesystem (fs) and repository (repo) wise
+ fs = {}
+ repo = {}
+ for cols in eschema._unique_together or ():
+ fs[unique_index_name(repoeschema, cols)] = sorted(cols)
+ schemaentity = self.cnx.entity_from_eid(repoeschema.eid)
+ for entity in schemaentity.related('constraint_of', 'object',
+ targettypes=('CWUniqueTogetherConstraint',)).entities():
+ repo[entity.name] = sorted(rel.name for rel in entity.relations)
+ added = set(fs) - set(repo)
+ removed = set(repo) - set(fs)
+
+ for name in removed:
+ self.rqlexec('DELETE CWUniqueTogetherConstraint C WHERE C name %(name)s',
+ {'name': name})
+
+ def possible_unique_constraint(cols):
+ for name in cols:
+ rschema = repoeschema.subjrels.get(name)
+ if rschema is None:
+ print('dont add %s unique constraint on %s, missing %s' % (
+ ','.join(cols), eschema, name))
+ return False
+ if not (rschema.final or rschema.inlined):
+ print('dont add %s unique constraint on %s, %s is neither final nor inlined' % (
+ ','.join(cols), eschema, name))
+ return False
+ return True
+
+ for name in added:
+ if possible_unique_constraint(fs[name]):
+ rql, substs = ss._uniquetogether2rql(eschema, fs[name])
+ substs['x'] = repoeschema.eid
+ substs['name'] = name
+ self.rqlexec(rql, substs)
+
+ def _synchronize_rdef_schema(self, subjtype, rtype, objtype,
+ syncperms=True, syncprops=True):
+ """synchronize properties of the persistent relation definition schema
+ against its current definition:
+ * order and other properties
+ * constraints
+ * permissions
+ """
+ subjtype, objtype = str(subjtype), str(objtype)
+ rschema = self.fs_schema.rschema(rtype)
+ if rschema.rule:
+ raise ExecutionError('Cannot synchronize a relation definition for a '
+ 'computed relation (%s)' % rschema)
+ reporschema = self.repo.schema.rschema(rschema)
+ if (subjtype, rschema, objtype) in self._synchronized:
+ return
+ if syncperms and syncprops:
+ self._synchronized.add((subjtype, rschema, objtype))
+ if rschema.symmetric:
+ self._synchronized.add((objtype, rschema, subjtype))
+ rdef = rschema.rdef(subjtype, objtype)
+ if rdef.infered:
+ return # don't try to synchronize infered relation defs
+ repordef = reporschema.rdef(subjtype, objtype)
+ confirm = self.verbosity >= 2
+ if syncprops:
+ # properties
+ self.rqlexecall(ss.updaterdef2rql(rdef, repordef.eid),
+ ask_confirm=confirm)
+ # constraints
+ # 0. eliminate the set of unmodified constraints from the sets of
+ # old/new constraints
+ newconstraints = set(rdef.constraints)
+ oldconstraints = set(repordef.constraints)
+ unchanged_constraints = newconstraints & oldconstraints
+ newconstraints -= unchanged_constraints
+ oldconstraints -= unchanged_constraints
+ # 1. remove old constraints and update constraints of the same type
+ # NOTE: don't use rschema.constraint_by_type because it may be
+ # out of sync with newconstraints when multiple
+ # constraints of the same type are used
+ for cstr in oldconstraints:
+ self.rqlexec('DELETE CWConstraint C WHERE C eid %(x)s',
+ {'x': cstr.eid}, ask_confirm=confirm)
+ # 2. add new constraints
+ cstrtype_map = self.cstrtype_mapping()
+ self.rqlexecall(ss.constraints2rql(cstrtype_map, newconstraints,
+ repordef.eid),
+ ask_confirm=confirm)
+ if syncperms and not rschema in VIRTUAL_RTYPES:
+ self._synchronize_permissions(rdef, repordef.eid)
+
+ # base actions ############################################################
+
+ def checkpoint(self, ask_confirm=True):
+ """checkpoint action"""
+ if not ask_confirm or self.confirm('Commit now ?', shell=False):
+ self.commit()
+
+ def cmd_add_cube(self, cube, update_database=True):
+ self.cmd_add_cubes( (cube,), update_database)
+
+ def cmd_add_cubes(self, cubes, update_database=True):
+ """update_database is telling if the database schema should be updated
+ or if only the relevant eproperty should be inserted (for the case where
+ a cube has been extracted from an existing instance, so the
+ cube schema is already in there)
+ """
+ newcubes = super(ServerMigrationHelper, self).cmd_add_cubes(cubes)
+ if not newcubes:
+ return
+ for cube in newcubes:
+ self.cmd_set_property('system.version.'+cube,
+ self.config.cube_version(cube))
+ # ensure added cube is in config cubes
+ # XXX worth restoring on error?
+ if not cube in self.config._cubes:
+ self.config._cubes += (cube,)
+ if not update_database:
+ self.commit()
+ return
+ newcubes_schema = self.config.load_schema(construction_mode='non-strict')
+ # XXX we have to replace fs_schema, used in cmd_add_relation_type
+ # etc. and fsschema of migration script contexts
+ self.fs_schema = newcubes_schema
+ self.update_context('fsschema', self.fs_schema)
+ new = set()
+ # execute pre-create files
+ driver = self.repo.system_source.dbdriver
+ for cube in reversed(newcubes):
+ self.cmd_install_custom_sql_scripts(cube)
+ self.cmd_exec_event_script('precreate', cube)
+ # add new entity and relation types
+ for rschema in newcubes_schema.relations():
+ if not rschema in self.repo.schema:
+ self.cmd_add_relation_type(rschema.type)
+ new.add(rschema.type)
+ toadd = [eschema for eschema in newcubes_schema.entities()
+ if not eschema in self.repo.schema]
+ for eschema in order_eschemas(toadd):
+ self.cmd_add_entity_type(eschema.type)
+ new.add(eschema.type)
+ # check if attributes has been added to existing entities
+ for rschema in newcubes_schema.relations():
+ existingschema = self.repo.schema.rschema(rschema.type)
+ for (fromtype, totype) in rschema.rdefs:
+ # if rdef already exists or is infered from inheritance,
+ # don't add it
+ if (fromtype, totype) in existingschema.rdefs \
+ or rschema.rdefs[(fromtype, totype)].infered:
+ continue
+ # check we should actually add the relation definition
+ if not (fromtype in new or totype in new or rschema in new):
+ continue
+ self.cmd_add_relation_definition(str(fromtype), rschema.type,
+ str(totype))
+ # execute post-create files
+ for cube in reversed(newcubes):
+ with self.cnx.allow_all_hooks_but():
+ self.cmd_exec_event_script('postcreate', cube)
+ self.commit()
+
+ def cmd_drop_cube(self, cube, removedeps=False):
+ removedcubes = super(ServerMigrationHelper, self).cmd_drop_cube(
+ cube, removedeps)
+ if not removedcubes:
+ return
+ fsschema = self.fs_schema
+ removedcubes_schema = self.config.load_schema(construction_mode='non-strict')
+ reposchema = self.repo.schema
+ # execute pre-remove files
+ for cube in reversed(removedcubes):
+ self.cmd_exec_event_script('preremove', cube)
+ # remove cubes'entity and relation types
+ for rschema in fsschema.relations():
+ if not rschema in removedcubes_schema and rschema in reposchema:
+ self.cmd_drop_relation_type(rschema.type)
+ toremove = [eschema for eschema in fsschema.entities()
+ if not eschema in removedcubes_schema
+ and eschema in reposchema]
+ for eschema in reversed(order_eschemas(toremove)):
+ self.cmd_drop_entity_type(eschema.type)
+ for rschema in fsschema.relations():
+ if rschema in removedcubes_schema and rschema in reposchema:
+ # check if attributes/relations has been added to entities from
+ # other cubes
+ for fromtype, totype in rschema.rdefs:
+ if (fromtype, totype) not in removedcubes_schema[rschema.type].rdefs and \
+ (fromtype, totype) in reposchema[rschema.type].rdefs:
+ self.cmd_drop_relation_definition(
+ str(fromtype), rschema.type, str(totype))
+ # execute post-remove files
+ for cube in reversed(removedcubes):
+ self.cmd_exec_event_script('postremove', cube)
+ self.rqlexec('DELETE CWProperty X WHERE X pkey %(pk)s',
+ {'pk': u'system.version.'+cube}, ask_confirm=False)
+ self.commit()
+
+ # schema migration actions ################################################
+
+ def cmd_add_attribute(self, etype, attrname, attrtype=None, commit=True):
+ """add a new attribute on the given entity type"""
+ if attrtype is None:
+ rschema = self.fs_schema.rschema(attrname)
+ attrtype = rschema.objects(etype)[0]
+ self.cmd_add_relation_definition(etype, attrname, attrtype, commit=commit)
+
+ def cmd_drop_attribute(self, etype, attrname, commit=True):
+ """drop an existing attribute from the given entity type
+
+ `attrname` is a string giving the name of the attribute to drop
+ """
+ try:
+ rschema = self.repo.schema.rschema(attrname)
+ attrtype = rschema.objects(etype)[0]
+ except KeyError:
+ print('warning: attribute %s %s is not known, skip deletion' % (
+ etype, attrname))
+ else:
+ self.cmd_drop_relation_definition(etype, attrname, attrtype,
+ commit=commit)
+
+ def cmd_rename_attribute(self, etype, oldname, newname, commit=True):
+ """rename an existing attribute of the given entity type
+
+ `oldname` is a string giving the name of the existing attribute
+ `newname` is a string giving the name of the renamed attribute
+ """
+ eschema = self.fs_schema.eschema(etype)
+ attrtype = eschema.destination(newname)
+ # have to commit this first step anyway to get the definition
+ # actually in the schema
+ self.cmd_add_attribute(etype, newname, attrtype, commit=True)
+ # skipp NULL values if the attribute is required
+ rql = 'SET X %s VAL WHERE X is %s, X %s VAL' % (newname, etype, oldname)
+ card = eschema.rdef(newname).cardinality[0]
+ if card == '1':
+ rql += ', NOT X %s NULL' % oldname
+ self.rqlexec(rql, ask_confirm=self.verbosity>=2)
+ # XXX if both attributes fulltext indexed, should skip fti rebuild
+ # XXX if old attribute was fti indexed but not the new one old value
+ # won't be removed from the index (this occurs on other kind of
+ # fulltextindexed change...)
+ self.cmd_drop_attribute(etype, oldname, commit=commit)
+
+ def cmd_add_entity_type(self, etype, auto=True, commit=True):
+ """register a new entity type
+
+ in auto mode, automatically register entity's relation where the
+ targeted type is known
+ """
+ instschema = self.repo.schema
+ eschema = self.fs_schema.eschema(etype)
+ if etype in instschema and not (eschema.final and eschema.eid is None):
+ print('warning: %s already known, skip addition' % etype)
+ return
+ confirm = self.verbosity >= 2
+ groupmap = self.group_mapping()
+ cstrtypemap = self.cstrtype_mapping()
+ # register the entity into CWEType
+ execute = self.cnx.execute
+ if eschema.final and eschema not in instschema:
+ # final types are expected to be in the living schema by default, but they are not if
+ # the type is defined in a cube that is being added
+ edef = EntityType(eschema.type, __permissions__=eschema.permissions)
+ instschema.add_entity_type(edef)
+ ss.execschemarql(execute, eschema, ss.eschema2rql(eschema, groupmap))
+ # add specializes relation if needed
+ specialized = eschema.specializes()
+ if specialized:
+ try:
+ specialized.eid = instschema[specialized].eid
+ except KeyError:
+ raise ExecutionError('trying to add entity type but parent type is '
+ 'not yet in the database schema')
+ self.rqlexecall(ss.eschemaspecialize2rql(eschema), ask_confirm=confirm)
+ # register entity's attributes
+ for rschema, attrschema in eschema.attribute_definitions():
+ # ignore those meta relations, they will be automatically added
+ if rschema.type in META_RTYPES:
+ continue
+ if not attrschema.type in instschema:
+ self.cmd_add_entity_type(attrschema.type, False, False)
+ if not rschema.type in instschema:
+ # need to add the relation type and to commit to get it
+ # actually in the schema
+ self.cmd_add_relation_type(rschema.type, False, commit=True)
+ # register relation definition
+ rdef = self._get_rdef(rschema, eschema, eschema.destination(rschema))
+ ss.execschemarql(execute, rdef, ss.rdef2rql(rdef, cstrtypemap, groupmap),)
+ # take care to newly introduced base class
+ # XXX some part of this should probably be under the "if auto" block
+ for spschema in eschema.specialized_by(recursive=False):
+ try:
+ instspschema = instschema[spschema]
+ except KeyError:
+ # specialized entity type not in schema, ignore
+ continue
+ if instspschema.specializes() != eschema:
+ self.rqlexec('SET D specializes P WHERE D eid %(d)s, P name %(pn)s',
+ {'d': instspschema.eid, 'pn': eschema.type},
+ ask_confirm=confirm)
+ for rschema, tschemas, role in spschema.relation_definitions(True):
+ for tschema in tschemas:
+ if not tschema in instschema:
+ continue
+ if role == 'subject':
+ subjschema = spschema
+ objschema = tschema
+ if rschema.final and rschema in instspschema.subjrels:
+ # attribute already set, has_rdef would check if
+ # it's of the same type, we don't want this so
+ # simply skip here
+ continue
+ elif role == 'object':
+ subjschema = tschema
+ objschema = spschema
+ if (rschema.rdef(subjschema, objschema).infered
+ or (instschema.has_relation(rschema) and
+ (subjschema, objschema) in instschema[rschema].rdefs)):
+ continue
+ self.cmd_add_relation_definition(
+ subjschema.type, rschema.type, objschema.type)
+ if auto:
+ # we have commit here to get relation types actually in the schema
+ self.commit()
+ added = []
+ for rschema in eschema.subject_relations():
+ # attribute relation have already been processed and
+ # 'owned_by'/'created_by' will be automatically added
+ if rschema.final or rschema.type in META_RTYPES:
+ continue
+ rtypeadded = rschema.type in instschema
+ for targetschema in rschema.objects(etype):
+ # ignore relations where the targeted type is not in the
+ # current instance schema
+ targettype = targetschema.type
+ if not targettype in instschema and targettype != etype:
+ continue
+ if not rtypeadded:
+ # need to add the relation type and to commit to get it
+ # actually in the schema
+ added.append(rschema.type)
+ self.cmd_add_relation_type(rschema.type, False, commit=True)
+ rtypeadded = True
+ # register relation definition
+ # remember this two avoid adding twice non symmetric relation
+ # such as "Emailthread forked_from Emailthread"
+ added.append((etype, rschema.type, targettype))
+ rdef = self._get_rdef(rschema, eschema, targetschema)
+ ss.execschemarql(execute, rdef,
+ ss.rdef2rql(rdef, cstrtypemap, groupmap))
+ for rschema in eschema.object_relations():
+ if rschema.type in META_RTYPES:
+ continue
+ rtypeadded = rschema.type in instschema or rschema.type in added
+ for targetschema in rschema.subjects(etype):
+ # ignore relations where the targeted type is not in the
+ # current instance schema
+ targettype = targetschema.type
+ # don't check targettype != etype since in this case the
+ # relation has already been added as a subject relation
+ if not targettype in instschema:
+ continue
+ if not rtypeadded:
+ # need to add the relation type and to commit to get it
+ # actually in the schema
+ self.cmd_add_relation_type(rschema.type, False, commit=True)
+ rtypeadded = True
+ elif (targettype, rschema.type, etype) in added:
+ continue
+ # register relation definition
+ rdef = self._get_rdef(rschema, targetschema, eschema)
+ ss.execschemarql(execute, rdef,
+ ss.rdef2rql(rdef, cstrtypemap, groupmap))
+ if commit:
+ self.commit()
+
+ def cmd_drop_entity_type(self, etype, commit=True):
+ """Drop an existing entity type.
+
+ This will trigger deletion of necessary relation types and definitions.
+ Note that existing entities of the given type will be deleted without
+ any hooks called.
+ """
+ # XXX what if we delete an entity type which is specialized by other types
+ # unregister the entity from CWEType
+ self.rqlexec('DELETE CWEType X WHERE X name %(etype)s', {'etype': etype},
+ ask_confirm=self.verbosity>=2)
+ if commit:
+ self.commit()
+
+ def cmd_rename_entity_type(self, oldname, newname, attrs=None, commit=True):
+ """rename an existing entity type in the persistent schema
+
+ `oldname` is a string giving the name of the existing entity type
+ `newname` is a string giving the name of the renamed entity type
+ """
+ schema = self.repo.schema
+ if oldname not in schema:
+ print('warning: entity type %s is unknown, skip renaming' % oldname)
+ return
+ # if merging two existing entity types
+ if newname in schema:
+ assert oldname in ETYPE_NAME_MAP, \
+ '%s should be mapped to %s in ETYPE_NAME_MAP' % (oldname,
+ newname)
+ if attrs is None:
+ attrs = ','.join(SQL_PREFIX + rschema.type
+ for rschema in schema[newname].subject_relations()
+ if (rschema.final or rschema.inlined)
+ and not rschema in PURE_VIRTUAL_RTYPES)
+ else:
+ attrs += ('eid', 'creation_date', 'modification_date', 'cwuri')
+ attrs = ','.join(SQL_PREFIX + attr for attr in attrs)
+ self.sqlexec('INSERT INTO %s%s(%s) SELECT %s FROM %s%s' % (
+ SQL_PREFIX, newname, attrs, attrs, SQL_PREFIX, oldname),
+ ask_confirm=False)
+ # old entity type has not been added to the schema, can't gather it
+ new = schema.eschema(newname)
+ oldeid = self.rqlexec('CWEType ET WHERE ET name %(on)s',
+ {'on': oldname}, ask_confirm=False)[0][0]
+ # backport old type relations to new type
+ # XXX workflows, other relations?
+ for r1, rr1 in [('from_entity', 'to_entity'),
+ ('to_entity', 'from_entity')]:
+ self.rqlexec('SET X %(r1)s NET WHERE X %(r1)s OET, '
+ 'NOT EXISTS(X2 %(r1)s NET, X relation_type XRT, '
+ 'X2 relation_type XRT, X %(rr1)s XTE, X2 %(rr1)s XTE), '
+ 'OET eid %%(o)s, NET eid %%(n)s' % locals(),
+ {'o': oldeid, 'n': new.eid}, ask_confirm=False)
+ # backport is / is_instance_of relation to new type
+ for rtype in ('is', 'is_instance_of'):
+ self.sqlexec('UPDATE %s_relation SET eid_to=%s WHERE eid_to=%s'
+ % (rtype, new.eid, oldeid), ask_confirm=False)
+ # delete relations using SQL to avoid relations content removal
+ # triggered by schema synchronization hooks.
+ for rdeftype in ('CWRelation', 'CWAttribute'):
+ thispending = set( (eid for eid, in self.sqlexec(
+ 'SELECT cw_eid FROM cw_%s WHERE cw_from_entity=%%(eid)s OR '
+ ' cw_to_entity=%%(eid)s' % rdeftype,
+ {'eid': oldeid}, ask_confirm=False)) )
+ # we should add deleted eids into pending eids else we may
+ # get some validation error on commit since integrity hooks
+ # may think some required relation is missing... This also ensure
+ # repository caches are properly cleanup
+ hook.CleanupDeletedEidsCacheOp.get_instance(self.cnx).union(thispending)
+ # and don't forget to remove record from system tables
+ entities = [self.cnx.entity_from_eid(eid, rdeftype) for eid in thispending]
+ self.repo.system_source.delete_info_multi(self.cnx, entities)
+ self.sqlexec('DELETE FROM cw_%s WHERE cw_from_entity=%%(eid)s OR '
+ 'cw_to_entity=%%(eid)s' % rdeftype,
+ {'eid': oldeid}, ask_confirm=False)
+ # now we have to manually cleanup relations pointing to deleted
+ # entities
+ thiseids = ','.join(str(eid) for eid in thispending)
+ for rschema, ttypes, role in schema[rdeftype].relation_definitions():
+ if rschema.type in VIRTUAL_RTYPES:
+ continue
+ sqls = []
+ if role == 'object':
+ if rschema.inlined:
+ for eschema in ttypes:
+ sqls.append('DELETE FROM cw_%s WHERE cw_%s IN(%%s)'
+ % (eschema, rschema))
+ else:
+ sqls.append('DELETE FROM %s_relation WHERE eid_to IN(%%s)'
+ % rschema)
+ elif not rschema.inlined:
+ sqls.append('DELETE FROM %s_relation WHERE eid_from IN(%%s)'
+ % rschema)
+ for sql in sqls:
+ self.sqlexec(sql % thiseids, ask_confirm=False)
+ # remove the old type: use rql to propagate deletion
+ self.rqlexec('DELETE CWEType ET WHERE ET name %(on)s', {'on': oldname},
+ ask_confirm=False)
+ # elif simply renaming an entity type
+ else:
+ self.rqlexec('SET ET name %(newname)s WHERE ET is CWEType, ET name %(on)s',
+ {'newname' : text_type(newname), 'on' : oldname},
+ ask_confirm=False)
+ if commit:
+ self.commit()
+
+ def cmd_add_relation_type(self, rtype, addrdef=True, commit=True):
+ """register a new relation type named `rtype`, as described in the
+ schema description file.
+
+ `addrdef` is a boolean value; when True, it will also add all relations
+ of the type just added found in the schema definition file. Note that it
+ implies an intermediate "commit" which commits the relation type
+ creation (but not the relation definitions themselves, for which
+ committing depends on the `commit` argument value).
+
+ """
+ reposchema = self.repo.schema
+ rschema = self.fs_schema.rschema(rtype)
+ execute = self.cnx.execute
+ if rtype in reposchema:
+ print('warning: relation type %s is already known, skip addition' % (
+ rtype))
+ elif rschema.rule:
+ gmap = self.group_mapping()
+ ss.execschemarql(execute, rschema, ss.crschema2rql(rschema, gmap))
+ else:
+ # register the relation into CWRType and insert necessary relation
+ # definitions
+ ss.execschemarql(execute, rschema, ss.rschema2rql(rschema, addrdef=False))
+ if not rschema.rule and addrdef:
+ self.commit()
+ gmap = self.group_mapping()
+ cmap = self.cstrtype_mapping()
+ done = set()
+ for subj, obj in rschema.rdefs:
+ if not (reposchema.has_entity(subj)
+ and reposchema.has_entity(obj)):
+ continue
+ # symmetric relations appears twice
+ if (subj, obj) in done:
+ continue
+ done.add( (subj, obj) )
+ self.cmd_add_relation_definition(subj, rtype, obj)
+ if rtype in META_RTYPES:
+ # if the relation is in META_RTYPES, ensure we're adding it for
+ # all entity types *in the persistent schema*, not only those in
+ # the fs schema
+ for etype in self.repo.schema.entities():
+ if not etype in self.fs_schema:
+ # get sample object type and rproperties
+ objtypes = rschema.objects()
+ assert len(objtypes) == 1, objtypes
+ objtype = objtypes[0]
+ rdef = copy(rschema.rdef(rschema.subjects(objtype)[0], objtype))
+ rdef.subject = etype
+ rdef.rtype = self.repo.schema.rschema(rschema)
+ rdef.object = self.repo.schema.eschema(objtype)
+ ss.execschemarql(execute, rdef,
+ ss.rdef2rql(rdef, cmap, gmap))
+ if commit:
+ self.commit()
+
+ def cmd_drop_relation_type(self, rtype, commit=True):
+ """Drop an existing relation type.
+
+ Note that existing relations of the given type will be deleted without
+ any hooks called.
+ """
+ self.rqlexec('DELETE CWRType X WHERE X name %r' % rtype,
+ ask_confirm=self.verbosity>=2)
+ self.rqlexec('DELETE CWComputedRType X WHERE X name %r' % rtype,
+ ask_confirm=self.verbosity>=2)
+ if commit:
+ self.commit()
+
+ def cmd_rename_relation_type(self, oldname, newname, commit=True, force=False):
+ """rename an existing relation
+
+ `oldname` is a string giving the name of the existing relation
+ `newname` is a string giving the name of the renamed relation
+
+ If `force` is True, proceed even if `oldname` still appears in the fs schema
+ """
+ if oldname in self.fs_schema and not force:
+ if not self.confirm('Relation %s is still present in the filesystem schema,'
+ ' do you really want to drop it?' % oldname,
+ default='n'):
+ return
+ self.cmd_add_relation_type(newname, commit=True)
+ if not self.repo.schema[oldname].rule:
+ self.rqlexec('SET X %s Y WHERE X %s Y' % (newname, oldname),
+ ask_confirm=self.verbosity>=2)
+ self.cmd_drop_relation_type(oldname, commit=commit)
+
+ def cmd_add_relation_definition(self, subjtype, rtype, objtype, commit=True):
+ """register a new relation definition, from its definition found in the
+ schema definition file
+ """
+ rschema = self.fs_schema.rschema(rtype)
+ if rschema.rule:
+ raise ExecutionError('Cannot add a relation definition for a '
+ 'computed relation (%s)' % rschema)
+ if not rtype in self.repo.schema:
+ self.cmd_add_relation_type(rtype, addrdef=False, commit=True)
+ if (subjtype, objtype) in self.repo.schema.rschema(rtype).rdefs:
+ print('warning: relation %s %s %s is already known, skip addition' % (
+ subjtype, rtype, objtype))
+ return
+ rdef = self._get_rdef(rschema, subjtype, objtype)
+ ss.execschemarql(self.cnx.execute, rdef,
+ ss.rdef2rql(rdef, self.cstrtype_mapping(),
+ self.group_mapping()))
+ if commit:
+ self.commit()
+
+ def _get_rdef(self, rschema, subjtype, objtype):
+ return self._set_rdef_eid(rschema.rdefs[(subjtype, objtype)])
+
+ def _set_rdef_eid(self, rdef):
+ for attr in ('rtype', 'subject', 'object'):
+ schemaobj = getattr(rdef, attr)
+ if getattr(schemaobj, 'eid', None) is None:
+ schemaobj.eid = self.repo.schema[schemaobj].eid
+ assert schemaobj.eid is not None, schemaobj
+ return rdef
+
+ def cmd_drop_relation_definition(self, subjtype, rtype, objtype, commit=True):
+ """Drop an existing relation definition.
+
+ Note that existing relations of the given definition will be deleted
+ without any hooks called.
+ """
+ rschema = self.repo.schema.rschema(rtype)
+ if rschema.rule:
+ raise ExecutionError('Cannot drop a relation definition for a '
+ 'computed relation (%s)' % rschema)
+ # unregister the definition from CWAttribute or CWRelation
+ if rschema.final:
+ etype = 'CWAttribute'
+ else:
+ etype = 'CWRelation'
+ rql = ('DELETE %s X WHERE X from_entity FE, FE name "%s",'
+ 'X relation_type RT, RT name "%s", X to_entity TE, TE name "%s"')
+ self.rqlexec(rql % (etype, subjtype, rtype, objtype),
+ ask_confirm=self.verbosity>=2)
+ if commit:
+ self.commit()
+
+ def cmd_sync_schema_props_perms(self, ertype=None, syncperms=True,
+ syncprops=True, syncrdefs=True, commit=True):
+ """synchronize the persistent schema against the current definition
+ schema.
+
+ `ertype` can be :
+ - None, in that case everything will be synced ;
+ - a string, it should be an entity type or
+ a relation type. In that case, only the corresponding
+ entities / relations will be synced ;
+ - an rdef object to synchronize only this specific relation definition
+
+ It will synch common stuff between the definition schema and the
+ actual persistent schema, it won't add/remove any entity or relation.
+ """
+ assert syncperms or syncprops, 'nothing to do'
+ if ertype is not None:
+ if isinstance(ertype, RelationDefinitionSchema):
+ ertype = ertype.as_triple()
+ if isinstance(ertype, (tuple, list)):
+ assert len(ertype) == 3, 'not a relation definition'
+ self._synchronize_rdef_schema(ertype[0], ertype[1], ertype[2],
+ syncperms=syncperms,
+ syncprops=syncprops)
+ else:
+ erschema = self.repo.schema[ertype]
+ if isinstance(erschema, CubicWebRelationSchema):
+ self._synchronize_rschema(erschema, syncrdefs=syncrdefs,
+ syncperms=syncperms,
+ syncprops=syncprops)
+ else:
+ self._synchronize_eschema(erschema, syncrdefs=syncrdefs,
+ syncperms=syncperms,
+ syncprops=syncprops)
+ else:
+ for etype in self.repo.schema.entities():
+ if etype.eid is None:
+ # not yet added final etype (thing to BigInt defined in
+ # yams though 3.13 migration not done yet)
+ continue
+ self._synchronize_eschema(etype, syncrdefs=syncrdefs,
+ syncprops=syncprops, syncperms=syncperms)
+ if commit:
+ self.commit()
+
+ def cmd_change_relation_props(self, subjtype, rtype, objtype,
+ commit=True, **kwargs):
+ """change some properties of a relation definition
+
+ you usually want to use sync_schema_props_perms instead.
+ """
+ assert kwargs
+ restriction = []
+ if subjtype and subjtype != 'Any':
+ restriction.append('X from_entity FE, FE name "%s"' % subjtype)
+ if objtype and objtype != 'Any':
+ restriction.append('X to_entity TE, TE name "%s"' % objtype)
+ if rtype and rtype != 'Any':
+ restriction.append('X relation_type RT, RT name "%s"' % rtype)
+ assert restriction
+ values = []
+ for k, v in kwargs.items():
+ values.append('X %s %%(%s)s' % (k, k))
+ if PY2 and isinstance(v, str):
+ kwargs[k] = unicode(v)
+ rql = 'SET %s WHERE %s' % (','.join(values), ','.join(restriction))
+ self.rqlexec(rql, kwargs, ask_confirm=self.verbosity>=2)
+ if commit:
+ self.commit()
+
+ def cmd_set_size_constraint(self, etype, rtype, size, commit=True):
+ """set change size constraint of a string attribute
+
+ if size is None any size constraint will be removed.
+
+ you usually want to use sync_schema_props_perms instead.
+ """
+ oldvalue = None
+ for constr in self.repo.schema.eschema(etype).rdef(rtype).constraints:
+ if isinstance(constr, SizeConstraint):
+ oldvalue = constr.max
+ if oldvalue == size:
+ return
+ if oldvalue is None and not size is None:
+ ceid = self.rqlexec('INSERT CWConstraint C: C value %(v)s, C cstrtype CT '
+ 'WHERE CT name "SizeConstraint"',
+ {'v': SizeConstraint(size).serialize()},
+ ask_confirm=self.verbosity>=2)[0][0]
+ self.rqlexec('SET X constrained_by C WHERE X from_entity S, X relation_type R, '
+ 'S name "%s", R name "%s", C eid %s' % (etype, rtype, ceid),
+ ask_confirm=self.verbosity>=2)
+ elif not oldvalue is None:
+ if not size is None:
+ self.rqlexec('SET C value %%(v)s WHERE X from_entity S, X relation_type R,'
+ 'X constrained_by C, C cstrtype CT, CT name "SizeConstraint",'
+ 'S name "%s", R name "%s"' % (etype, rtype),
+ {'v': text_type(SizeConstraint(size).serialize())},
+ ask_confirm=self.verbosity>=2)
+ else:
+ self.rqlexec('DELETE X constrained_by C WHERE X from_entity S, X relation_type R,'
+ 'X constrained_by C, C cstrtype CT, CT name "SizeConstraint",'
+ 'S name "%s", R name "%s"' % (etype, rtype),
+ ask_confirm=self.verbosity>=2)
+ # cleanup unused constraints
+ self.rqlexec('DELETE CWConstraint C WHERE NOT X constrained_by C')
+ if commit:
+ self.commit()
+
+ # Workflows handling ######################################################
+
+ def cmd_make_workflowable(self, etype):
+ """add workflow relations to an entity type to make it workflowable"""
+ self.cmd_add_relation_definition(etype, 'in_state', 'State')
+ self.cmd_add_relation_definition(etype, 'custom_workflow', 'Workflow')
+ self.cmd_add_relation_definition('TrInfo', 'wf_info_for', etype)
+
+ def cmd_add_workflow(self, name, wfof, default=True, commit=False,
+ ensure_workflowable=True, **kwargs):
+ """
+ create a new workflow and links it to entity types
+ :type name: unicode
+ :param name: name of the workflow
+
+ :type wfof: string or list/tuple of strings
+ :param wfof: entity type(s) having this workflow
+
+ :type default: bool
+ :param default: tells wether this is the default workflow
+ for the specified entity type(s); set it to false in
+ the case of a subworkflow
+
+ :rtype: `Workflow`
+ """
+ wf = self.cmd_create_entity('Workflow', name=text_type(name),
+ **kwargs)
+ if not isinstance(wfof, (list, tuple)):
+ wfof = (wfof,)
+ def _missing_wf_rel(etype):
+ return 'missing workflow relations, see make_workflowable(%s)' % etype
+ for etype in wfof:
+ eschema = self.repo.schema[etype]
+ etype = text_type(etype)
+ if ensure_workflowable:
+ assert 'in_state' in eschema.subjrels, _missing_wf_rel(etype)
+ assert 'custom_workflow' in eschema.subjrels, _missing_wf_rel(etype)
+ assert 'wf_info_for' in eschema.objrels, _missing_wf_rel(etype)
+ rset = self.rqlexec(
+ 'SET X workflow_of ET WHERE X eid %(x)s, ET name %(et)s',
+ {'x': wf.eid, 'et': text_type(etype)}, ask_confirm=False)
+ assert rset, 'unexistant entity type %s' % etype
+ if default:
+ self.rqlexec(
+ 'SET ET default_workflow X WHERE X eid %(x)s, ET name %(et)s',
+ {'x': wf.eid, 'et': text_type(etype)}, ask_confirm=False)
+ if commit:
+ self.commit()
+ return wf
+
+ def cmd_get_workflow_for(self, etype):
+ """return default workflow for the given entity type"""
+ rset = self.rqlexec('Workflow X WHERE ET default_workflow X, ET name %(et)s',
+ {'et': etype})
+ return rset.get_entity(0, 0)
+
+ # CWProperty handling ######################################################
+
+ def cmd_property_value(self, pkey):
+ """retreive the site-wide persistent property value for the given key.
+
+ To get a user specific property value, use appropriate method on CWUser
+ instance.
+ """
+ rset = self.rqlexec(
+ 'Any V WHERE X is CWProperty, X pkey %(k)s, X value V, NOT X for_user U',
+ {'k': pkey}, ask_confirm=False)
+ return rset[0][0]
+
+ def cmd_set_property(self, pkey, value):
+ """set the site-wide persistent property value for the given key to the
+ given value.
+
+ To set a user specific property value, use appropriate method on CWUser
+ instance.
+ """
+ value = text_type(value)
+ try:
+ prop = self.rqlexec(
+ 'CWProperty X WHERE X pkey %(k)s, NOT X for_user U',
+ {'k': text_type(pkey)}, ask_confirm=False).get_entity(0, 0)
+ except Exception:
+ self.cmd_create_entity('CWProperty', pkey=text_type(pkey), value=value)
+ else:
+ prop.cw_set(value=value)
+
+ # other data migration commands ###########################################
+
+ def cmd_storage_changed(self, etype, attribute):
+ """migrate entities to a custom storage. The new storage is expected to
+ be set, it will be temporarily removed for the migration.
+ """
+ from logilab.common.shellutils import ProgressBar
+ source = self.repo.system_source
+ storage = source.storage(etype, attribute)
+ source.unset_storage(etype, attribute)
+ rset = self.rqlexec('Any X WHERE X is %s' % etype, ask_confirm=False)
+ pb = ProgressBar(len(rset))
+ for entity in rset.entities():
+ # fill cache. Do not fetch that attribute using the global rql query
+ # since we may exhaust memory doing that....
+ getattr(entity, attribute)
+ storage.migrate_entity(entity, attribute)
+ # remove from entity cache to avoid memory exhaustion
+ del entity.cw_attr_cache[attribute]
+ pb.update()
+ print()
+ source.set_storage(etype, attribute, storage)
+
+ def cmd_create_entity(self, etype, commit=False, **kwargs):
+ """add a new entity of the given type"""
+ entity = self.cnx.create_entity(etype, **kwargs)
+ if commit:
+ self.commit()
+ return entity
+
+ def cmd_find(self, etype, **kwargs):
+ """find entities of the given type and attribute values"""
+ return self.cnx.find(etype, **kwargs)
+
+ @deprecated("[3.19] use find(*args, **kwargs).entities() instead")
+ def cmd_find_entities(self, etype, **kwargs):
+ """find entities of the given type and attribute values"""
+ return self.cnx.find(etype, **kwargs).entities()
+
+ @deprecated("[3.19] use find(*args, **kwargs).one() instead")
+ def cmd_find_one_entity(self, etype, **kwargs):
+ """find one entity of the given type and attribute values.
+
+ raise :exc:`cubicweb.req.FindEntityError` if can not return one and only
+ one entity.
+ """
+ return self.cnx.find(etype, **kwargs).one()
+
+ def cmd_update_etype_fti_weight(self, etype, weight):
+ if self.repo.system_source.dbdriver == 'postgres':
+ self.sqlexec('UPDATE appears SET weight=%(weight)s '
+ 'FROM entities as X '
+ 'WHERE X.eid=appears.uid AND X.type=%(type)s',
+ {'type': etype, 'weight': weight}, ask_confirm=False)
+
+ def cmd_reindex_entities(self, etypes=None):
+ """force reindexaction of entities of the given types or of all
+ indexable entity types
+ """
+ from cubicweb.server.checkintegrity import reindex_entities
+ reindex_entities(self.repo.schema, self.cnx, etypes=etypes)
+
+ @contextmanager
+ def cmd_dropped_constraints(self, etype, attrname, cstrtype=None,
+ droprequired=False):
+ """context manager to drop constraints temporarily on fs_schema
+
+ `cstrtype` should be a constraint class (or a tuple of classes)
+ and will be passed to isinstance directly
+
+ For instance::
+
+ >>> with dropped_constraints('MyType', 'myattr',
+ ... UniqueConstraint, droprequired=True):
+ ... add_attribute('MyType', 'myattr')
+ ... # + instructions to fill MyType.myattr column
+ ...
+ >>>
+
+ """
+ rdef = self.fs_schema.eschema(etype).rdef(attrname)
+ original_constraints = rdef.constraints
+ # remove constraints
+ if cstrtype:
+ rdef.constraints = [cstr for cstr in original_constraints
+ if not (cstrtype and isinstance(cstr, cstrtype))]
+ if droprequired:
+ original_cardinality = rdef.cardinality
+ rdef.cardinality = '?' + rdef.cardinality[1]
+ yield
+ # restore original constraints
+ rdef.constraints = original_constraints
+ if droprequired:
+ rdef.cardinality = original_cardinality
+ # update repository schema
+ self.cmd_sync_schema_props_perms(rdef, syncperms=False)
+
+ def sqlexec(self, sql, args=None, ask_confirm=True):
+ """execute the given sql if confirmed
+
+ should only be used for low level stuff undoable with existing higher
+ level actions
+ """
+ if not ask_confirm or self.confirm('Execute sql: %s ?' % sql):
+ try:
+ cu = self.cnx.system_sql(sql, args)
+ except Exception:
+ ex = sys.exc_info()[1]
+ if self.confirm('Error: %s\nabort?' % ex, pdb=True):
+ raise
+ return
+ try:
+ return cu.fetchall()
+ except Exception:
+ # no result to fetch
+ return
+
+ def rqlexec(self, rql, kwargs=None, build_descr=True,
+ ask_confirm=False):
+ """rql action"""
+ if not isinstance(rql, (tuple, list)):
+ rql = ( (rql, kwargs), )
+ res = None
+ execute = self.cnx.execute
+ for rql, kwargs in rql:
+ if kwargs:
+ msg = '%s (%s)' % (rql, kwargs)
+ else:
+ msg = rql
+ if not ask_confirm or self.confirm('Execute rql: %s ?' % msg):
+ try:
+ res = execute(rql, kwargs, build_descr=build_descr)
+ except Exception as ex:
+ if self.confirm('Error: %s\nabort?' % ex, pdb=True):
+ raise
+ return res
+
+ def rqliter(self, rql, kwargs=None, ask_confirm=True):
+ return ForRqlIterator(self, rql, kwargs, ask_confirm)
+
+ # low-level commands to repair broken system database ######################
+
+ def cmd_change_attribute_type(self, etype, attr, newtype, commit=True):
+ """low level method to change the type of an entity attribute. This is
+ a quick hack which has some drawback:
+ * only works when the old type can be changed to the new type by the
+ underlying rdbms (eg using ALTER TABLE)
+ * the actual schema won't be updated until next startup
+ """
+ rschema = self.repo.schema.rschema(attr)
+ oldschema = rschema.objects(etype)[0]
+ rdef = rschema.rdef(etype, oldschema)
+ sql = ("UPDATE cw_CWAttribute "
+ "SET cw_to_entity=(SELECT cw_eid FROM cw_CWEType WHERE cw_name='%s')"
+ "WHERE cw_eid=%s") % (newtype, rdef.eid)
+ self.sqlexec(sql, ask_confirm=False)
+ dbhelper = self.repo.system_source.dbhelper
+ newrdef = self.fs_schema.rschema(attr).rdef(etype, newtype)
+ sqltype = sql_type(dbhelper, newrdef)
+ cursor = self.cnx.cnxset.cu
+ # consider former cardinality by design, since cardinality change is not handled here
+ allownull = rdef.cardinality[0] != '1'
+ dbhelper.change_col_type(cursor, 'cw_%s' % etype, 'cw_%s' % attr, sqltype, allownull)
+ if commit:
+ self.commit()
+ # manually update live schema
+ eschema = self.repo.schema[etype]
+ rschema._subj_schemas[eschema].remove(oldschema)
+ rschema._obj_schemas[oldschema].remove(eschema)
+ newschema = self.repo.schema[newtype]
+ rschema._update(eschema, newschema)
+ rdef.object = newschema
+ del rschema.rdefs[(eschema, oldschema)]
+ rschema.rdefs[(eschema, newschema)] = rdef
+
+ def cmd_add_entity_type_table(self, etype, commit=True):
+ """low level method to create the sql table for an existing entity.
+ This may be useful on accidental desync between the repository schema
+ and a sql database
+ """
+ dbhelper = self.repo.system_source.dbhelper
+ tablesql = eschema2sql(dbhelper, self.repo.schema.eschema(etype),
+ prefix=SQL_PREFIX)
+ for sql in tablesql.split(';'):
+ if sql.strip():
+ self.sqlexec(sql)
+ if commit:
+ self.commit()
+
+ def cmd_add_relation_type_table(self, rtype, commit=True):
+ """low level method to create the sql table for an existing relation.
+ This may be useful on accidental desync between the repository schema
+ and a sql database
+ """
+ tablesql = rschema2sql(self.repo.schema.rschema(rtype))
+ for sql in tablesql.split(';'):
+ if sql.strip():
+ self.sqlexec(sql)
+ if commit:
+ self.commit()
+
+ @deprecated("[3.15] use rename_relation_type(oldname, newname)")
+ def cmd_rename_relation(self, oldname, newname, commit=True):
+ self.cmd_rename_relation_type(oldname, newname, commit)
+
+
+class ForRqlIterator:
+ """specific rql iterator to make the loop skipable"""
+ def __init__(self, helper, rql, kwargs, ask_confirm):
+ self._h = helper
+ self.rql = rql
+ self.kwargs = kwargs
+ self.ask_confirm = ask_confirm
+ self._rsetit = None
+
+ def __iter__(self):
+ return self
+
+ def _get_rset(self):
+ rql, kwargs = self.rql, self.kwargs
+ if kwargs:
+ msg = '%s (%s)' % (rql, kwargs)
+ else:
+ msg = rql
+ if self.ask_confirm:
+ if not self._h.confirm('Execute rql: %s ?' % msg):
+ raise StopIteration
+ try:
+ return self._h._cw.execute(rql, kwargs)
+ except Exception as ex:
+ if self._h.confirm('Error: %s\nabort?' % ex):
+ raise
+ else:
+ raise StopIteration
+
+ def __next__(self):
+ if self._rsetit is not None:
+ return next(self._rsetit)
+ rset = self._get_rset()
+ self._rsetit = iter(rset)
+ return next(self._rsetit)
+
+ next = __next__
+
+ def entities(self):
+ try:
+ rset = self._get_rset()
+ except StopIteration:
+ return []
+ return rset.entities()