action, view and service so managers can start source synchronization from the web ui
Closes #5474286
# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr## This file is part of CubicWeb.## CubicWeb is free software: you can redistribute it and/or modify it under the# terms of the GNU Lesser General Public License as published by the Free# Software Foundation, either version 2.1 of the License, or (at your option)# any later version.## CubicWeb is distributed in the hope that it will be useful, but WITHOUT# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more# details.## You should have received a copy of the GNU Lesser General Public License along# with CubicWeb. If not, see <http://www.gnu.org/licenses/>."""allways executed before all others in server migrationit should only include low level schema changes"""from__future__importprint_functionfromsiximporttext_typefromcubicwebimportConfigurationErrorfromcubicweb.server.sessionimporthooks_controlfromcubicweb.serverimportschemaserialasssapplcubicwebversion,cubicwebversion=versions_map['cubicweb']def_add_relation_definition_no_perms(subjtype,rtype,objtype):rschema=fsschema.rschema(rtype)rdef=rschema.rdefs[(subjtype,objtype)]rdef.rtype=schema.rschema(rtype)rdef.subject=schema.eschema(subjtype)rdef.object=schema.eschema(objtype)ss.execschemarql(rql,rdef,ss.rdef2rql(rdef,CSTRMAP,groupmap=None))commit(ask_confirm=False)defreplace_eid_sequence_with_eid_numrange(session):dbh=session.repo.system_source.dbhelpercursor=session.cnxset.cutry:cursor.execute(dbh.sql_sequence_current_state('entities_id_seq'))lasteid=cursor.fetchone()[0]except:# programming error, already migratedreturncursor.execute(dbh.sql_drop_sequence('entities_id_seq'))cursor.execute(dbh.sql_create_numrange('entities_id_seq'))cursor.execute(dbh.sql_restart_numrange('entities_id_seq',initial_value=lasteid))session.commit()ifapplcubicwebversion<=(3,13,0)andcubicwebversion>=(3,13,1):sql('ALTER TABLE entities ADD asource VARCHAR(64)')sql('UPDATE entities SET asource=cw_name ''FROM cw_CWSource, cw_source_relation ''WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid')commit()ifapplcubicwebversion<=(3,14,4)andcubicwebversion>=(3,14,4):fromcubicweb.serverimportschema2sqlasy2sqldbhelper=repo.system_source.dbhelperrdefdef=schema['CWSource'].rdef('name')attrtype=y2sql.type_from_constraints(dbhelper,rdefdef.object,rdefdef.constraints).split()[0]cursor=session.cnxset.cusql('UPDATE entities SET asource = source WHERE asource is NULL')dbhelper.change_col_type(cursor,'entities','asource',attrtype,False)dbhelper.change_col_type(cursor,'entities','source',attrtype,False)# we now have a functional asource column, start using the normal eid_type_source methodifrepo.system_source.eid_type_source==repo.system_source.eid_type_source_pre_131:delrepo.system_source.eid_type_sourceifapplcubicwebversion<(3,19,0)andcubicwebversion>=(3,19,0):try:# need explicit drop of the indexes on some database systems (sqlserver)sql(repo.system_source.dbhelper.sql_drop_index('entities','mtime'))sql('ALTER TABLE "entities" DROP COLUMN "mtime"')sql('ALTER TABLE "entities" DROP COLUMN "source"')except:# programming error, already migratedprint("Failed to drop mtime or source database columns")print("'entities' table of the database has probably been already updated")commit()replace_eid_sequence_with_eid_numrange(session)ifapplcubicwebversion<(3,20,0)andcubicwebversion>=(3,20,0):ss._IGNORED_PROPS.append('formula')add_attribute('CWAttribute','formula',commit=False)ss._IGNORED_PROPS.remove('formula')commit()add_entity_type('CWComputedRType')commit()ifschema['TZDatetime'].eidisNone:add_entity_type('TZDatetime',auto=False)ifschema['TZTime'].eidisNone:add_entity_type('TZTime',auto=False)ifapplcubicwebversion<(3,18,0)andcubicwebversion>=(3,18,0):driver=config.system_source_config['db-driver']ifnot(driver=='postgres'ordriver.startswith('sqlserver')):importsysprint('This migration is not supported for backends other than sqlserver or postgres (yet).',file=sys.stderr)sys.exit(1)add_relation_definition('CWAttribute','add_permission','CWGroup')add_relation_definition('CWAttribute','add_permission','RQLExpression')# a bad defaultval in 3.13.8 schema was fixed in 3.13.9, but the migration was missedrql('SET ATTR defaultval NULL WHERE ATTR from_entity E, E name "CWSource", ATTR relation_type T, T name "in_synchronization"')# the migration gets confused when we change rdefs out from under it. So# explicitly remove this size constraint so it doesn't stick around and break# things later.rdefeid=schema['defaultval'].rdefs.values()[0].eidrql('DELETE CWConstraint C WHERE C cstrtype T, T name "SizeConstraint", R constrained_by C, R eid %(eid)s',{'eid':rdefeid})sync_schema_props_perms('defaultval')defconvert_defaultval(cwattr,default):fromdecimalimportDecimalimportyamsfromcubicwebimportBinaryifdefaultisNone:returnifisinstance(default,Binary):# partially migrated instance, try to be idempotentreturndefaultatype=cwattr.to_entity[0].nameifatype=='Boolean':# boolean attributes with default=False were stored as ''assertdefaultin('True','False',''),repr(default)default=default=='True'elifatypein('Int','BigInt'):default=int(default)elifatype=='Float':default=float(default)elifatype=='Decimal':default=Decimal(default)elifatypein('Date','Datetime','TZDatetime','Time'):try:# handle NOW and TODAY, keep them stored as stringsyams.KEYWORD_MAP[atype][default.upper()]default=default.upper()exceptKeyError:# otherwise get an actual date or datetimedefault=yams.DATE_FACTORY_MAP[atype](default)else:assertatype=='String',atypedefault=text_type(default)returnBinary.zpickle(default)dbh=repo.system_source.dbhelpersql('ALTER TABLE cw_cwattribute ADD new_defaultval %s'%dbh.TYPE_MAPPING['Bytes'])forcwattrinrql('CWAttribute X').entities():olddefault=cwattr.defaultvalifolddefaultisnotNone:req="UPDATE cw_cwattribute SET new_defaultval = %(val)s WHERE cw_eid = %(eid)s"args={'val':dbh.binary_value(convert_defaultval(cwattr,olddefault).getvalue()),'eid':cwattr.eid}sql(req,args,ask_confirm=False)sql('ALTER TABLE cw_cwattribute DROP COLUMN cw_defaultval')ifdriver=='postgres':sql('ALTER TABLE cw_cwattribute RENAME COLUMN new_defaultval TO cw_defaultval')else:# sqlserversql("sp_rename 'cw_cwattribute.new_defaultval', 'cw_defaultval', 'COLUMN'")# Set object type to "Bytes" for CWAttribute's "defaultval" attributerql('SET X to_entity B WHERE X is CWAttribute, X from_entity Y, Y name "CWAttribute", ''X relation_type Z, Z name "defaultval", B name "Bytes", NOT X to_entity B')oldrdef=schema['CWAttribute'].rdef('defaultval')importyams.buildobjsasybonewrdef=ybo.RelationDefinition('CWAttribute','defaultval','Bytes')newrdef.eid=oldrdef.eidschema.add_relation_def(newrdef)schema.del_relation_def('CWAttribute','defaultval','String')commit()sync_schema_props_perms('defaultval')forrschemainschema.relations():ifrschema.symmetric:subjects=set(repr(e.type)foreinrschema.subjects())objects=set(repr(e.type)foreinrschema.objects())assertsubjects==objectsmartians=set(str(eid)foreid,insql('SELECT eid_to FROM %s_relation, entities WHERE eid_to = eid AND type NOT IN (%s)'%(rschema.type,','.join(subjects))))martians|=set(str(eid)foreid,insql('SELECT eid_from FROM %s_relation, entities WHERE eid_from = eid AND type NOT IN (%s)'%(rschema.type,','.join(subjects))))ifmartians:martians=','.join(martians)print('deleting broken relations %s for eids %s'%(rschema.type,martians))sql('DELETE FROM %s_relation WHERE eid_from IN (%s) OR eid_to IN (%s)'%(rschema.type,martians,martians))withsession.deny_all_hooks_but():rql('SET X %(r)s Y WHERE Y %(r)s X, NOT X %(r)s Y'%{'r':rschema.type})commit()# multi columns unique constraints regenerationfromcubicweb.serverimportschemaserial# syncschema hooks would try to remove indices but# 1) we already do that below# 2) the hook expects the CWUniqueTogetherConstraint.name attribute that hasn't# yet been addedwithsession.allow_all_hooks_but('syncschema'):rql('DELETE CWUniqueTogetherConstraint C')commit()add_attribute('CWUniqueTogetherConstraint','name')# low-level wipe code for postgres & sqlserver, plain sql ...ifdriver=='postgres':forindexname,insql('select indexname from pg_indexes'):ifindexname.startswith('unique_'):print('dropping index',indexname)sql('DROP INDEX %s'%indexname)commit()elifdriver.startswith('sqlserver'):forviewname,insql('select name from sys.views'):ifviewname.startswith('utv_'):print('dropping view (index should be cascade-deleted)',viewname)sql('DROP VIEW %s'%viewname)commit()# recreate the constraints, hook will lead to low-level recreationforeschemainsorted(schema.entities()):ifeschema._unique_together:print('recreate unique indexes for',eschema)rql_args=schemaserial.uniquetogether2rqls(eschema)forrql,argsinrql_args:args['x']=eschema.eidsession.execute(rql,args)commit()# all attributes perms have to be refreshed ...forrschemainsorted(schema.relations()):ifrschema.final:ifrschema.typeinfsschema:print('sync perms for',rschema.type)sync_schema_props_perms(rschema.type,syncprops=False,ask_confirm=False,commit=False)else:print('WARNING: attribute %s missing from fs schema'%rschema.type)commit()ifapplcubicwebversion<(3,17,0)andcubicwebversion>=(3,17,0):try:add_cube('sioc',update_database=False)exceptConfigurationError:ifnotconfirm('In cubicweb 3.17 sioc views have been moved to the sioc ''cube, which is not installed. Continue anyway?'):raisetry:add_cube('embed',update_database=False)exceptConfigurationError:ifnotconfirm('In cubicweb 3.17 embedding views have been moved to the embed ''cube, which is not installed. Continue anyway?'):raisetry:add_cube('geocoding',update_database=False)exceptConfigurationError:ifnotconfirm('In cubicweb 3.17 geocoding views have been moved to the geocoding ''cube, which is not installed. Continue anyway?'):raiseifapplcubicwebversion<=(3,14,0)andcubicwebversion>=(3,14,0):if'require_permission'inschemaandnot'localperms'inrepo.config.cubes():fromcubicwebimportExecutionErrortry:add_cube('localperms',update_database=False)exceptConfigurationError:raiseExecutionError('In cubicweb 3.14, CWPermission and related stuff ''has been moved to cube localperms. Install it first.')ifapplcubicwebversion==(3,6,0)andcubicwebversion>=(3,6,0):CSTRMAP=dict(rql('Any T, X WHERE X is CWConstraintType, X name T',ask_confirm=False))_add_relation_definition_no_perms('CWAttribute','update_permission','CWGroup')_add_relation_definition_no_perms('CWAttribute','update_permission','RQLExpression')rql('SET X update_permission Y WHERE X is CWAttribute, X add_permission Y')drop_relation_definition('CWAttribute','delete_permission','CWGroup')drop_relation_definition('CWAttribute','delete_permission','RQLExpression')elifapplcubicwebversion<(3,6,0)andcubicwebversion>=(3,6,0):CSTRMAP=dict(rql('Any T, X WHERE X is CWConstraintType, X name T',ask_confirm=False))session.set_cnxset()permsdict=ss.deserialize_ertype_permissions(session)withhooks_control(session,session.HOOKS_ALLOW_ALL,'integrity'):forrschemainrepo.schema.relations():rpermsdict=permsdict.get(rschema.eid,{})forrdefinrschema.rdefs.values():foractioninrdef.ACTIONS:actperms=[]forsomethinginrpermsdict.get(action=='update'and'add'oraction,()):ifisinstance(something,tuple):actperms.append(rdef.rql_expression(*something))else:# group nameactperms.append(something)rdef.set_action_permissions(action,actperms)foractionin('read','add','delete'):_add_relation_definition_no_perms('CWRelation','%s_permission'%action,'CWGroup')_add_relation_definition_no_perms('CWRelation','%s_permission'%action,'RQLExpression')foractionin('read','update'):_add_relation_definition_no_perms('CWAttribute','%s_permission'%action,'CWGroup')_add_relation_definition_no_perms('CWAttribute','%s_permission'%action,'RQLExpression')foractionin('read','add','delete'):rql('SET X %s_permission Y WHERE X is CWRelation, ''RT %s_permission Y, X relation_type RT, Y is CWGroup'%(action,action))rql('INSERT RQLExpression Y: Y exprtype YET, Y mainvars YMV, Y expression YEX, ''X %s_permission Y WHERE X is CWRelation, ''X relation_type RT, RT %s_permission Y2, Y2 exprtype YET, ''Y2 mainvars YMV, Y2 expression YEX'%(action,action))rql('SET X read_permission Y WHERE X is CWAttribute, ''RT read_permission Y, X relation_type RT, Y is CWGroup')rql('INSERT RQLExpression Y: Y exprtype YET, Y mainvars YMV, Y expression YEX, ''X read_permission Y WHERE X is CWAttribute, ''X relation_type RT, RT read_permission Y2, Y2 exprtype YET, ''Y2 mainvars YMV, Y2 expression YEX')rql('SET X update_permission Y WHERE X is CWAttribute, ''RT add_permission Y, X relation_type RT, Y is CWGroup')rql('INSERT RQLExpression Y: Y exprtype YET, Y mainvars YMV, Y expression YEX, ''X update_permission Y WHERE X is CWAttribute, ''X relation_type RT, RT add_permission Y2, Y2 exprtype YET, ''Y2 mainvars YMV, Y2 expression YEX')foractionin('read','add','delete'):drop_relation_definition('CWRType','%s_permission'%action,'CWGroup',commit=False)drop_relation_definition('CWRType','%s_permission'%action,'RQLExpression')sync_schema_props_perms('read_permission',syncperms=False)# fix read_permission cardinalityifapplcubicwebversion<(3,9,6)andcubicwebversion>=(3,9,6)andnot'CWUniqueTogetherConstraint'inschema:add_entity_type('CWUniqueTogetherConstraint')ifnot('CWUniqueTogetherConstraint','CWRType')inschema['relations'].rdefs:add_relation_definition('CWUniqueTogetherConstraint','relations','CWRType')rql('SET C relations RT WHERE C relations RDEF, RDEF relation_type RT')commit()drop_relation_definition('CWUniqueTogetherConstraint','relations','CWAttribute')drop_relation_definition('CWUniqueTogetherConstraint','relations','CWRelation')ifapplcubicwebversion<(3,4,0)andcubicwebversion>=(3,4,0):withhooks_control(session,session.HOOKS_ALLOW_ALL,'integrity'):session.set_shared_data('do-not-insert-cwuri',True)add_relation_type('cwuri')base_url=session.base_url()foreid,inrql('Any X',ask_confirm=False):type,source,extid=session.describe(eid)ifsource=='system':rql('SET X cwuri %(u)s WHERE X eid %(x)s',{'x':eid,'u':u'%s%s'%(base_url,eid)})isession.commit()session.set_shared_data('do-not-insert-cwuri',False)ifapplcubicwebversion<(3,5,0)andcubicwebversion>=(3,5,0):# check that migration is not doomedrset=rql('Any X,Y WHERE X transition_of E, Y transition_of E, ''X name N, Y name N, NOT X identity Y',ask_confirm=False)ifrset:fromlogilab.common.shellutilsimportASKifnotASK.confirm('Migration will fail because of transitions with the same name. ''Continue anyway ?'):importsyssys.exit(1)# proceed with migrationadd_entity_type('Workflow')add_entity_type('BaseTransition')add_entity_type('WorkflowTransition')add_entity_type('SubWorkflowExitPoint')# drop explicit 'State allowed_transition Transition' since it should be# infered due to yams inheritance. However we've to disable the schema# sync hook first to avoid to destroy existing data...try:fromcubicweb.hooksimportsyncschemarepo.vreg.unregister(syncschema.AfterDelRelationTypeHook)try:drop_relation_definition('State','allowed_transition','Transition')finally:repo.vreg.register(syncschema.AfterDelRelationTypeHook)exceptImportError:# syncschema is in CW >= 3.6 onlyfromcubicweb.server.schemahooksimportafter_del_relation_typerepo.hm.unregister_hook(after_del_relation_type,'after_delete_relation','relation_type')try:drop_relation_definition('State','allowed_transition','Transition')finally:repo.hm.register_hook(after_del_relation_type,'after_delete_relation','relation_type')schema.rebuild_infered_relations()# need to be explicitly called once everything is in placeforetinrql('DISTINCT Any ET,ETN WHERE S state_of ET, ET name ETN',ask_confirm=False).entities():wf=add_workflow(u'default %s workflow'%et.name,et.name,ask_confirm=False)rql('SET S state_of WF WHERE S state_of ET, ET eid %(et)s, WF eid %(wf)s',{'et':et.eid,'wf':wf.eid},'et',ask_confirm=False)rql('SET T transition_of WF WHERE T transition_of ET, ET eid %(et)s, WF eid %(wf)s',{'et':et.eid,'wf':wf.eid},'et',ask_confirm=False)rql('SET WF initial_state S WHERE ET initial_state S, ET eid %(et)s, WF eid %(wf)s',{'et':et.eid,'wf':wf.eid},'et',ask_confirm=False)rql('DELETE TrInfo TI WHERE NOT TI from_state S')rql('SET TI by_transition T WHERE TI from_state FS, TI to_state TS, ''FS allowed_transition T, T destination_state TS')commit()drop_relation_definition('State','state_of','CWEType')drop_relation_definition('Transition','transition_of','CWEType')drop_relation_definition('CWEType','initial_state','State')sync_schema_props_perms()ifapplcubicwebversion<(3,2,2)andcubicwebversion>=(3,2,1):frombase64importb64encodeforeid,extidinsql('SELECT eid, extid FROM entities ''WHERE extid is NOT NULL',ask_confirm=False):sql('UPDATE entities SET extid=%(extid)s WHERE eid=%(eid)s',{'extid':b64encode(extid),'eid':eid},ask_confirm=False)commit()ifapplcubicwebversion<(3,2,0)andcubicwebversion>=(3,2,0):add_cube('card',update_database=False)ifapplcubicwebversion<(3,21,1)andcubicwebversion>=(3,21,1):add_relation_definition('CWComputedRType','read_permission','CWGroup')add_relation_definition('CWComputedRType','read_permission','RQLExpression')defsync_constraint_types():"""Make sure the repository knows about all constraint types defined in the code"""fromcubicweb.schemaimportCONSTRAINTSrepo_constraints=set(row[0]forrowinrql('Any N WHERE X is CWConstraintType, X name N'))forcstrtypeinset(CONSTRAINTS)-repo_constraints:ifcstrtype=='BoundConstraint':# was renamed to BoundaryConstraint, we don't need the old namecontinuerql('INSERT CWConstraintType X: X name %(name)s',{'name':cstrtype})commit()sync_constraint_types()