[predicates] use select_or_none rather than select+try/except, expected to be more efficient
fromdatetimeimportdatetimeforrtypein('cw_support','cw_dont_cross','cw_may_cross'):drop_relation_type(rtype)add_entity_type('CWSourceSchemaConfig')ifnot'url'inschema['CWSource'].subjrels:add_attribute('CWSource','url')add_attribute('CWSource','parser')add_attribute('CWSource','latest_retrieval')try:fromcubicweb.server.sources.pyrorqlimportPyroRQLSourceexceptImportError:passelse:fromos.pathimportjoin# function to read old python mapping filedefload_mapping_file(source):mappingfile=source.config['mapping-file']mappingfile=join(source.repo.config.apphome,mappingfile)mapping={}execfile(mappingfile,mapping)forjunkin('__builtins__','__doc__'):mapping.pop(junk,None)mapping.setdefault('support_relations',{})mapping.setdefault('dont_cross_relations',set())mapping.setdefault('cross_relations',set())# do some basic checks of the mapping contentassert'support_entities'inmapping, \'mapping file should at least define support_entities'assertisinstance(mapping['support_entities'],dict)assertisinstance(mapping['support_relations'],dict)assertisinstance(mapping['dont_cross_relations'],set)assertisinstance(mapping['cross_relations'],set)unknown=set(mapping)-set(('support_entities','support_relations','dont_cross_relations','cross_relations'))assertnotunknown,'unknown mapping attribute(s): %s'%unknown# relations that are necessarily not crossedforrtypein('is','is_instance_of','cw_source'):assertrtypenotinmapping['dont_cross_relations'], \'%s relation should not be in dont_cross_relations'%rtypeassertrtypenotinmapping['support_relations'], \'%s relation should not be in support_relations'%rtypereturnmapping# for now, only pyrorql sources have a mappingforsourceinrepo.sources_by_uri.itervalues():ifnotisinstance(source,PyroRQLSource):continuesourceentity=session.entity_from_eid(source.eid)mapping=load_mapping_file(source)# write mapping as entitiesprint'migrating map for',sourceforetype,writeinmapping['support_entities'].items():create_entity('CWSourceSchemaConfig',cw_for_source=sourceentity,cw_schema=session.entity_from_eid(schema[etype].eid),options=writeandu'write'orNone,ask_confirm=False)forrtype,writeinmapping['support_relations'].items():options=[]ifwrite:options.append(u'write')ifrtypeinmapping['cross_relations']:options.append(u'maycross')create_entity('CWSourceSchemaConfig',cw_for_source=sourceentity,cw_schema=session.entity_from_eid(schema[rtype].eid),options=u':'.join(options)orNone,ask_confirm=False)forrtypeinmapping['dont_cross_relations']:create_entity('CWSourceSchemaConfig',cw_for_source=source,cw_schema=session.entity_from_eid(schema[rtype].eid),options=u'dontcross',ask_confirm=False)# latest update time cwproperty is now a source attribute (latest_retrieval)pkey=u'sources.%s.latest-update-time'%source.urirset=session.execute('Any V WHERE X is CWProperty, X value V, X pkey %(k)s',{'k':pkey})timestamp=int(rset[0][0])sourceentity.cw_set(latest_retrieval=datetime.fromtimestamp(timestamp))session.execute('DELETE CWProperty X WHERE X pkey %(k)s',{'k':pkey})