7 |
7 |
8 if not 'url' in schema['CWSource'].subjrels: |
8 if not 'url' in schema['CWSource'].subjrels: |
9 add_attribute('CWSource', 'url') |
9 add_attribute('CWSource', 'url') |
10 add_attribute('CWSource', 'parser') |
10 add_attribute('CWSource', 'parser') |
11 add_attribute('CWSource', 'latest_retrieval') |
11 add_attribute('CWSource', 'latest_retrieval') |
12 |
|
13 try: |
|
14 from cubicweb.server.sources.pyrorql import PyroRQLSource |
|
15 except ImportError: |
|
16 pass |
|
17 else: |
|
18 |
|
19 from os.path import join |
|
20 # function to read old python mapping file |
|
21 def load_mapping_file(source): |
|
22 mappingfile = source.config['mapping-file'] |
|
23 mappingfile = join(source.repo.config.apphome, mappingfile) |
|
24 mapping = {} |
|
25 execfile(mappingfile, mapping) |
|
26 for junk in ('__builtins__', '__doc__'): |
|
27 mapping.pop(junk, None) |
|
28 mapping.setdefault('support_relations', {}) |
|
29 mapping.setdefault('dont_cross_relations', set()) |
|
30 mapping.setdefault('cross_relations', set()) |
|
31 # do some basic checks of the mapping content |
|
32 assert 'support_entities' in mapping, \ |
|
33 'mapping file should at least define support_entities' |
|
34 assert isinstance(mapping['support_entities'], dict) |
|
35 assert isinstance(mapping['support_relations'], dict) |
|
36 assert isinstance(mapping['dont_cross_relations'], set) |
|
37 assert isinstance(mapping['cross_relations'], set) |
|
38 unknown = set(mapping) - set( ('support_entities', 'support_relations', |
|
39 'dont_cross_relations', 'cross_relations') ) |
|
40 assert not unknown, 'unknown mapping attribute(s): %s' % unknown |
|
41 # relations that are necessarily not crossed |
|
42 for rtype in ('is', 'is_instance_of', 'cw_source'): |
|
43 assert rtype not in mapping['dont_cross_relations'], \ |
|
44 '%s relation should not be in dont_cross_relations' % rtype |
|
45 assert rtype not in mapping['support_relations'], \ |
|
46 '%s relation should not be in support_relations' % rtype |
|
47 return mapping |
|
48 # for now, only pyrorql sources have a mapping |
|
49 for source in repo.sources_by_uri.itervalues(): |
|
50 if not isinstance(source, PyroRQLSource): |
|
51 continue |
|
52 sourceentity = session.entity_from_eid(source.eid) |
|
53 mapping = load_mapping_file(source) |
|
54 # write mapping as entities |
|
55 print 'migrating map for', source |
|
56 for etype, write in mapping['support_entities'].items(): |
|
57 create_entity('CWSourceSchemaConfig', |
|
58 cw_for_source=sourceentity, |
|
59 cw_schema=session.entity_from_eid(schema[etype].eid), |
|
60 options=write and u'write' or None, |
|
61 ask_confirm=False) |
|
62 for rtype, write in mapping['support_relations'].items(): |
|
63 options = [] |
|
64 if write: |
|
65 options.append(u'write') |
|
66 if rtype in mapping['cross_relations']: |
|
67 options.append(u'maycross') |
|
68 create_entity('CWSourceSchemaConfig', |
|
69 cw_for_source=sourceentity, |
|
70 cw_schema=session.entity_from_eid(schema[rtype].eid), |
|
71 options=u':'.join(options) or None, |
|
72 ask_confirm=False) |
|
73 for rtype in mapping['dont_cross_relations']: |
|
74 create_entity('CWSourceSchemaConfig', |
|
75 cw_for_source=source, |
|
76 cw_schema=session.entity_from_eid(schema[rtype].eid), |
|
77 options=u'dontcross', |
|
78 ask_confirm=False) |
|
79 # latest update time cwproperty is now a source attribute (latest_retrieval) |
|
80 pkey = u'sources.%s.latest-update-time' % source.uri |
|
81 rset = session.execute('Any V WHERE X is CWProperty, X value V, X pkey %(k)s', |
|
82 {'k': pkey}) |
|
83 timestamp = int(rset[0][0]) |
|
84 sourceentity.cw_set(latest_retrieval=datetime.fromtimestamp(timestamp)) |
|
85 session.execute('DELETE CWProperty X WHERE X pkey %(k)s', {'k': pkey}) |
|