43 |
43 |
44 It should be used as follows: |
44 It should be used as follows: |
45 |
45 |
46 store = MassiveObjectStore(cnx) |
46 store = MassiveObjectStore(cnx) |
47 eid_p = store.prepare_insert_entity('Person', |
47 eid_p = store.prepare_insert_entity('Person', |
48 cwuri='http://dbpedia.org/toto', |
48 cwuri=u'http://dbpedia.org/toto', |
49 name='Toto') |
49 name=u'Toto') |
50 eid_loc = store.prepare_insert_entity('Location', |
50 eid_loc = store.prepare_insert_entity('Location', |
51 cwuri='http://geonames.org/11111', |
51 cwuri=u'http://geonames.org/11111', |
52 name='Somewhere') |
52 name=u'Somewhere') |
53 store.prepare_insert_relation(eid_p, 'lives_in', eid_loc) |
53 store.prepare_insert_relation(eid_p, 'lives_in', eid_loc) |
54 store.flush() |
54 store.flush() |
55 ... |
55 ... |
56 store.commit() |
56 store.commit() |
57 store.finish() |
57 store.finish() |
63 def __init__(self, cnx, |
63 def __init__(self, cnx, |
64 on_commit_callback=None, on_rollback_callback=None, |
64 on_commit_callback=None, on_rollback_callback=None, |
65 slave_mode=False, |
65 slave_mode=False, |
66 eids_seq_range=10000, |
66 eids_seq_range=10000, |
67 metagen=None): |
67 metagen=None): |
68 """ Create a MassiveObject store, with the following attributes: |
68 """Create a MassiveObject store, with the following arguments: |
69 |
69 |
70 - cnx: CubicWeb cnx |
70 - `cnx`, a connection to the repository |
71 - eids_seq_range: size of eid range reserved by the store for each batch |
71 - `metagen`, optional :class:`MetadataGenerator` instance |
|
72 - `eids_seq_range`: size of eid range reserved by the store for each batch |
72 """ |
73 """ |
73 super(MassiveObjectStore, self).__init__(cnx) |
74 super(MassiveObjectStore, self).__init__(cnx) |
74 self.on_commit_callback = on_commit_callback |
75 self.on_commit_callback = on_commit_callback |
75 self.on_rollback_callback = on_rollback_callback |
76 self.on_rollback_callback = on_rollback_callback |
76 self.slave_mode = slave_mode |
77 self.slave_mode = slave_mode |
79 metagen = stores.MetadataGenerator(cnx) |
80 metagen = stores.MetadataGenerator(cnx) |
80 self.metagen = metagen |
81 self.metagen = metagen |
81 |
82 |
82 self.logger = logging.getLogger('dataimport.massive_store') |
83 self.logger = logging.getLogger('dataimport.massive_store') |
83 self.sql = cnx.system_sql |
84 self.sql = cnx.system_sql |
84 self.schema = self._cnx.vreg.schema |
85 self.schema = cnx.vreg.schema |
85 self.default_values = get_default_values(self.schema) |
86 self.default_values = get_default_values(self.schema) |
86 self.get_next_eid = lambda g=self._get_eid_gen(): next(g) |
87 self.get_next_eid = lambda g=self._get_eid_gen(): next(g) |
87 self._dbh = PGHelper(cnx) |
88 self._dbh = PGHelper(cnx) |
88 |
89 |
89 cnx.read_security = False |
|
90 cnx.write_security = False |
|
91 |
|
92 self._data_entities = defaultdict(list) |
90 self._data_entities = defaultdict(list) |
93 self._data_relations = defaultdict(list) |
91 self._data_relations = defaultdict(list) |
94 self._initialized = set() |
92 self._initialized = set() |
95 |
|
96 self._constraints_dropped = self.slave_mode |
93 self._constraints_dropped = self.slave_mode |
97 |
94 |
98 def _get_eid_gen(self): |
95 def _get_eid_gen(self): |
99 """ Function getting the next eid. This is done by preselecting |
96 """ Function getting the next eid. This is done by preselecting |
100 a given number of eids from the 'entities_id_seq', and then |
97 a given number of eids from the 'entities_id_seq', and then |