33 from datetime import datetime |
33 from datetime import datetime |
34 from base64 import b64decode, b64encode |
34 from base64 import b64decode, b64encode |
35 from contextlib import contextmanager |
35 from contextlib import contextmanager |
36 from os.path import abspath |
36 from os.path import abspath |
37 import re |
37 import re |
|
38 import itertools |
38 |
39 |
39 from logilab.common.compat import any |
40 from logilab.common.compat import any |
40 from logilab.common.cache import Cache |
41 from logilab.common.cache import Cache |
41 from logilab.common.decorators import cached, clear_cache |
42 from logilab.common.decorators import cached, clear_cache |
42 from logilab.common.configuration import Method |
43 from logilab.common.configuration import Method |
551 # For instance, the BFSStorage will replace the `data` |
552 # For instance, the BFSStorage will replace the `data` |
552 # binary value with a Binary containing the destination path |
553 # binary value with a Binary containing the destination path |
553 # on the filesystem. To make the entity.data usage absolutely |
554 # on the filesystem. To make the entity.data usage absolutely |
554 # transparent, we'll have to reset entity.data to its binary |
555 # transparent, we'll have to reset entity.data to its binary |
555 # value once the SQL query will be executed |
556 # value once the SQL query will be executed |
556 restore_values = {} |
557 restore_values = [] |
557 etype = entity.__regid__ |
558 if isinstance(entity, list): |
|
559 entities = entity |
|
560 else: |
|
561 entities = [entity] |
|
562 etype = entities[0].__regid__ |
558 for attr, storage in self._storages.get(etype, {}).items(): |
563 for attr, storage in self._storages.get(etype, {}).items(): |
559 try: |
564 for entity in entities: |
560 edited = entity.cw_edited |
565 try: |
561 except AttributeError: |
566 edited = entity.cw_edited |
562 assert event == 'deleted' |
567 except AttributeError: |
563 getattr(storage, 'entity_deleted')(entity, attr) |
568 assert event == 'deleted' |
564 else: |
569 getattr(storage, 'entity_deleted')(entity, attr) |
565 if attr in edited: |
570 else: |
566 handler = getattr(storage, 'entity_%s' % event) |
571 if attr in edited: |
567 restore_values[attr] = handler(entity, attr) |
572 handler = getattr(storage, 'entity_%s' % event) |
|
573 to_restore = handler(entity, attr) |
|
574 restore_values.append((entity, attr, to_restore)) |
568 try: |
575 try: |
569 yield # 2/ execute the source's instructions |
576 yield # 2/ execute the source's instructions |
570 finally: |
577 finally: |
571 # 3/ restore original values |
578 # 3/ restore original values |
572 for attr, value in restore_values.items(): |
579 for entity, attr, value in restore_values: |
573 entity.cw_edited.edited_attribute(attr, value) |
580 entity.cw_edited.edited_attribute(attr, value) |
574 |
581 |
575 def add_entity(self, session, entity): |
582 def add_entity(self, session, entity): |
576 """add a new entity to the source""" |
583 """add a new entity to the source""" |
577 with self._storage_handler(entity, 'added'): |
584 with self._storage_handler(entity, 'added'): |
921 * update the fti |
928 * update the fti |
922 * remove record from the entities table |
929 * remove record from the entities table |
923 * transfer it to the deleted_entities table if the entity's type is |
930 * transfer it to the deleted_entities table if the entity's type is |
924 multi-sources |
931 multi-sources |
925 """ |
932 """ |
926 self.fti_unindex_entity(session, entity.eid) |
933 self.fti_unindex_entities(session, [entity]) |
927 attrs = {'eid': entity.eid} |
934 attrs = {'eid': entity.eid} |
928 self.doexec(session, self.sqlgen.delete('entities', attrs), attrs) |
935 self.doexec(session, self.sqlgen.delete('entities', attrs), attrs) |
929 if not entity.__regid__ in self.multisources_etypes: |
936 if not entity.__regid__ in self.multisources_etypes: |
930 return |
937 return |
931 if extid is not None: |
938 if extid is not None: |
932 assert isinstance(extid, str), type(extid) |
939 assert isinstance(extid, str), type(extid) |
933 extid = b64encode(extid) |
940 extid = b64encode(extid) |
934 attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': extid, |
941 attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': extid, |
935 'source': uri, 'dtime': datetime.now()} |
942 'source': uri, 'dtime': datetime.now()} |
936 self.doexec(session, self.sqlgen.insert('deleted_entities', attrs), attrs) |
943 self.doexec(session, self.sqlgen.insert('deleted_entities', attrs), attrs) |
|
944 |
|
945 def delete_info_multi(self, session, entities, uri, extids): |
|
946 """delete system information on deletion of an entity: |
|
947 * update the fti |
|
948 * remove record from the entities table |
|
949 * transfer it to the deleted_entities table if the entity's type is |
|
950 multi-sources |
|
951 """ |
|
952 self.fti_unindex_entities(session, entities) |
|
953 attrs = {'eid': '(%s)' % ','.join([str(_e.eid) for _e in entities])} |
|
954 self.doexec(session, self.sqlgen.delete_many('entities', attrs), attrs) |
|
955 if entities[0].__regid__ not in self.multisources_etypes: |
|
956 return |
|
957 attrs = {'type': entities[0].__regid__, |
|
958 'source': uri, 'dtime': datetime.now()} |
|
959 for entity, extid in itertools.izip(entities, extids): |
|
960 if extid is not None: |
|
961 assert isinstance(extid, str), type(extid) |
|
962 extid = b64encode(extid) |
|
963 attrs.update({'eid': entity.eid, 'extid': extid}) |
|
964 self.doexec(session, self.sqlgen.insert('deleted_entities', attrs), attrs) |
937 |
965 |
938 def modified_entities(self, session, etypes, mtime): |
966 def modified_entities(self, session, etypes, mtime): |
939 """return a 2-uple: |
967 """return a 2-uple: |
940 * list of (etype, eid) of entities of the given types which have been |
968 * list of (etype, eid) of entities of the given types which have been |
941 modified since the given timestamp (actually entities whose full text |
969 modified since the given timestamp (actually entities whose full text |
1300 """create an operation to [re]index textual content of the given entity |
1328 """create an operation to [re]index textual content of the given entity |
1301 on commit |
1329 on commit |
1302 """ |
1330 """ |
1303 FTIndexEntityOp.get_instance(session).add_data(entity.eid) |
1331 FTIndexEntityOp.get_instance(session).add_data(entity.eid) |
1304 |
1332 |
1305 def fti_unindex_entity(self, session, eid): |
1333 def fti_unindex_entities(self, session, entities): |
1306 """remove text content for entity with the given eid from the full text |
1334 """remove text content for entities from the full text index |
1307 index |
1335 """ |
1308 """ |
1336 cursor = session.pool['system'] |
1309 try: |
1337 cursor_unindex_object = self.dbhelper.cursor_unindex_object |
1310 self.dbhelper.cursor_unindex_object(eid, session.pool['system']) |
1338 try: |
|
1339 for entity in entities: |
|
1340 cursor_unindex_object(entity.eid, cursor) |
1311 except Exception: # let KeyboardInterrupt / SystemExit propagate |
1341 except Exception: # let KeyboardInterrupt / SystemExit propagate |
1312 self.exception('error while unindexing %s', eid) |
1342 self.exception('error while unindexing %s', entity) |
1313 |
1343 |
1314 def fti_index_entity(self, session, entity): |
1344 |
1315 """add text content of a created/modified entity to the full text index |
1345 def fti_index_entities(self, session, entities): |
1316 """ |
1346 """add text content of created/modified entities to the full text index |
1317 self.debug('reindexing %r', entity.eid) |
1347 """ |
|
1348 cursor_index_object = self.dbhelper.cursor_index_object |
|
1349 cursor = session.pool['system'] |
1318 try: |
1350 try: |
1319 # use cursor_index_object, not cursor_reindex_object since |
1351 # use cursor_index_object, not cursor_reindex_object since |
1320 # unindexing done in the FTIndexEntityOp |
1352 # unindexing done in the FTIndexEntityOp |
1321 self.dbhelper.cursor_index_object(entity.eid, |
1353 for entity in entities: |
1322 entity.cw_adapt_to('IFTIndexable'), |
1354 cursor_index_object(entity.eid, |
1323 session.pool['system']) |
1355 entity.cw_adapt_to('IFTIndexable'), |
|
1356 cursor) |
1324 except Exception: # let KeyboardInterrupt / SystemExit propagate |
1357 except Exception: # let KeyboardInterrupt / SystemExit propagate |
1325 self.exception('error while reindexing %s', entity) |
1358 self.exception('error while indexing %s', entity) |
1326 |
1359 |
1327 |
1360 |
1328 class FTIndexEntityOp(hook.DataOperationMixIn, hook.LateOperation): |
1361 class FTIndexEntityOp(hook.DataOperationMixIn, hook.LateOperation): |
1329 """operation to delay entity full text indexation to commit |
1362 """operation to delay entity full text indexation to commit |
1330 |
1363 |
1336 def precommit_event(self): |
1369 def precommit_event(self): |
1337 session = self.session |
1370 session = self.session |
1338 source = session.repo.system_source |
1371 source = session.repo.system_source |
1339 pendingeids = session.transaction_data.get('pendingeids', ()) |
1372 pendingeids = session.transaction_data.get('pendingeids', ()) |
1340 done = session.transaction_data.setdefault('indexedeids', set()) |
1373 done = session.transaction_data.setdefault('indexedeids', set()) |
|
1374 to_reindex = set() |
1341 for eid in self.get_data(): |
1375 for eid in self.get_data(): |
1342 if eid in pendingeids or eid in done: |
1376 if eid in pendingeids or eid in done: |
1343 # entity added and deleted in the same transaction or already |
1377 # entity added and deleted in the same transaction or already |
1344 # processed |
1378 # processed |
1345 return |
1379 continue |
1346 done.add(eid) |
1380 done.add(eid) |
1347 iftindexable = session.entity_from_eid(eid).cw_adapt_to('IFTIndexable') |
1381 iftindexable = session.entity_from_eid(eid).cw_adapt_to('IFTIndexable') |
1348 for container in iftindexable.fti_containers(): |
1382 to_reindex |= set(iftindexable.fti_containers()) |
1349 source.fti_unindex_entity(session, container.eid) |
1383 source.fti_unindex_entities(session, to_reindex) |
1350 source.fti_index_entity(session, container) |
1384 source.fti_index_entities(session, to_reindex) |
1351 |
|
1352 |
1385 |
1353 def sql_schema(driver): |
1386 def sql_schema(driver): |
1354 helper = get_db_helper(driver) |
1387 helper = get_db_helper(driver) |
1355 typemap = helper.TYPE_MAPPING |
1388 typemap = helper.TYPE_MAPPING |
1356 schema = """ |
1389 schema = """ |