33 from datetime import datetime |
33 from datetime import datetime |
34 from base64 import b64decode, b64encode |
34 from base64 import b64decode, b64encode |
35 from contextlib import contextmanager |
35 from contextlib import contextmanager |
36 from os.path import abspath |
36 from os.path import abspath |
37 import re |
37 import re |
|
38 import itertools |
38 |
39 |
39 from logilab.common.compat import any |
40 from logilab.common.compat import any |
40 from logilab.common.cache import Cache |
41 from logilab.common.cache import Cache |
41 from logilab.common.decorators import cached, clear_cache |
42 from logilab.common.decorators import cached, clear_cache |
42 from logilab.common.configuration import Method |
43 from logilab.common.configuration import Method |
545 # For instance, the BFSStorage will replace the `data` |
546 # For instance, the BFSStorage will replace the `data` |
546 # binary value with a Binary containing the destination path |
547 # binary value with a Binary containing the destination path |
547 # on the filesystem. To make the entity.data usage absolutely |
548 # on the filesystem. To make the entity.data usage absolutely |
548 # transparent, we'll have to reset entity.data to its binary |
549 # transparent, we'll have to reset entity.data to its binary |
549 # value once the SQL query will be executed |
550 # value once the SQL query will be executed |
550 restore_values = {} |
551 restore_values = [] |
551 etype = entity.__regid__ |
552 if isinstance(entity, list): |
|
553 entities = entity |
|
554 else: |
|
555 entities = [entity] |
|
556 etype = entities[0].__regid__ |
552 for attr, storage in self._storages.get(etype, {}).items(): |
557 for attr, storage in self._storages.get(etype, {}).items(): |
553 try: |
558 for entity in entities: |
554 edited = entity.cw_edited |
559 try: |
555 except AttributeError: |
560 edited = entity.cw_edited |
556 assert event == 'deleted' |
561 except AttributeError: |
557 getattr(storage, 'entity_deleted')(entity, attr) |
562 assert event == 'deleted' |
558 else: |
563 getattr(storage, 'entity_deleted')(entity, attr) |
559 if attr in edited: |
564 else: |
560 handler = getattr(storage, 'entity_%s' % event) |
565 if attr in edited: |
561 restore_values[attr] = handler(entity, attr) |
566 handler = getattr(storage, 'entity_%s' % event) |
|
567 to_restore = handler(entity, attr) |
|
568 restore_values.append((entity, attr, to_restore)) |
562 try: |
569 try: |
563 yield # 2/ execute the source's instructions |
570 yield # 2/ execute the source's instructions |
564 finally: |
571 finally: |
565 # 3/ restore original values |
572 # 3/ restore original values |
566 for attr, value in restore_values.items(): |
573 for entity, attr, value in restore_values: |
567 entity.cw_edited.edited_attribute(attr, value) |
574 entity.cw_edited.edited_attribute(attr, value) |
568 |
575 |
569 def add_entity(self, session, entity): |
576 def add_entity(self, session, entity): |
570 """add a new entity to the source""" |
577 """add a new entity to the source""" |
571 with self._storage_handler(entity, 'added'): |
578 with self._storage_handler(entity, 'added'): |
916 * update the fti |
923 * update the fti |
917 * remove record from the entities table |
924 * remove record from the entities table |
918 * transfer it to the deleted_entities table if the entity's type is |
925 * transfer it to the deleted_entities table if the entity's type is |
919 multi-sources |
926 multi-sources |
920 """ |
927 """ |
921 self.fti_unindex_entity(session, entity.eid) |
928 self.fti_unindex_entities(session, [entity]) |
922 attrs = {'eid': entity.eid} |
929 attrs = {'eid': entity.eid} |
923 self.doexec(session, self.sqlgen.delete('entities', attrs), attrs) |
930 self.doexec(session, self.sqlgen.delete('entities', attrs), attrs) |
924 if not entity.__regid__ in self.multisources_etypes: |
931 if not entity.__regid__ in self.multisources_etypes: |
925 return |
932 return |
926 if extid is not None: |
933 if extid is not None: |
927 assert isinstance(extid, str), type(extid) |
934 assert isinstance(extid, str), type(extid) |
928 extid = b64encode(extid) |
935 extid = b64encode(extid) |
929 attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': extid, |
936 attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': extid, |
930 'source': uri, 'dtime': datetime.now()} |
937 'source': uri, 'dtime': datetime.now()} |
931 self.doexec(session, self.sqlgen.insert('deleted_entities', attrs), attrs) |
938 self.doexec(session, self.sqlgen.insert('deleted_entities', attrs), attrs) |
|
939 |
|
940 def delete_info_multi(self, session, entities, uri, extids): |
|
941 """delete system information on deletion of an entity: |
|
942 * update the fti |
|
943 * remove record from the entities table |
|
944 * transfer it to the deleted_entities table if the entity's type is |
|
945 multi-sources |
|
946 """ |
|
947 self.fti_unindex_entities(session, entities) |
|
948 attrs = {'eid': '(%s)' % ','.join([str(_e.eid) for _e in entities])} |
|
949 self.doexec(session, self.sqlgen.delete_many('entities', attrs), attrs) |
|
950 if entities[0].__regid__ not in self.multisources_etypes: |
|
951 return |
|
952 attrs = {'type': entities[0].__regid__, |
|
953 'source': uri, 'dtime': datetime.now()} |
|
954 for entity, extid in itertools.izip(entities, extids): |
|
955 if extid is not None: |
|
956 assert isinstance(extid, str), type(extid) |
|
957 extid = b64encode(extid) |
|
958 attrs.update({'eid': entity.eid, 'extid': extid}) |
|
959 self.doexec(session, self.sqlgen.insert('deleted_entities', attrs), attrs) |
932 |
960 |
933 def modified_entities(self, session, etypes, mtime): |
961 def modified_entities(self, session, etypes, mtime): |
934 """return a 2-uple: |
962 """return a 2-uple: |
935 * list of (etype, eid) of entities of the given types which have been |
963 * list of (etype, eid) of entities of the given types which have been |
936 modified since the given timestamp (actually entities whose full text |
964 modified since the given timestamp (actually entities whose full text |
1295 """create an operation to [re]index textual content of the given entity |
1323 """create an operation to [re]index textual content of the given entity |
1296 on commit |
1324 on commit |
1297 """ |
1325 """ |
1298 FTIndexEntityOp.get_instance(session).add_data(entity.eid) |
1326 FTIndexEntityOp.get_instance(session).add_data(entity.eid) |
1299 |
1327 |
1300 def fti_unindex_entity(self, session, eid): |
1328 def fti_unindex_entities(self, session, entities): |
1301 """remove text content for entity with the given eid from the full text |
1329 """remove text content for entities from the full text index |
1302 index |
1330 """ |
1303 """ |
1331 cursor = session.pool['system'] |
1304 try: |
1332 cursor_unindex_object = self.dbhelper.cursor_unindex_object |
1305 self.dbhelper.cursor_unindex_object(eid, session.pool['system']) |
1333 try: |
|
1334 for entity in entities: |
|
1335 cursor_unindex_object(entity.eid, cursor) |
1306 except Exception: # let KeyboardInterrupt / SystemExit propagate |
1336 except Exception: # let KeyboardInterrupt / SystemExit propagate |
1307 self.exception('error while unindexing %s', eid) |
1337 self.exception('error while unindexing %s', entity) |
1308 |
1338 |
1309 def fti_index_entity(self, session, entity): |
1339 |
1310 """add text content of a created/modified entity to the full text index |
1340 def fti_index_entities(self, session, entities): |
1311 """ |
1341 """add text content of created/modified entities to the full text index |
1312 self.debug('reindexing %r', entity.eid) |
1342 """ |
|
1343 cursor_index_object = self.dbhelper.cursor_index_object |
|
1344 cursor = session.pool['system'] |
1313 try: |
1345 try: |
1314 # use cursor_index_object, not cursor_reindex_object since |
1346 # use cursor_index_object, not cursor_reindex_object since |
1315 # unindexing done in the FTIndexEntityOp |
1347 # unindexing done in the FTIndexEntityOp |
1316 self.dbhelper.cursor_index_object(entity.eid, |
1348 for entity in entities: |
1317 entity.cw_adapt_to('IFTIndexable'), |
1349 cursor_index_object(entity.eid, |
1318 session.pool['system']) |
1350 entity.cw_adapt_to('IFTIndexable'), |
|
1351 cursor) |
1319 except Exception: # let KeyboardInterrupt / SystemExit propagate |
1352 except Exception: # let KeyboardInterrupt / SystemExit propagate |
1320 self.exception('error while reindexing %s', entity) |
1353 self.exception('error while indexing %s', entity) |
1321 |
1354 |
1322 |
1355 |
1323 class FTIndexEntityOp(hook.DataOperationMixIn, hook.LateOperation): |
1356 class FTIndexEntityOp(hook.DataOperationMixIn, hook.LateOperation): |
1324 """operation to delay entity full text indexation to commit |
1357 """operation to delay entity full text indexation to commit |
1325 |
1358 |
1331 def precommit_event(self): |
1364 def precommit_event(self): |
1332 session = self.session |
1365 session = self.session |
1333 source = session.repo.system_source |
1366 source = session.repo.system_source |
1334 pendingeids = session.transaction_data.get('pendingeids', ()) |
1367 pendingeids = session.transaction_data.get('pendingeids', ()) |
1335 done = session.transaction_data.setdefault('indexedeids', set()) |
1368 done = session.transaction_data.setdefault('indexedeids', set()) |
|
1369 to_reindex = set() |
1336 for eid in self.get_data(): |
1370 for eid in self.get_data(): |
1337 if eid in pendingeids or eid in done: |
1371 if eid in pendingeids or eid in done: |
1338 # entity added and deleted in the same transaction or already |
1372 # entity added and deleted in the same transaction or already |
1339 # processed |
1373 # processed |
1340 return |
1374 continue |
1341 done.add(eid) |
1375 done.add(eid) |
1342 iftindexable = session.entity_from_eid(eid).cw_adapt_to('IFTIndexable') |
1376 iftindexable = session.entity_from_eid(eid).cw_adapt_to('IFTIndexable') |
1343 for container in iftindexable.fti_containers(): |
1377 to_reindex |= set(iftindexable.fti_containers()) |
1344 source.fti_unindex_entity(session, container.eid) |
1378 source.fti_unindex_entities(session, to_reindex) |
1345 source.fti_index_entity(session, container) |
1379 source.fti_index_entities(session, to_reindex) |
1346 |
|
1347 |
1380 |
1348 def sql_schema(driver): |
1381 def sql_schema(driver): |
1349 helper = get_db_helper(driver) |
1382 helper = get_db_helper(driver) |
1350 typemap = helper.TYPE_MAPPING |
1383 typemap = helper.TYPE_MAPPING |
1351 schema = """ |
1384 schema = """ |