15 # |
15 # |
16 # You should have received a copy of the GNU Lesser General Public License along |
16 # You should have received a copy of the GNU Lesser General Public License along |
17 # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
17 # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
18 """Adapters for native cubicweb sources.""" |
18 """Adapters for native cubicweb sources.""" |
19 |
19 |
20 from __future__ import print_function |
|
21 |
|
22 from threading import Lock |
20 from threading import Lock |
23 from datetime import datetime |
21 from datetime import datetime |
24 from contextlib import contextmanager |
22 from contextlib import contextmanager |
25 from os.path import basename |
23 from os.path import basename |
|
24 import pickle |
26 import re |
25 import re |
27 import itertools |
26 import itertools |
28 import zipfile |
27 import zipfile |
29 import logging |
28 import logging |
30 import sys |
29 import sys |
31 |
|
32 from six import PY2, text_type, string_types |
|
33 from six.moves import range, cPickle as pickle, zip |
|
34 |
30 |
35 from logilab.common.decorators import cached, clear_cache |
31 from logilab.common.decorators import cached, clear_cache |
36 from logilab.common.configuration import Method |
32 from logilab.common.configuration import Method |
37 from logilab.common.shellutils import getlogin, ASK |
33 from logilab.common.shellutils import getlogin, ASK |
38 from logilab.database import get_db_helper, sqlgen |
34 from logilab.database import get_db_helper, sqlgen |
524 except KeyError: |
519 except KeyError: |
525 self.cache_miss += 1 |
520 self.cache_miss += 1 |
526 sql, qargs, cbs = self._rql_sqlgen.generate(union, args) |
521 sql, qargs, cbs = self._rql_sqlgen.generate(union, args) |
527 self._cache[cachekey] = sql, qargs, cbs |
522 self._cache[cachekey] = sql, qargs, cbs |
528 args = self.merge_args(args, qargs) |
523 args = self.merge_args(args, qargs) |
529 assert isinstance(sql, string_types), repr(sql) |
524 assert isinstance(sql, str), repr(sql) |
530 cursor = cnx.system_sql(sql, args) |
525 cursor = cnx.system_sql(sql, args) |
531 results = self.process_result(cursor, cnx, cbs) |
526 results = self.process_result(cursor, cnx, cbs) |
532 assert dbg_results(results) |
527 assert dbg_results(results) |
533 return results |
528 return results |
534 |
529 |
579 attrs = self.preprocess_entity(entity) |
574 attrs = self.preprocess_entity(entity) |
580 sql = self.sqlgen.insert(SQL_PREFIX + entity.cw_etype, attrs) |
575 sql = self.sqlgen.insert(SQL_PREFIX + entity.cw_etype, attrs) |
581 self.doexec(cnx, sql, attrs) |
576 self.doexec(cnx, sql, attrs) |
582 if cnx.ertype_supports_undo(entity.cw_etype): |
577 if cnx.ertype_supports_undo(entity.cw_etype): |
583 self._record_tx_action(cnx, 'tx_entity_actions', u'C', |
578 self._record_tx_action(cnx, 'tx_entity_actions', u'C', |
584 etype=text_type(entity.cw_etype), eid=entity.eid) |
579 etype=entity.cw_etype, eid=entity.eid) |
585 |
580 |
586 def update_entity(self, cnx, entity): |
581 def update_entity(self, cnx, entity): |
587 """replace an entity in the source""" |
582 """replace an entity in the source""" |
588 with self._storage_handler(cnx, entity, 'updated'): |
583 with self._storage_handler(cnx, entity, 'updated'): |
589 attrs = self.preprocess_entity(entity) |
584 attrs = self.preprocess_entity(entity) |
590 if cnx.ertype_supports_undo(entity.cw_etype): |
585 if cnx.ertype_supports_undo(entity.cw_etype): |
591 changes = self._save_attrs(cnx, entity, attrs) |
586 changes = self._save_attrs(cnx, entity, attrs) |
592 self._record_tx_action(cnx, 'tx_entity_actions', u'U', |
587 self._record_tx_action(cnx, 'tx_entity_actions', u'U', |
593 etype=text_type(entity.cw_etype), eid=entity.eid, |
588 etype=entity.cw_etype, eid=entity.eid, |
594 changes=self._binary(pickle.dumps(changes))) |
589 changes=self._binary(pickle.dumps(changes))) |
595 sql = self.sqlgen.update(SQL_PREFIX + entity.cw_etype, attrs, |
590 sql = self.sqlgen.update(SQL_PREFIX + entity.cw_etype, attrs, |
596 ['cw_eid']) |
591 ['cw_eid']) |
597 self.doexec(cnx, sql, attrs) |
592 self.doexec(cnx, sql, attrs) |
598 |
593 |
603 attrs = [SQL_PREFIX + r.type |
598 attrs = [SQL_PREFIX + r.type |
604 for r in entity.e_schema.subject_relations() |
599 for r in entity.e_schema.subject_relations() |
605 if (r.final or r.inlined) and r not in VIRTUAL_RTYPES] |
600 if (r.final or r.inlined) and r not in VIRTUAL_RTYPES] |
606 changes = self._save_attrs(cnx, entity, attrs) |
601 changes = self._save_attrs(cnx, entity, attrs) |
607 self._record_tx_action(cnx, 'tx_entity_actions', u'D', |
602 self._record_tx_action(cnx, 'tx_entity_actions', u'D', |
608 etype=text_type(entity.cw_etype), eid=entity.eid, |
603 etype=entity.cw_etype, eid=entity.eid, |
609 changes=self._binary(pickle.dumps(changes))) |
604 changes=self._binary(pickle.dumps(changes))) |
610 attrs = {'cw_eid': entity.eid} |
605 attrs = {'cw_eid': entity.eid} |
611 sql = self.sqlgen.delete(SQL_PREFIX + entity.cw_etype, attrs) |
606 sql = self.sqlgen.delete(SQL_PREFIX + entity.cw_etype, attrs) |
612 self.doexec(cnx, sql, attrs) |
607 self.doexec(cnx, sql, attrs) |
613 |
608 |
614 def add_relation(self, cnx, subject, rtype, object, inlined=False): |
609 def add_relation(self, cnx, subject, rtype, object, inlined=False): |
615 """add a relation to the source""" |
610 """add a relation to the source""" |
616 self._add_relations(cnx, rtype, [(subject, object)], inlined) |
611 self._add_relations(cnx, rtype, [(subject, object)], inlined) |
617 if cnx.ertype_supports_undo(rtype): |
612 if cnx.ertype_supports_undo(rtype): |
618 self._record_tx_action(cnx, 'tx_relation_actions', u'A', |
613 self._record_tx_action(cnx, 'tx_relation_actions', u'A', |
619 eid_from=subject, rtype=text_type(rtype), eid_to=object) |
614 eid_from=subject, rtype=rtype, eid_to=object) |
620 |
615 |
621 def add_relations(self, cnx, rtype, subj_obj_list, inlined=False): |
616 def add_relations(self, cnx, rtype, subj_obj_list, inlined=False): |
622 """add a relations to the source""" |
617 """add a relations to the source""" |
623 self._add_relations(cnx, rtype, subj_obj_list, inlined) |
618 self._add_relations(cnx, rtype, subj_obj_list, inlined) |
624 if cnx.ertype_supports_undo(rtype): |
619 if cnx.ertype_supports_undo(rtype): |
625 for subject, object in subj_obj_list: |
620 for subject, object in subj_obj_list: |
626 self._record_tx_action(cnx, 'tx_relation_actions', u'A', |
621 self._record_tx_action(cnx, 'tx_relation_actions', u'A', |
627 eid_from=subject, rtype=text_type(rtype), eid_to=object) |
622 eid_from=subject, rtype=rtype, eid_to=object) |
628 |
623 |
629 def _add_relations(self, cnx, rtype, subj_obj_list, inlined=False): |
624 def _add_relations(self, cnx, rtype, subj_obj_list, inlined=False): |
630 """add a relation to the source""" |
625 """add a relation to the source""" |
631 sql = [] |
626 sql = [] |
632 if inlined is False: |
627 if inlined is False: |
654 """delete a relation from the source""" |
649 """delete a relation from the source""" |
655 rschema = self.schema.rschema(rtype) |
650 rschema = self.schema.rschema(rtype) |
656 self._delete_relation(cnx, subject, rtype, object, rschema.inlined) |
651 self._delete_relation(cnx, subject, rtype, object, rschema.inlined) |
657 if cnx.ertype_supports_undo(rtype): |
652 if cnx.ertype_supports_undo(rtype): |
658 self._record_tx_action(cnx, 'tx_relation_actions', u'R', |
653 self._record_tx_action(cnx, 'tx_relation_actions', u'R', |
659 eid_from=subject, rtype=text_type(rtype), eid_to=object) |
654 eid_from=subject, rtype=rtype, eid_to=object) |
660 |
655 |
661 def _delete_relation(self, cnx, subject, rtype, object, inlined=False): |
656 def _delete_relation(self, cnx, subject, rtype, object, inlined=False): |
662 """delete a relation from the source""" |
657 """delete a relation from the source""" |
663 if inlined: |
658 if inlined: |
664 table = SQL_PREFIX + cnx.entity_type(subject) |
659 table = SQL_PREFIX + cnx.entity_type(subject) |
830 |
825 |
831 def add_info(self, cnx, entity, source): |
826 def add_info(self, cnx, entity, source): |
832 """add type and source info for an eid into the system table""" |
827 """add type and source info for an eid into the system table""" |
833 assert cnx.cnxset is not None |
828 assert cnx.cnxset is not None |
834 # begin by inserting eid/type/source into the entities table |
829 # begin by inserting eid/type/source into the entities table |
835 attrs = {'type': text_type(entity.cw_etype), 'eid': entity.eid} |
830 attrs = {'type': entity.cw_etype, 'eid': entity.eid} |
836 self._handle_insert_entity_sql(cnx, self.sqlgen.insert('entities', attrs), attrs) |
831 self._handle_insert_entity_sql(cnx, self.sqlgen.insert('entities', attrs), attrs) |
837 # insert core relations: is, is_instance_of and cw_source |
832 # insert core relations: is, is_instance_of and cw_source |
838 |
833 |
839 if entity.e_schema.eid is not None: # else schema has not yet been serialized |
834 if entity.e_schema.eid is not None: # else schema has not yet been serialized |
840 self._handle_is_relation_sql( |
835 self._handle_is_relation_sql( |
905 if key == 'etype': |
900 if key == 'etype': |
906 # filtering on etype implies filtering on entity actions |
901 # filtering on etype implies filtering on entity actions |
907 # only, and with no eid specified |
902 # only, and with no eid specified |
908 assert actionfilters.get('action', 'C') in 'CUD' |
903 assert actionfilters.get('action', 'C') in 'CUD' |
909 assert 'eid' not in actionfilters |
904 assert 'eid' not in actionfilters |
910 tearestr['etype'] = text_type(val) |
905 tearestr['etype'] = val |
911 elif key == 'eid': |
906 elif key == 'eid': |
912 # eid filter may apply to 'eid' of tx_entity_actions or to |
907 # eid filter may apply to 'eid' of tx_entity_actions or to |
913 # 'eid_from' OR 'eid_to' of tx_relation_actions |
908 # 'eid_from' OR 'eid_to' of tx_relation_actions |
914 if actionfilters.get('action', 'C') in 'CUD': |
909 if actionfilters.get('action', 'C') in 'CUD': |
915 tearestr['eid'] = val |
910 tearestr['eid'] = val |
916 if actionfilters.get('action', 'A') in 'AR': |
911 if actionfilters.get('action', 'A') in 'AR': |
917 trarestr['eid_from'] = val |
912 trarestr['eid_from'] = val |
918 trarestr['eid_to'] = val |
913 trarestr['eid_to'] = val |
919 elif key == 'action': |
914 elif key == 'action': |
920 if val in 'CUD': |
915 if val in 'CUD': |
921 tearestr['txa_action'] = text_type(val) |
916 tearestr['txa_action'] = val |
922 else: |
917 else: |
923 assert val in 'AR' |
918 assert val in 'AR' |
924 trarestr['txa_action'] = text_type(val) |
919 trarestr['txa_action'] = val |
925 else: |
920 else: |
926 raise AssertionError('unknow filter %s' % key) |
921 raise AssertionError('unknow filter %s' % key) |
927 assert trarestr or tearestr, "can't only filter on 'public'" |
922 assert trarestr or tearestr, "can't only filter on 'public'" |
928 subqsqls = [] |
923 subqsqls = [] |
929 # append subqueries to the original query, using EXISTS() |
924 # append subqueries to the original query, using EXISTS() |
953 # turn results into transaction objects |
948 # turn results into transaction objects |
954 return [tx.Transaction(cnx, *args) for args in cu.fetchall()] |
949 return [tx.Transaction(cnx, *args) for args in cu.fetchall()] |
955 |
950 |
956 def tx_info(self, cnx, txuuid): |
951 def tx_info(self, cnx, txuuid): |
957 """See :class:`cubicweb.repoapi.Connection.transaction_info`""" |
952 """See :class:`cubicweb.repoapi.Connection.transaction_info`""" |
958 return tx.Transaction(cnx, txuuid, *self._tx_info(cnx, text_type(txuuid))) |
953 return tx.Transaction(cnx, txuuid, *self._tx_info(cnx, txuuid)) |
959 |
954 |
960 def tx_actions(self, cnx, txuuid, public): |
955 def tx_actions(self, cnx, txuuid, public): |
961 """See :class:`cubicweb.repoapi.Connection.transaction_actions`""" |
956 """See :class:`cubicweb.repoapi.Connection.transaction_actions`""" |
962 txuuid = text_type(txuuid) |
|
963 self._tx_info(cnx, txuuid) |
957 self._tx_info(cnx, txuuid) |
964 restr = {'tx_uuid': txuuid} |
958 restr = {'tx_uuid': txuuid} |
965 if public: |
959 if public: |
966 restr['txa_public'] = True |
960 restr['txa_public'] = True |
967 # XXX use generator to avoid loading everything in memory? |
961 # XXX use generator to avoid loading everything in memory? |
1131 errors = [] |
1123 errors = [] |
1132 subj, rtype, obj = action.eid_from, action.rtype, action.eid_to |
1124 subj, rtype, obj = action.eid_from, action.rtype, action.eid_to |
1133 try: |
1125 try: |
1134 sentity, oentity, rdef = _undo_rel_info(cnx, subj, rtype, obj) |
1126 sentity, oentity, rdef = _undo_rel_info(cnx, subj, rtype, obj) |
1135 except _UndoException as ex: |
1127 except _UndoException as ex: |
1136 errors.append(text_type(ex)) |
1128 errors.append(str(ex)) |
1137 else: |
1129 else: |
1138 for role, entity in (('subject', sentity), |
1130 for role, entity in (('subject', sentity), |
1139 ('object', oentity)): |
1131 ('object', oentity)): |
1140 try: |
1132 try: |
1141 _undo_check_relation_target(entity, rdef, role) |
1133 _undo_check_relation_target(entity, rdef, role) |
1142 except _UndoException as ex: |
1134 except _UndoException as ex: |
1143 errors.append(text_type(ex)) |
1135 errors.append(str(ex)) |
1144 continue |
1136 continue |
1145 if not errors: |
1137 if not errors: |
1146 self.repo.hm.call_hooks('before_add_relation', cnx, |
1138 self.repo.hm.call_hooks('before_add_relation', cnx, |
1147 eidfrom=subj, rtype=rtype, eidto=obj) |
1139 eidfrom=subj, rtype=rtype, eidto=obj) |
1148 # add relation in the database |
1140 # add relation in the database |
1213 errors = [] |
1205 errors = [] |
1214 subj, rtype, obj = action.eid_from, action.rtype, action.eid_to |
1206 subj, rtype, obj = action.eid_from, action.rtype, action.eid_to |
1215 try: |
1207 try: |
1216 sentity, oentity, rdef = _undo_rel_info(cnx, subj, rtype, obj) |
1208 sentity, oentity, rdef = _undo_rel_info(cnx, subj, rtype, obj) |
1217 except _UndoException as ex: |
1209 except _UndoException as ex: |
1218 errors.append(text_type(ex)) |
1210 errors.append(str(ex)) |
1219 else: |
1211 else: |
1220 rschema = rdef.rtype |
1212 rschema = rdef.rtype |
1221 if rschema.inlined: |
1213 if rschema.inlined: |
1222 sql = 'SELECT 1 FROM cw_%s WHERE cw_eid=%s and cw_%s=%s'\ |
1214 sql = 'SELECT 1 FROM cw_%s WHERE cw_eid=%s and cw_%s=%s'\ |
1223 % (sentity.cw_etype, subj, rtype, obj) |
1215 % (sentity.cw_etype, subj, rtype, obj) |