author | Adrien Di Mascio <Adrien.DiMascio@logilab.fr> |
Wed, 16 Sep 2009 21:29:17 +0200 | |
branch | 3.5 |
changeset 3275 | 5247789df541 |
parent 3240 | 8604a15995d1 |
child 4212 | ab6573088b4a |
permissions | -rw-r--r-- |
0 | 1 |
"""Adapter for google appengine source. |
2 |
||
3 |
:organization: Logilab |
|
1977
606923dff11b
big bunch of copyright / docstring update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
4 |
:copyright: 2008-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. |
0 | 5 |
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr |
1977
606923dff11b
big bunch of copyright / docstring update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1802
diff
changeset
|
6 |
:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses |
0 | 7 |
""" |
8 |
__docformat__ = "restructuredtext en" |
|
9 |
||
1132 | 10 |
from cubicweb import AuthenticationError, UnknownEid |
0 | 11 |
from cubicweb.server.sources import AbstractSource, ConnectionWrapper |
12 |
from cubicweb.server.pool import SingleOperation |
|
13 |
from cubicweb.server.utils import crypt_password |
|
14 |
from cubicweb.goa.dbinit import set_user_groups |
|
15 |
from cubicweb.goa.rqlinterpreter import RQLInterpreter |
|
16 |
||
1132 | 17 |
from google.appengine.api.datastore import Key, Entity, Put, Delete |
0 | 18 |
from google.appengine.api import datastore_errors, users |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
19 |
|
0 | 20 |
def _init_groups(guser, euser): |
21 |
# set default groups |
|
22 |
if guser is None: |
|
23 |
groups = ['guests'] |
|
24 |
else: |
|
25 |
groups = ['users'] |
|
26 |
if users.is_current_user_admin(): |
|
27 |
groups.append('managers') |
|
28 |
set_user_groups(euser, groups) |
|
29 |
||
30 |
def _clear_related_cache(session, gaesubject, rtype, gaeobject): |
|
31 |
subject, object = str(gaesubject.key()), str(gaeobject.key()) |
|
32 |
for eid, role in ((subject, 'subject'), (object, 'object')): |
|
33 |
# clear related cache if necessary |
|
34 |
try: |
|
35 |
entity = session.entity_cache(eid) |
|
36 |
except KeyError: |
|
37 |
pass |
|
38 |
else: |
|
39 |
entity.clear_related_cache(rtype, role) |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1132
diff
changeset
|
40 |
if gaesubject.kind() == 'CWUser': |
0 | 41 |
for asession in session.repo._sessions.itervalues(): |
42 |
if asession.user.eid == subject: |
|
43 |
asession.user.clear_related_cache(rtype, 'subject') |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1132
diff
changeset
|
44 |
if gaeobject.kind() == 'CWUser': |
0 | 45 |
for asession in session.repo._sessions.itervalues(): |
46 |
if asession.user.eid == object: |
|
47 |
asession.user.clear_related_cache(rtype, 'object') |
|
48 |
||
49 |
def _mark_modified(session, gaeentity): |
|
2102
268659907769
finish to update transaction data api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
50 |
modified = session.transaction_data.setdefault('modifiedentities', {}) |
0 | 51 |
modified[str(gaeentity.key())] = gaeentity |
52 |
DatastorePutOp(session) |
|
53 |
||
54 |
def _rinfo(session, subject, rtype, object): |
|
55 |
gaesubj = session.datastore_get(subject) |
|
56 |
gaeobj = session.datastore_get(object) |
|
57 |
rschema = session.vreg.schema.rschema(rtype) |
|
58 |
cards = rschema.rproperty(gaesubj.kind(), gaeobj.kind(), 'cardinality') |
|
59 |
return gaesubj, gaeobj, cards |
|
60 |
||
61 |
def _radd(session, gaeentity, targetkey, relation, card): |
|
62 |
if card in '?1': |
|
63 |
gaeentity[relation] = targetkey |
|
64 |
else: |
|
65 |
try: |
|
66 |
related = gaeentity[relation] |
|
67 |
except KeyError: |
|
68 |
related = [] |
|
69 |
else: |
|
70 |
if related is None: |
|
71 |
related = [] |
|
72 |
related.append(targetkey) |
|
73 |
gaeentity[relation] = related |
|
74 |
_mark_modified(session, gaeentity) |
|
75 |
||
76 |
def _rdel(session, gaeentity, targetkey, relation, card): |
|
77 |
if card in '?1': |
|
78 |
gaeentity[relation] = None |
|
79 |
else: |
|
80 |
related = gaeentity[relation] |
|
81 |
if related is not None: |
|
82 |
related = [key for key in related if not key == targetkey] |
|
83 |
gaeentity[relation] = related or None |
|
84 |
_mark_modified(session, gaeentity) |
|
85 |
||
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
86 |
|
0 | 87 |
class DatastorePutOp(SingleOperation): |
88 |
"""delayed put of entities to have less datastore write api calls |
|
89 |
||
90 |
* save all modified entities at precommit (should be the first operation |
|
91 |
processed, hence the 0 returned by insert_index()) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
92 |
|
0 | 93 |
* in case others precommit operations modify some entities, resave modified |
94 |
entities at commit. This suppose that no db changes will occurs during |
|
95 |
commit event but it should be the case. |
|
96 |
""" |
|
97 |
def insert_index(self): |
|
98 |
return 0 |
|
99 |
||
100 |
def _put_entities(self): |
|
2102
268659907769
finish to update transaction data api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
101 |
pending = self.session.transaction_data.get('pendingeids', ()) |
268659907769
finish to update transaction data api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
102 |
modified = self.session.transaction_data.get('modifiedentities', {}) |
0 | 103 |
for eid, gaeentity in modified.iteritems(): |
104 |
assert not eid in pending |
|
105 |
Put(gaeentity) |
|
106 |
modified.clear() |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
107 |
|
0 | 108 |
def commit_event(self): |
109 |
self._put_entities() |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
110 |
|
0 | 111 |
def precommit_event(self): |
112 |
self._put_entities() |
|
113 |
||
114 |
||
115 |
class GAESource(AbstractSource): |
|
116 |
"""adapter for a system source on top of google appengine datastore""" |
|
117 |
||
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1132
diff
changeset
|
118 |
passwd_rql = "Any P WHERE X is CWUser, X login %(login)s, X upassword P" |
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1132
diff
changeset
|
119 |
auth_rql = "Any X WHERE X is CWUser, X login %(login)s, X upassword %(pwd)s" |
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1132
diff
changeset
|
120 |
_sols = ({'X': 'CWUser', 'P': 'Password'},) |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
121 |
|
0 | 122 |
options = () |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
123 |
|
0 | 124 |
def __init__(self, repo, appschema, source_config, *args, **kwargs): |
125 |
AbstractSource.__init__(self, repo, appschema, source_config, |
|
126 |
*args, **kwargs) |
|
127 |
if repo.config['use-google-auth']: |
|
128 |
self.info('using google authentication service') |
|
129 |
self.authenticate = self.authenticate_gauth |
|
130 |
else: |
|
131 |
self.authenticate = self.authenticate_local |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
132 |
|
0 | 133 |
def reset_caches(self): |
134 |
"""method called during test to reset potential source caches""" |
|
135 |
pass |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
136 |
|
0 | 137 |
def init_creating(self): |
138 |
pass |
|
139 |
||
140 |
def init(self): |
|
141 |
# XXX unregister unsupported hooks |
|
142 |
from cubicweb.server.hooks import sync_owner_after_add_composite_relation |
|
143 |
self.repo.hm.unregister_hook(sync_owner_after_add_composite_relation, |
|
144 |
'after_add_relation', '') |
|
145 |
||
146 |
def get_connection(self): |
|
147 |
return ConnectionWrapper() |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
148 |
|
0 | 149 |
# ISource interface ####################################################### |
150 |
||
151 |
def compile_rql(self, rql): |
|
3240
8604a15995d1
refactor so that rql rewriter may be used outside the server. Enhance it to be usable for RRQLExpression as well
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
152 |
rqlst = self.repo.vreg.parse(rql) |
0 | 153 |
rqlst.restricted_vars = () |
154 |
rqlst.children[0].solutions = self._sols |
|
155 |
return rqlst |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
156 |
|
0 | 157 |
def set_schema(self, schema): |
2476
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2102
diff
changeset
|
158 |
"""set the instance'schema""" |
0 | 159 |
self.interpreter = RQLInterpreter(schema) |
160 |
self.schema = schema |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1132
diff
changeset
|
161 |
if 'CWUser' in schema and not self.repo.config['use-google-auth']: |
0 | 162 |
# rql syntax trees used to authenticate users |
163 |
self._passwd_rqlst = self.compile_rql(self.passwd_rql) |
|
164 |
self._auth_rqlst = self.compile_rql(self.auth_rql) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
165 |
|
0 | 166 |
def support_entity(self, etype, write=False): |
167 |
"""return true if the given entity's type is handled by this adapter |
|
168 |
if write is true, return true only if it's a RW support |
|
169 |
""" |
|
170 |
return True |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
171 |
|
0 | 172 |
def support_relation(self, rtype, write=False): |
173 |
"""return true if the given relation's type is handled by this adapter |
|
174 |
if write is true, return true only if it's a RW support |
|
175 |
""" |
|
176 |
return True |
|
177 |
||
178 |
def authenticate_gauth(self, session, login, password): |
|
179 |
guser = users.get_current_user() |
|
180 |
# allowing or not anonymous connection should be done in the app.yaml |
|
181 |
# file, suppose it's authorized if we are there |
|
182 |
if guser is None: |
|
183 |
login = u'anonymous' |
|
184 |
else: |
|
185 |
login = unicode(guser.nickname()) |
|
186 |
# XXX http://code.google.com/appengine/docs/users/userobjects.html |
|
187 |
# use a reference property to automatically work with email address |
|
188 |
# changes after the propagation feature is implemented |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1132
diff
changeset
|
189 |
key = Key.from_path('CWUser', 'key_' + login, parent=None) |
0 | 190 |
try: |
191 |
euser = session.datastore_get(key) |
|
192 |
# XXX fix user. Required until we find a better way to fix broken records |
|
193 |
if not euser.get('s_in_group'): |
|
194 |
_init_groups(guser, euser) |
|
195 |
Put(euser) |
|
196 |
return str(key) |
|
197 |
except datastore_errors.EntityNotFoundError: |
|
198 |
# create a record for this user |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1132
diff
changeset
|
199 |
euser = Entity('CWUser', name='key_' + login) |
0 | 200 |
euser['s_login'] = login |
201 |
_init_groups(guser, euser) |
|
202 |
Put(euser) |
|
203 |
return str(euser.key()) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
204 |
|
0 | 205 |
def authenticate_local(self, session, login, password): |
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1132
diff
changeset
|
206 |
"""return CWUser eid for the given login/password if this account is |
0 | 207 |
defined in this source, else raise `AuthenticationError` |
208 |
||
209 |
two queries are needed since passwords are stored crypted, so we have |
|
210 |
to fetch the salt first |
|
211 |
""" |
|
212 |
args = {'login': login, 'pwd' : password} |
|
213 |
if password is not None: |
|
214 |
rset = self.syntax_tree_search(session, self._passwd_rqlst, args) |
|
215 |
try: |
|
216 |
pwd = rset[0][0] |
|
217 |
except IndexError: |
|
218 |
raise AuthenticationError('bad login') |
|
219 |
# passwords are stored using the bytea type, so we get a StringIO |
|
220 |
if pwd is not None: |
|
221 |
args['pwd'] = crypt_password(password, pwd[:2]) |
|
222 |
# get eid from login and (crypted) password |
|
223 |
rset = self.syntax_tree_search(session, self._auth_rqlst, args) |
|
224 |
try: |
|
225 |
return rset[0][0] |
|
226 |
except IndexError: |
|
227 |
raise AuthenticationError('bad password') |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
228 |
|
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
229 |
def syntax_tree_search(self, session, union, args=None, cachekey=None, |
0 | 230 |
varmap=None): |
231 |
"""return result from this source for a rql query (actually from a rql |
|
232 |
syntax tree and a solution dictionary mapping each used variable to a |
|
233 |
possible type). If cachekey is given, the query necessary to fetch the |
|
234 |
results (but not the results themselves) may be cached using this key. |
|
235 |
""" |
|
236 |
results, description = self.interpreter.interpret(union, args, |
|
237 |
session.datastore_get) |
|
238 |
return results # XXX description |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
239 |
|
0 | 240 |
def flying_insert(self, table, session, union, args=None, varmap=None): |
241 |
raise NotImplementedError |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
242 |
|
0 | 243 |
def add_entity(self, session, entity): |
244 |
"""add a new entity to the source""" |
|
245 |
# do not delay add_entity as other modifications, new created entity |
|
246 |
# needs an eid |
|
247 |
entity.put() |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
248 |
|
0 | 249 |
def update_entity(self, session, entity): |
250 |
"""replace an entity in the source""" |
|
251 |
gaeentity = entity.to_gae_model() |
|
252 |
_mark_modified(session, entity.to_gae_model()) |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1132
diff
changeset
|
253 |
if gaeentity.kind() == 'CWUser': |
0 | 254 |
for asession in self.repo._sessions.itervalues(): |
255 |
if asession.user.eid == entity.eid: |
|
256 |
asession.user.update(dict(gaeentity)) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
257 |
|
0 | 258 |
def delete_entity(self, session, etype, eid): |
259 |
"""delete an entity from the source""" |
|
260 |
# do not delay delete_entity as other modifications to ensure |
|
261 |
# consistency |
|
262 |
key = Key(eid) |
|
263 |
Delete(key) |
|
264 |
session.clear_datastore_cache(key) |
|
265 |
session.drop_entity_cache(eid) |
|
2102
268659907769
finish to update transaction data api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
266 |
session.transaction_data.get('modifiedentities', {}).pop(eid, None) |
0 | 267 |
|
268 |
def add_relation(self, session, subject, rtype, object): |
|
269 |
"""add a relation to the source""" |
|
270 |
gaesubj, gaeobj, cards = _rinfo(session, subject, rtype, object) |
|
271 |
_radd(session, gaesubj, gaeobj.key(), 's_' + rtype, cards[0]) |
|
272 |
_radd(session, gaeobj, gaesubj.key(), 'o_' + rtype, cards[1]) |
|
273 |
_clear_related_cache(session, gaesubj, rtype, gaeobj) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
274 |
|
0 | 275 |
def delete_relation(self, session, subject, rtype, object): |
276 |
"""delete a relation from the source""" |
|
277 |
gaesubj, gaeobj, cards = _rinfo(session, subject, rtype, object) |
|
2102
268659907769
finish to update transaction data api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
278 |
pending = session.transaction_data.setdefault('pendingeids', set()) |
0 | 279 |
if not subject in pending: |
280 |
_rdel(session, gaesubj, gaeobj.key(), 's_' + rtype, cards[0]) |
|
281 |
if not object in pending: |
|
282 |
_rdel(session, gaeobj, gaesubj.key(), 'o_' + rtype, cards[1]) |
|
283 |
_clear_related_cache(session, gaesubj, rtype, gaeobj) |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
284 |
|
0 | 285 |
# system source interface ################################################# |
286 |
||
287 |
def eid_type_source(self, session, eid): |
|
288 |
"""return a tuple (type, source, extid) for the entity with id <eid>""" |
|
289 |
try: |
|
290 |
key = Key(eid) |
|
291 |
except datastore_errors.BadKeyError: |
|
292 |
raise UnknownEid(eid) |
|
293 |
return key.kind(), 'system', None |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
294 |
|
0 | 295 |
def create_eid(self, session): |
296 |
return None # let the datastore generating key |
|
297 |
||
298 |
def add_info(self, session, entity, source, extid=None): |
|
299 |
"""add type and source info for an eid into the system table""" |
|
300 |
pass |
|
301 |
||
302 |
def delete_info(self, session, eid, etype, uri, extid): |
|
303 |
"""delete system information on deletion of an entity by transfering |
|
304 |
record from the entities table to the deleted_entities table |
|
305 |
""" |
|
306 |
pass |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
307 |
|
0 | 308 |
def fti_unindex_entity(self, session, eid): |
309 |
"""remove text content for entity with the given eid from the full text |
|
310 |
index |
|
311 |
""" |
|
312 |
pass |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
313 |
|
0 | 314 |
def fti_index_entity(self, session, entity): |
315 |
"""add text content of a created/modified entity to the full text index |
|
316 |
""" |
|
317 |
pass |