1 # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
|
2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
|
3 # |
|
4 # This file is part of CubicWeb. |
|
5 # |
|
6 # CubicWeb is free software: you can redistribute it and/or modify it under the |
|
7 # terms of the GNU Lesser General Public License as published by the Free |
|
8 # Software Foundation, either version 2.1 of the License, or (at your option) |
|
9 # any later version. |
|
10 # |
|
11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
|
12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
|
13 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
|
14 # details. |
|
15 # |
|
16 # You should have received a copy of the GNU Lesser General Public License along |
|
17 # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
|
18 """custom storages for the system source""" |
|
19 |
|
20 import os |
|
21 import sys |
|
22 from os import unlink, path as osp |
|
23 from contextlib import contextmanager |
|
24 import tempfile |
|
25 |
|
26 from six import PY2, PY3, text_type, binary_type |
|
27 |
|
28 from logilab.common import nullobject |
|
29 |
|
30 from yams.schema import role_name |
|
31 |
|
32 from cubicweb import Binary, ValidationError |
|
33 from cubicweb.server import hook |
|
34 from cubicweb.server.edition import EditedEntity |
|
35 |
|
36 |
|
37 def set_attribute_storage(repo, etype, attr, storage): |
|
38 repo.system_source.set_storage(etype, attr, storage) |
|
39 |
|
40 def unset_attribute_storage(repo, etype, attr): |
|
41 repo.system_source.unset_storage(etype, attr) |
|
42 |
|
43 |
|
44 class Storage(object): |
|
45 """abstract storage |
|
46 |
|
47 * If `source_callback` is true (by default), the callback will be run during |
|
48 query result process of fetched attribute's value and should have the |
|
49 following prototype:: |
|
50 |
|
51 callback(self, source, cnx, value) |
|
52 |
|
53 where `value` is the value actually stored in the backend. None values |
|
54 will be skipped (eg callback won't be called). |
|
55 |
|
56 * if `source_callback` is false, the callback will be run during sql |
|
57 generation when some attribute with a custom storage is accessed and |
|
58 should have the following prototype:: |
|
59 |
|
60 callback(self, generator, relation, linkedvar) |
|
61 |
|
62 where `generator` is the sql generator, `relation` the current rql syntax |
|
63 tree relation and linkedvar the principal syntax tree variable holding the |
|
64 attribute. |
|
65 """ |
|
66 is_source_callback = True |
|
67 |
|
68 def callback(self, *args): |
|
69 """see docstring for prototype, which vary according to is_source_callback |
|
70 """ |
|
71 raise NotImplementedError() |
|
72 |
|
73 def entity_added(self, entity, attr): |
|
74 """an entity using this storage for attr has been added""" |
|
75 raise NotImplementedError() |
|
76 def entity_updated(self, entity, attr): |
|
77 """an entity using this storage for attr has been updatded""" |
|
78 raise NotImplementedError() |
|
79 def entity_deleted(self, entity, attr): |
|
80 """an entity using this storage for attr has been deleted""" |
|
81 raise NotImplementedError() |
|
82 def migrate_entity(self, entity, attribute): |
|
83 """migrate an entity attribute to the storage""" |
|
84 raise NotImplementedError() |
|
85 |
|
86 # TODO |
|
87 # * make it configurable without code |
|
88 # * better file path attribution |
|
89 # * handle backup/restore |
|
90 |
|
91 def uniquify_path(dirpath, basename): |
|
92 """return a file descriptor and unique file name for `basename` in `dirpath` |
|
93 """ |
|
94 path = basename.replace(osp.sep, '-') |
|
95 base, ext = osp.splitext(path) |
|
96 return tempfile.mkstemp(prefix=base, suffix=ext, dir=dirpath) |
|
97 |
|
98 @contextmanager |
|
99 def fsimport(cnx): |
|
100 present = 'fs_importing' in cnx.transaction_data |
|
101 old_value = cnx.transaction_data.get('fs_importing') |
|
102 cnx.transaction_data['fs_importing'] = True |
|
103 yield |
|
104 if present: |
|
105 cnx.transaction_data['fs_importing'] = old_value |
|
106 else: |
|
107 del cnx.transaction_data['fs_importing'] |
|
108 |
|
109 |
|
110 _marker = nullobject() |
|
111 |
|
112 |
|
113 class BytesFileSystemStorage(Storage): |
|
114 """store Bytes attribute value on the file system""" |
|
115 def __init__(self, defaultdir, fsencoding=_marker, wmode=0o444): |
|
116 if PY3: |
|
117 if not isinstance(defaultdir, text_type): |
|
118 raise TypeError('defaultdir must be a unicode object in python 3') |
|
119 if fsencoding is not _marker: |
|
120 raise ValueError('fsencoding is no longer supported in python 3') |
|
121 else: |
|
122 self.fsencoding = fsencoding or 'utf-8' |
|
123 if isinstance(defaultdir, text_type): |
|
124 defaultdir = defaultdir.encode(fsencoding) |
|
125 self.default_directory = defaultdir |
|
126 # extra umask to use when creating file |
|
127 # 0444 as in "only allow read bit in permission" |
|
128 self._wmode = wmode |
|
129 |
|
130 def _writecontent(self, fd, binary): |
|
131 """write the content of a binary in readonly file |
|
132 |
|
133 As the bfss never alters an existing file it does not prevent it from |
|
134 working as intended. This is a better safe than sorry approach. |
|
135 """ |
|
136 os.fchmod(fd, self._wmode) |
|
137 fileobj = os.fdopen(fd, 'wb') |
|
138 binary.to_file(fileobj) |
|
139 fileobj.close() |
|
140 |
|
141 |
|
142 def callback(self, source, cnx, value): |
|
143 """sql generator callback when some attribute with a custom storage is |
|
144 accessed |
|
145 """ |
|
146 fpath = source.binary_to_str(value) |
|
147 try: |
|
148 return Binary.from_file(fpath) |
|
149 except EnvironmentError as ex: |
|
150 source.critical("can't open %s: %s", value, ex) |
|
151 return None |
|
152 |
|
153 def entity_added(self, entity, attr): |
|
154 """an entity using this storage for attr has been added""" |
|
155 if entity._cw.transaction_data.get('fs_importing'): |
|
156 binary = Binary.from_file(entity.cw_edited[attr].getvalue()) |
|
157 entity._cw_dont_cache_attribute(attr, repo_side=True) |
|
158 else: |
|
159 binary = entity.cw_edited.pop(attr) |
|
160 fd, fpath = self.new_fs_path(entity, attr) |
|
161 # bytes storage used to store file's path |
|
162 binary_obj = Binary(fpath if PY2 else fpath.encode('utf-8')) |
|
163 entity.cw_edited.edited_attribute(attr, binary_obj) |
|
164 self._writecontent(fd, binary) |
|
165 AddFileOp.get_instance(entity._cw).add_data(fpath) |
|
166 return binary |
|
167 |
|
168 def entity_updated(self, entity, attr): |
|
169 """an entity using this storage for attr has been updated""" |
|
170 # get the name of the previous file containing the value |
|
171 oldpath = self.current_fs_path(entity, attr) |
|
172 if entity._cw.transaction_data.get('fs_importing'): |
|
173 # If we are importing from the filesystem, the file already exists. |
|
174 # We do not need to create it but we need to fetch the content of |
|
175 # the file as the actual content of the attribute |
|
176 fpath = entity.cw_edited[attr].getvalue() |
|
177 entity._cw_dont_cache_attribute(attr, repo_side=True) |
|
178 assert fpath is not None |
|
179 binary = Binary.from_file(fpath) |
|
180 else: |
|
181 # We must store the content of the attributes |
|
182 # into a file to stay consistent with the behaviour of entity_add. |
|
183 # Moreover, the BytesFileSystemStorage expects to be able to |
|
184 # retrieve the current value of the attribute at anytime by reading |
|
185 # the file on disk. To be able to rollback things, use a new file |
|
186 # and keep the old one that will be removed on commit if everything |
|
187 # went ok. |
|
188 # |
|
189 # fetch the current attribute value in memory |
|
190 binary = entity.cw_edited.pop(attr) |
|
191 if binary is None: |
|
192 fpath = None |
|
193 else: |
|
194 # Get filename for it |
|
195 fd, fpath = self.new_fs_path(entity, attr) |
|
196 # write attribute value on disk |
|
197 self._writecontent(fd, binary) |
|
198 # Mark the new file as added during the transaction. |
|
199 # The file will be removed on rollback |
|
200 AddFileOp.get_instance(entity._cw).add_data(fpath) |
|
201 # reinstall poped value |
|
202 if fpath is None: |
|
203 entity.cw_edited.edited_attribute(attr, None) |
|
204 else: |
|
205 # register the new location for the file. |
|
206 binary_obj = Binary(fpath if PY2 else fpath.encode('utf-8')) |
|
207 entity.cw_edited.edited_attribute(attr, binary_obj) |
|
208 if oldpath is not None and oldpath != fpath: |
|
209 # Mark the old file as useless so the file will be removed at |
|
210 # commit. |
|
211 DeleteFileOp.get_instance(entity._cw).add_data(oldpath) |
|
212 return binary |
|
213 |
|
214 def entity_deleted(self, entity, attr): |
|
215 """an entity using this storage for attr has been deleted""" |
|
216 fpath = self.current_fs_path(entity, attr) |
|
217 if fpath is not None: |
|
218 DeleteFileOp.get_instance(entity._cw).add_data(fpath) |
|
219 |
|
220 def new_fs_path(self, entity, attr): |
|
221 # We try to get some hint about how to name the file using attribute's |
|
222 # name metadata, so we use the real file name and extension when |
|
223 # available. Keeping the extension is useful for example in the case of |
|
224 # PIL processing that use filename extension to detect content-type, as |
|
225 # well as providing more understandable file names on the fs. |
|
226 if PY2: |
|
227 attr = attr.encode('ascii') |
|
228 basename = [str(entity.eid), attr] |
|
229 name = entity.cw_attr_metadata(attr, 'name') |
|
230 if name is not None: |
|
231 basename.append(name.encode(self.fsencoding) if PY2 else name) |
|
232 fd, fspath = uniquify_path(self.default_directory, |
|
233 '_'.join(basename)) |
|
234 if fspath is None: |
|
235 msg = entity._cw._('failed to uniquify path (%s, %s)') % ( |
|
236 self.default_directory, '_'.join(basename)) |
|
237 raise ValidationError(entity.eid, {role_name(attr, 'subject'): msg}) |
|
238 assert isinstance(fspath, str) # bytes on py2, unicode on py3 |
|
239 return fd, fspath |
|
240 |
|
241 def current_fs_path(self, entity, attr): |
|
242 """return the current fs_path of the attribute, or None is the attr is |
|
243 not stored yet. |
|
244 """ |
|
245 sysource = entity._cw.repo.system_source |
|
246 cu = sysource.doexec(entity._cw, |
|
247 'SELECT cw_%s FROM cw_%s WHERE cw_eid=%s' % ( |
|
248 attr, entity.cw_etype, entity.eid)) |
|
249 rawvalue = cu.fetchone()[0] |
|
250 if rawvalue is None: # no previous value |
|
251 return None |
|
252 fspath = sysource._process_value(rawvalue, cu.description[0], |
|
253 binarywrap=binary_type) |
|
254 if PY3: |
|
255 fspath = fspath.decode('utf-8') |
|
256 assert isinstance(fspath, str) # bytes on py2, unicode on py3 |
|
257 return fspath |
|
258 |
|
259 def migrate_entity(self, entity, attribute): |
|
260 """migrate an entity attribute to the storage""" |
|
261 entity.cw_edited = EditedEntity(entity, **entity.cw_attr_cache) |
|
262 self.entity_added(entity, attribute) |
|
263 cnx = entity._cw |
|
264 source = cnx.repo.system_source |
|
265 attrs = source.preprocess_entity(entity) |
|
266 sql = source.sqlgen.update('cw_' + entity.cw_etype, attrs, |
|
267 ['cw_eid']) |
|
268 source.doexec(cnx, sql, attrs) |
|
269 entity.cw_edited = None |
|
270 |
|
271 |
|
272 class AddFileOp(hook.DataOperationMixIn, hook.Operation): |
|
273 def rollback_event(self): |
|
274 for filepath in self.get_data(): |
|
275 assert isinstance(filepath, str) # bytes on py2, unicode on py3 |
|
276 try: |
|
277 unlink(filepath) |
|
278 except Exception as ex: |
|
279 self.error("can't remove %s: %s" % (filepath, ex)) |
|
280 |
|
281 class DeleteFileOp(hook.DataOperationMixIn, hook.Operation): |
|
282 def postcommit_event(self): |
|
283 for filepath in self.get_data(): |
|
284 assert isinstance(filepath, str) # bytes on py2, unicode on py3 |
|
285 try: |
|
286 unlink(filepath) |
|
287 except Exception as ex: |
|
288 self.error("can't remove %s: %s" % (filepath, ex)) |
|