fix potential problems when BFSS uses a Windows SMB share (closes #2131435)
files on windows network share must be read / written by chunks of reasonable size
or you get some unusual os level errors.
--- a/__init__.py Mon Dec 12 12:09:49 2011 +0100
+++ b/__init__.py Mon Dec 19 19:03:27 2011 +0100
@@ -76,6 +76,49 @@
"Binary objects must use raw strings, not %s" % data.__class__
StringIO.write(self, data)
+ def to_file(self, filename):
+ """write a binary to disk
+
+ the writing is performed in a safe way for files stored on
+ Windows SMB shares
+ """
+ pos = self.tell()
+ with open(filename, 'wb') as fobj:
+ self.seek(0)
+ if sys.platform == 'win32':
+ while True:
+ # the 16kB chunksize comes from the shutil module
+ # in stdlib
+ chunk = self.read(16*1024)
+ if not chunk:
+ break
+ fobj.write(chunk)
+ else:
+ fobj.write(self.read())
+ self.seek(pos)
+
+ @staticmethod
+ def from_file(filename):
+ """read a file and returns its contents in a Binary
+
+ the reading is performed in a safe way for files stored on
+ Windows SMB shares
+ """
+ binary = Binary()
+ with open(filename, 'rb') as fobj:
+ if sys.platform == 'win32':
+ while True:
+ # the 16kB chunksize comes from the shutil module
+ # in stdlib
+ chunk = fobj.read(16*1024)
+ if not chunk:
+ break
+ binary.write(chunk)
+ else:
+ binary.write(fobj.read())
+ return binary
+
+
# use this dictionary to rename entity types while keeping bw compat
ETYPE_NAME_MAP = {}
--- a/server/sources/storages.py Mon Dec 12 12:09:49 2011 +0100
+++ b/server/sources/storages.py Mon Dec 19 19:03:27 2011 +0100
@@ -17,6 +17,7 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
"""custom storages for the system source"""
+import os
from os import unlink, path as osp
from contextlib import contextmanager
@@ -121,7 +122,7 @@
"""
fpath = source.binary_to_str(value)
try:
- return Binary(file(fpath, 'rb').read())
+ return Binary.from_file(fpath)
except EnvironmentError, ex:
source.critical("can't open %s: %s", value, ex)
return None
@@ -129,18 +130,18 @@
def entity_added(self, entity, attr):
"""an entity using this storage for attr has been added"""
if entity._cw.transaction_data.get('fs_importing'):
- binary = Binary(file(entity.cw_edited[attr].getvalue(), 'rb').read())
+ binary = Binary.from_file(entity.cw_edited[attr].getvalue())
else:
binary = entity.cw_edited.pop(attr)
fpath = self.new_fs_path(entity, attr)
# bytes storage used to store file's path
entity.cw_edited.edited_attribute(attr, Binary(fpath))
- file(fpath, 'wb').write(binary.getvalue())
+ binary.to_file(fpath)
AddFileOp.get_instance(entity._cw).add_data(fpath)
return binary
def entity_updated(self, entity, attr):
- """an entity using this storage for attr has been updatded"""
+ """an entity using this storage for attr has been updated"""
# get the name of the previous file containing the value
oldpath = self.current_fs_path(entity, attr)
if entity._cw.transaction_data.get('fs_importing'):
@@ -149,7 +150,7 @@
# the file as the actual content of the attribute
fpath = entity.cw_edited[attr].getvalue()
assert fpath is not None
- binary = Binary(file(fpath, 'rb').read())
+ binary = Binary.from_file(fpath)
else:
# We must store the content of the attributes
# into a file to stay consistent with the behaviour of entity_add.
@@ -168,7 +169,7 @@
fpath = self.new_fs_path(entity, attr)
assert not osp.exists(fpath)
# write attribute value on disk
- file(fpath, 'wb').write(binary.getvalue())
+ binary.to_file(fpath)
# Mark the new file as added during the transaction.
# The file will be removed on rollback
AddFileOp.get_instance(entity._cw).add_data(fpath)
@@ -208,9 +209,9 @@
return fspath
def current_fs_path(self, entity, attr):
- """return the current fs_path of the tribute.
-
- Return None is the attr is not stored yet."""
+ """return the current fs_path of the attribute, or None is the attr is
+ not stored yet.
+ """
sysource = entity._cw.cnxset.source('system')
cu = sysource.doexec(entity._cw,
'SELECT cw_%s FROM cw_%s WHERE cw_eid=%s' % (