# HG changeset patch # User Alexandre Fayolle # Date 1324317807 -3600 # Node ID a6654712ad502568efb51ee5253f0a2c24b26a7d # Parent 96d343a5e01bd47fa5b2a2431542c4b68a76ce04 fix potential problems when BFSS uses a Windows SMB share (closes #2131435) files on windows network share must be read / written by chunks of reasonable size or you get some unusual os level errors. diff -r 96d343a5e01b -r a6654712ad50 __init__.py --- a/__init__.py Mon Dec 12 12:09:49 2011 +0100 +++ b/__init__.py Mon Dec 19 19:03:27 2011 +0100 @@ -76,6 +76,49 @@ "Binary objects must use raw strings, not %s" % data.__class__ StringIO.write(self, data) + def to_file(self, filename): + """write a binary to disk + + the writing is performed in a safe way for files stored on + Windows SMB shares + """ + pos = self.tell() + with open(filename, 'wb') as fobj: + self.seek(0) + if sys.platform == 'win32': + while True: + # the 16kB chunksize comes from the shutil module + # in stdlib + chunk = self.read(16*1024) + if not chunk: + break + fobj.write(chunk) + else: + fobj.write(self.read()) + self.seek(pos) + + @staticmethod + def from_file(filename): + """read a file and returns its contents in a Binary + + the reading is performed in a safe way for files stored on + Windows SMB shares + """ + binary = Binary() + with open(filename, 'rb') as fobj: + if sys.platform == 'win32': + while True: + # the 16kB chunksize comes from the shutil module + # in stdlib + chunk = fobj.read(16*1024) + if not chunk: + break + binary.write(chunk) + else: + binary.write(fobj.read()) + return binary + + # use this dictionary to rename entity types while keeping bw compat ETYPE_NAME_MAP = {} diff -r 96d343a5e01b -r a6654712ad50 server/sources/storages.py --- a/server/sources/storages.py Mon Dec 12 12:09:49 2011 +0100 +++ b/server/sources/storages.py Mon Dec 19 19:03:27 2011 +0100 @@ -17,6 +17,7 @@ # with CubicWeb. If not, see . """custom storages for the system source""" +import os from os import unlink, path as osp from contextlib import contextmanager @@ -121,7 +122,7 @@ """ fpath = source.binary_to_str(value) try: - return Binary(file(fpath, 'rb').read()) + return Binary.from_file(fpath) except EnvironmentError, ex: source.critical("can't open %s: %s", value, ex) return None @@ -129,18 +130,18 @@ def entity_added(self, entity, attr): """an entity using this storage for attr has been added""" if entity._cw.transaction_data.get('fs_importing'): - binary = Binary(file(entity.cw_edited[attr].getvalue(), 'rb').read()) + binary = Binary.from_file(entity.cw_edited[attr].getvalue()) else: binary = entity.cw_edited.pop(attr) fpath = self.new_fs_path(entity, attr) # bytes storage used to store file's path entity.cw_edited.edited_attribute(attr, Binary(fpath)) - file(fpath, 'wb').write(binary.getvalue()) + binary.to_file(fpath) AddFileOp.get_instance(entity._cw).add_data(fpath) return binary def entity_updated(self, entity, attr): - """an entity using this storage for attr has been updatded""" + """an entity using this storage for attr has been updated""" # get the name of the previous file containing the value oldpath = self.current_fs_path(entity, attr) if entity._cw.transaction_data.get('fs_importing'): @@ -149,7 +150,7 @@ # the file as the actual content of the attribute fpath = entity.cw_edited[attr].getvalue() assert fpath is not None - binary = Binary(file(fpath, 'rb').read()) + binary = Binary.from_file(fpath) else: # We must store the content of the attributes # into a file to stay consistent with the behaviour of entity_add. @@ -168,7 +169,7 @@ fpath = self.new_fs_path(entity, attr) assert not osp.exists(fpath) # write attribute value on disk - file(fpath, 'wb').write(binary.getvalue()) + binary.to_file(fpath) # Mark the new file as added during the transaction. # The file will be removed on rollback AddFileOp.get_instance(entity._cw).add_data(fpath) @@ -208,9 +209,9 @@ return fspath def current_fs_path(self, entity, attr): - """return the current fs_path of the tribute. - - Return None is the attr is not stored yet.""" + """return the current fs_path of the attribute, or None is the attr is + not stored yet. + """ sysource = entity._cw.cnxset.source('system') cu = sysource.doexec(entity._cw, 'SELECT cw_%s FROM cw_%s WHERE cw_eid=%s' % (