server/sources/storages.py
author Sylvain Thénault <sylvain.thenault@logilab.fr>
Thu, 18 Feb 2010 10:54:50 +0100
branchstable
changeset 4624 1b46d5ece0d5
parent 4512 e7ac20bf3629
child 4721 8f63691ccb7f
permissions -rw-r--r--
turn default logging threshold to warning (we usually want them), and log 'no schema for eid' pb using warning instead of error, so we see them in logs but not during migration
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
4322
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     1
"""custom storages for the system source"""
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     2
from os import unlink, path as osp
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     3
4349
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
     4
from cubicweb import Binary
4322
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     5
from cubicweb.server.hook import Operation
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     6
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     7
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     8
ETYPE_ATTR_STORAGE = {}
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     9
def set_attribute_storage(repo, etype, attr, storage):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    10
    ETYPE_ATTR_STORAGE.setdefault(etype, {})[attr] = storage
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    11
    repo.system_source.map_attribute(etype, attr, storage.sqlgen_callback)
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    12
4512
e7ac20bf3629 unset_attribute_storage, for testing purpose at least
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4349
diff changeset
    13
def unset_attribute_storage(repo, etype, attr):
e7ac20bf3629 unset_attribute_storage, for testing purpose at least
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4349
diff changeset
    14
    ETYPE_ATTR_STORAGE.setdefault(etype, {}).pop(attr, None)
e7ac20bf3629 unset_attribute_storage, for testing purpose at least
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4349
diff changeset
    15
    repo.system_source.unmap_attribute(etype, attr)
e7ac20bf3629 unset_attribute_storage, for testing purpose at least
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4349
diff changeset
    16
4322
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    17
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    18
class Storage(object):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    19
    """abstract storage"""
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    20
    def sqlgen_callback(self, generator, relation, linkedvar):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    21
        """sql generator callback when some attribute with a custom storage is
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    22
        accessed
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    23
        """
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    24
        raise NotImplementedError()
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    25
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    26
    def entity_added(self, entity, attr):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    27
        """an entity using this storage for attr has been added"""
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    28
        raise NotImplementedError()
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    29
    def entity_updated(self, entity, attr):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    30
        """an entity using this storage for attr has been updatded"""
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    31
        raise NotImplementedError()
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    32
    def entity_deleted(self, entity, attr):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    33
        """an entity using this storage for attr has been deleted"""
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    34
        raise NotImplementedError()
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    35
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    36
# TODO
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    37
# * make it configurable without code
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    38
# * better file path attribution
4329
815e08c53548 add a reminder
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4322
diff changeset
    39
# * handle backup/restore
4322
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    40
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    41
class BytesFileSystemStorage(Storage):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    42
    """store Bytes attribute value on the file system"""
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    43
    def __init__(self, defaultdir):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    44
        self.default_directory = defaultdir
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    45
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    46
    def sqlgen_callback(self, generator, linkedvar, relation):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    47
        """sql generator callback when some attribute with a custom storage is
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    48
        accessed
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    49
        """
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    50
        linkedvar.accept(generator)
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    51
        return '_fsopen(%s.cw_%s)' % (
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    52
            linkedvar._q_sql.split('.', 1)[0], # table name
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    53
            relation.r_type) # attribute name
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    54
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    55
    def entity_added(self, entity, attr):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    56
        """an entity using this storage for attr has been added"""
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    57
        if not entity._cw.transaction_data.get('fs_importing'):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    58
            try:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    59
                value = entity.pop(attr)
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    60
            except KeyError:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    61
                pass
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    62
            else:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    63
                fpath = self.new_fs_path(entity, attr)
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    64
                # bytes storage used to store file's path
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    65
                entity[attr]= Binary(fpath)
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    66
                file(fpath, 'w').write(value.getvalue())
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    67
                AddFileOp(entity._cw, filepath=fpath)
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    68
        # else entity[attr] is expected to be an already existant file path
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    69
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    70
    def entity_updated(self, entity, attr):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    71
        """an entity using this storage for attr has been updatded"""
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    72
        try:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    73
            value = entity.pop(attr)
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    74
        except KeyError:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    75
            pass
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    76
        else:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    77
            fpath = self.current_fs_path(entity, attr)
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    78
            UpdateFileOp(entity._cw, filepath=fpath, filedata=value.getvalue())
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    79
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    80
    def entity_deleted(self, entity, attr):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    81
        """an entity using this storage for attr has been deleted"""
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    82
        DeleteFileOp(entity._cw, filepath=self.current_fs_path(entity, attr))
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    83
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    84
    def new_fs_path(self, entity, attr):
4349
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
    85
        fspath = osp.join(self.default_directory, '%s_%s' % (entity.eid, attr))
4322
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    86
        while osp.exists(fspath):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    87
            fspath = '_' + fspath
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    88
        return fspath
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    89
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    90
    def current_fs_path(self, entity, attr):
4349
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
    91
        sysource = entity._cw.pool.source('system')
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
    92
        cu = sysource.doexec(entity._cw,
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
    93
                             'SELECT cw_%s FROM cw_%s WHERE cw_eid=%s' % (
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
    94
                                 attr, entity.__regid__, entity.eid))
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
    95
        dbmod = sysource.dbapi_module
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
    96
        return dbmod.process_value(cu.fetchone()[0], [None, dbmod.BINARY],
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
    97
                                   binarywrap=str)
4322
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    98
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    99
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   100
class AddFileOp(Operation):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   101
    def rollback_event(self):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   102
        try:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   103
            unlink(self.filepath)
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   104
        except:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   105
            pass
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   106
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   107
class DeleteFileOp(Operation):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   108
    def commit_event(self):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   109
        try:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   110
            unlink(self.filepath)
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   111
        except:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   112
            pass
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   113
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   114
class UpdateFileOp(Operation):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   115
    def precommit_event(self):
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   116
        try:
f65743cc53e4 first draft for a simple hooks based custom attribute storage,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   117
            file(self.filepath, 'w').write(self.filedata)
4349
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
   118
        except Exception, ex:
48dadeeacfa5 [bfss] make it works when adding/updating entities with an attribute using bfss
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4329
diff changeset
   119
            self.exception(str(ex))