server/sources/extlite.py
branchtls-sprint
changeset 1482 93c613913912
parent 1409 f4dee84a618f
child 1977 606923dff11b
equal deleted inserted replaced
1481:8ea54e7be3e2 1482:93c613913912
    20     while not lock.acquire(False):
    20     while not lock.acquire(False):
    21         time.sleep(0.2)
    21         time.sleep(0.2)
    22         timeout -= 0.2
    22         timeout -= 0.2
    23         if timeout <= 0:
    23         if timeout <= 0:
    24             raise RuntimeError("svn source is busy, can't acquire connection lock")
    24             raise RuntimeError("svn source is busy, can't acquire connection lock")
    25         
    25 
    26 class ConnectionWrapper(object):
    26 class ConnectionWrapper(object):
    27     def __init__(self, source=None):
    27     def __init__(self, source=None):
    28         self.source = source
    28         self.source = source
    29         self._cnx = None
    29         self._cnx = None
    30 
    30 
    32     def cnx(self):
    32     def cnx(self):
    33         if self._cnx is None:
    33         if self._cnx is None:
    34             timeout_acquire(self.source._cnxlock, 5)
    34             timeout_acquire(self.source._cnxlock, 5)
    35             self._cnx = self.source._sqlcnx
    35             self._cnx = self.source._sqlcnx
    36         return self._cnx
    36         return self._cnx
    37     
    37 
    38     def commit(self):
    38     def commit(self):
    39         if self._cnx is not None:
    39         if self._cnx is not None:
    40             self._cnx.commit()
    40             self._cnx.commit()
    41         
    41 
    42     def rollback(self):
    42     def rollback(self):
    43         if self._cnx is not None:
    43         if self._cnx is not None:
    44             self._cnx.rollback()
    44             self._cnx.rollback()
    45         
    45 
    46     def cursor(self):
    46     def cursor(self):
    47         return self.cnx.cursor()
    47         return self.cnx.cursor()
    48 
    48 
    49     
    49 
    50 class SQLiteAbstractSource(AbstractSource):
    50 class SQLiteAbstractSource(AbstractSource):
    51     """an abstract class for external sources using a sqlite database helper
    51     """an abstract class for external sources using a sqlite database helper
    52     """
    52     """
    53     sqlgen_class = SQLGenerator
    53     sqlgen_class = SQLGenerator
    54     @classmethod
    54     @classmethod
    57         # system source
    57         # system source
    58         for etype in cls.support_entities:
    58         for etype in cls.support_entities:
    59             native.NONSYSTEM_ETYPES.add(etype)
    59             native.NONSYSTEM_ETYPES.add(etype)
    60         for rtype in cls.support_relations:
    60         for rtype in cls.support_relations:
    61             native.NONSYSTEM_RELATIONS.add(rtype)
    61             native.NONSYSTEM_RELATIONS.add(rtype)
    62         
    62 
    63     options = (
    63     options = (
    64         ('helper-db-path',
    64         ('helper-db-path',
    65          {'type' : 'string',
    65          {'type' : 'string',
    66           'default': None,
    66           'default': None,
    67           'help': 'path to the sqlite database file used to do queries on the \
    67           'help': 'path to the sqlite database file used to do queries on the \
    68 repository.',
    68 repository.',
    69           'inputlevel': 2,
    69           'inputlevel': 2,
    70           }),
    70           }),
    71     )
    71     )
    72             
    72 
    73     def __init__(self, repo, appschema, source_config, *args, **kwargs):
    73     def __init__(self, repo, appschema, source_config, *args, **kwargs):
    74         # the helper db is used to easy querying and will store everything but
    74         # the helper db is used to easy querying and will store everything but
    75         # actual file content 
    75         # actual file content
    76         dbpath = source_config.get('helper-db-path')
    76         dbpath = source_config.get('helper-db-path')
    77         if dbpath is None:
    77         if dbpath is None:
    78             dbpath = join(repo.config.appdatahome,
    78             dbpath = join(repo.config.appdatahome,
    79                           '%(uri)s.sqlite' % source_config)
    79                           '%(uri)s.sqlite' % source_config)
    80         self.dbpath = dbpath
    80         self.dbpath = dbpath
    89         # sql database can only be accessed by one connection at a time, and a
    89         # sql database can only be accessed by one connection at a time, and a
    90         # connection can only be used by the thread which created it so:
    90         # connection can only be used by the thread which created it so:
    91         # * create the connection when needed
    91         # * create the connection when needed
    92         # * use a lock to be sure only one connection is used
    92         # * use a lock to be sure only one connection is used
    93         self._cnxlock = threading.Lock()
    93         self._cnxlock = threading.Lock()
    94         
    94 
    95     @property
    95     @property
    96     def _sqlcnx(self):
    96     def _sqlcnx(self):
    97         # XXX: sqlite connections can only be used in the same thread, so
    97         # XXX: sqlite connections can only be used in the same thread, so
    98         #      create a new one each time necessary. If it appears to be time
    98         #      create a new one each time necessary. If it appears to be time
    99         #      consuming, find another way
    99         #      consuming, find another way
   136             # database file must be owned by the uid of the server process
   136             # database file must be owned by the uid of the server process
   137             self.warning('set %s as owner of the database file',
   137             self.warning('set %s as owner of the database file',
   138                          self.repo.config['uid'])
   138                          self.repo.config['uid'])
   139             chown(self.dbpath, self.repo.config['uid'])
   139             chown(self.dbpath, self.repo.config['uid'])
   140         restrict_perms_to_user(self.dbpath, self.info)
   140         restrict_perms_to_user(self.dbpath, self.info)
   141         
   141 
   142     def set_schema(self, schema):
   142     def set_schema(self, schema):
   143         super(SQLiteAbstractSource, self).set_schema(schema)
   143         super(SQLiteAbstractSource, self).set_schema(schema)
   144         if self._need_sql_create and self._is_schema_complete() and self.dbpath:
   144         if self._need_sql_create and self._is_schema_complete() and self.dbpath:
   145             self._create_database()
   145             self._create_database()
   146         self.rqlsqlgen = self.sqlgen_class(schema, self.sqladapter.dbhelper)
   146         self.rqlsqlgen = self.sqlgen_class(schema, self.sqladapter.dbhelper)
   147                 
   147 
   148     def get_connection(self):
   148     def get_connection(self):
   149         return ConnectionWrapper(self)
   149         return ConnectionWrapper(self)
   150 
   150 
   151     def check_connection(self, cnx):
   151     def check_connection(self, cnx):
   152         """check connection validity, return None if the connection is still valid
   152         """check connection validity, return None if the connection is still valid
   166             try:
   166             try:
   167                 cnx._cnx.close()
   167                 cnx._cnx.close()
   168                 cnx._cnx = None
   168                 cnx._cnx = None
   169             finally:
   169             finally:
   170                 self._cnxlock.release()
   170                 self._cnxlock.release()
   171         
   171 
   172     def syntax_tree_search(self, session, union,
   172     def syntax_tree_search(self, session, union,
   173                            args=None, cachekey=None, varmap=None, debug=0):
   173                            args=None, cachekey=None, varmap=None, debug=0):
   174         """return result from this source for a rql query (actually from a rql 
   174         """return result from this source for a rql query (actually from a rql
   175         syntax tree and a solution dictionary mapping each used variable to a 
   175         syntax tree and a solution dictionary mapping each used variable to a
   176         possible type). If cachekey is given, the query necessary to fetch the
   176         possible type). If cachekey is given, the query necessary to fetch the
   177         results (but not the results themselves) may be cached using this key.
   177         results (but not the results themselves) may be cached using this key.
   178         """
   178         """
   179         if self._need_sql_create:
   179         if self._need_sql_create:
   180             return []
   180             return []
   183             print self.uri, 'SOURCE RQL', union.as_string()
   183             print self.uri, 'SOURCE RQL', union.as_string()
   184             print 'GENERATED SQL', sql
   184             print 'GENERATED SQL', sql
   185         args = self.sqladapter.merge_args(args, query_args)
   185         args = self.sqladapter.merge_args(args, query_args)
   186         cursor = session.pool[self.uri]
   186         cursor = session.pool[self.uri]
   187         cursor.execute(sql, args)
   187         cursor.execute(sql, args)
   188         return self.sqladapter.process_result(cursor) 
   188         return self.sqladapter.process_result(cursor)
   189 
   189 
   190     def local_add_entity(self, session, entity):
   190     def local_add_entity(self, session, entity):
   191         """insert the entity in the local database.
   191         """insert the entity in the local database.
   192 
   192 
   193         This is not provided as add_entity implementation since usually source
   193         This is not provided as add_entity implementation since usually source
   196         """
   196         """
   197         cu = session.pool[self.uri]
   197         cu = session.pool[self.uri]
   198         attrs = self.sqladapter.preprocess_entity(entity)
   198         attrs = self.sqladapter.preprocess_entity(entity)
   199         sql = self.sqladapter.sqlgen.insert(SQL_PREFIX + str(entity.e_schema), attrs)
   199         sql = self.sqladapter.sqlgen.insert(SQL_PREFIX + str(entity.e_schema), attrs)
   200         cu.execute(sql, attrs)
   200         cu.execute(sql, attrs)
   201         
   201 
   202     def add_entity(self, session, entity):
   202     def add_entity(self, session, entity):
   203         """add a new entity to the source"""
   203         """add a new entity to the source"""
   204         raise NotImplementedError()
   204         raise NotImplementedError()
   205 
   205 
   206     def local_update_entity(self, session, entity, attrs=None):
   206     def local_update_entity(self, session, entity, attrs=None):
   211         and the source implementor may use this method if necessary
   211         and the source implementor may use this method if necessary
   212         """
   212         """
   213         cu = session.pool[self.uri]
   213         cu = session.pool[self.uri]
   214         if attrs is None:
   214         if attrs is None:
   215             attrs = self.sqladapter.preprocess_entity(entity)
   215             attrs = self.sqladapter.preprocess_entity(entity)
   216         sql = self.sqladapter.sqlgen.update(SQL_PREFIX + str(entity.e_schema), attrs,
   216         sql = self.sqladapter.sqlgen.update(SQL_PREFIX + str(entity.e_schema),
   217                                             [SQL_PREFIX + 'eid'])
   217                                             attrs, [SQL_PREFIX + 'eid'])
   218         cu.execute(sql, attrs)
   218         cu.execute(sql, attrs)
   219         
   219 
   220     def update_entity(self, session, entity):
   220     def update_entity(self, session, entity):
   221         """update an entity in the source"""
   221         """update an entity in the source"""
   222         raise NotImplementedError()
   222         raise NotImplementedError()
   223         
   223 
   224     def delete_entity(self, session, etype, eid):
   224     def delete_entity(self, session, etype, eid):
   225         """delete an entity from the source
   225         """delete an entity from the source
   226 
   226 
   227         this is not deleting a file in the svn but deleting entities from the
   227         this is not deleting a file in the svn but deleting entities from the
   228         source. Main usage is to delete repository content when a Repository
   228         source. Main usage is to delete repository content when a Repository
   229         entity is deleted.
   229         entity is deleted.
   230         """
   230         """
   231         sqlcursor = session.pool[self.uri]        
   231         sqlcursor = session.pool[self.uri]
   232         attrs = {SQL_PREFIX + 'eid': eid}
   232         attrs = {SQL_PREFIX + 'eid': eid}
   233         sql = self.sqladapter.sqlgen.delete(SQL_PREFIX + etype, attrs)
   233         sql = self.sqladapter.sqlgen.delete(SQL_PREFIX + etype, attrs)
   234         sqlcursor.execute(sql, attrs)
   234         sqlcursor.execute(sql, attrs)
   235     
   235 
   236     def delete_relation(self, session, subject, rtype, object):
   236     def delete_relation(self, session, subject, rtype, object):
   237         """delete a relation from the source"""
   237         """delete a relation from the source"""
   238         rschema = self.schema.rschema(rtype)
   238         rschema = self.schema.rschema(rtype)
   239         if rschema.inlined:
   239         if rschema.inlined:
   240             if subject in session.query_data('pendingeids', ()):
   240             if subject in session.query_data('pendingeids', ()):
   244             sql = 'UPDATE %s SET %s=NULL WHERE %seid=%%(eid)s' % (table, column, SQL_PREFIX)
   244             sql = 'UPDATE %s SET %s=NULL WHERE %seid=%%(eid)s' % (table, column, SQL_PREFIX)
   245             attrs = {'eid' : subject}
   245             attrs = {'eid' : subject}
   246         else:
   246         else:
   247             attrs = {'eid_from': subject, 'eid_to': object}
   247             attrs = {'eid_from': subject, 'eid_to': object}
   248             sql = self.sqladapter.sqlgen.delete('%s_relation' % rtype, attrs)
   248             sql = self.sqladapter.sqlgen.delete('%s_relation' % rtype, attrs)
   249         sqlcursor = session.pool[self.uri]        
   249         sqlcursor = session.pool[self.uri]
   250         sqlcursor.execute(sql, attrs)
   250         sqlcursor.execute(sql, attrs)