--- a/server/checkintegrity.py Wed Mar 05 12:35:37 2014 +0100
+++ b/server/checkintegrity.py Mon Mar 17 12:13:14 2014 +0100
@@ -35,7 +35,7 @@
sys.stderr.write(' [FIXED]')
sys.stderr.write('\n')
-def has_eid(session, sqlcursor, eid, eids):
+def has_eid(cnx, sqlcursor, eid, eids):
"""return true if the eid is a valid eid"""
if eid in eids:
return eids[eid]
@@ -45,7 +45,7 @@
except Exception:
eids[eid] = False
return False
- if etype not in session.vreg.schema:
+ if etype not in cnx.vreg.schema:
eids[eid] = False
return False
sqlcursor.execute('SELECT * FROM %s%s WHERE %seid=%s' % (SQL_PREFIX, etype,
@@ -87,11 +87,12 @@
# deactivate modification_date hook since we don't want them
# to be updated due to the reindexation
repo = cnx.repo
- cursor = cnx.cnxset.cu
dbhelper = repo.system_source.dbhelper
- if not dbhelper.has_fti_table(cursor):
- print 'no text index table'
- dbhelper.init_fti(cursor)
+ with cnx.ensure_cnx_set:
+ cursor = cnx.cnxset.cu
+ if not dbhelper.has_fti_table(cursor):
+ print 'no text index table'
+ dbhelper.init_fti(cursor)
repo.system_source.do_fti = True # ensure full-text indexation is activated
if etypes is None:
print 'Reindexing entities'
@@ -131,7 +132,7 @@
pb.update()
-def check_schema(schema, session, eids, fix=1):
+def check_schema(schema, cnx, eids, fix=1):
"""check serialized schema"""
print 'Checking serialized schema'
unique_constraints = ('SizeConstraint', 'FormatConstraint',
@@ -141,7 +142,7 @@
'WHERE X is CWConstraint, R constrained_by X, '
'R relation_type RT, RT name RN, R from_entity ST, ST name SN, '
'R to_entity OT, OT name ON, X cstrtype CT, CT name CTN')
- for count, rn, sn, on, cstrname in session.execute(rql):
+ for count, rn, sn, on, cstrname in cnx.execute(rql):
if count == 1:
continue
if cstrname in unique_constraints:
@@ -152,37 +153,38 @@
-def check_text_index(schema, session, eids, fix=1):
+def check_text_index(schema, cnx, eids, fix=1):
"""check all entities registered in the text index"""
print 'Checking text index'
msg = ' Entity with eid %s exists in the text index but in no source (autofix will remove from text index)'
- cursor = session.system_sql('SELECT uid FROM appears;')
+ cursor = cnx.system_sql('SELECT uid FROM appears;')
for row in cursor.fetchall():
eid = row[0]
- if not has_eid(session, cursor, eid, eids):
+ if not has_eid(cnx, cursor, eid, eids):
sys.stderr.write(msg % eid)
if fix:
- session.system_sql('DELETE FROM appears WHERE uid=%s;' % eid)
+ cnx.system_sql('DELETE FROM appears WHERE uid=%s;' % eid)
notify_fixed(fix)
-def check_entities(schema, session, eids, fix=1):
+def check_entities(schema, cnx, eids, fix=1):
"""check all entities registered in the repo system table"""
print 'Checking entities system table'
# system table but no source
msg = ' Entity %s with eid %s exists in the system table but in no source (autofix will delete the entity)'
- cursor = session.system_sql('SELECT eid,type FROM entities;')
+ cursor = cnx.system_sql('SELECT eid,type FROM entities;')
for row in cursor.fetchall():
eid, etype = row
- if not has_eid(session, cursor, eid, eids):
+ if not has_eid(cnx, cursor, eid, eids):
sys.stderr.write(msg % (etype, eid))
if fix:
- session.system_sql('DELETE FROM entities WHERE eid=%s;' % eid)
+ cnx.system_sql('DELETE FROM entities WHERE eid=%s;' % eid)
notify_fixed(fix)
# source in entities, but no relation cw_source
- applcwversion = session.repo.get_versions().get('cubicweb')
+ # XXX this (get_versions) requires a second connection to the db when we already have one open
+ applcwversion = cnx.repo.get_versions().get('cubicweb')
if applcwversion >= (3, 13, 1): # entities.asource appeared in 3.13.1
- cursor = session.system_sql('SELECT e.eid FROM entities as e, cw_CWSource as s '
+ cursor = cnx.system_sql('SELECT e.eid FROM entities as e, cw_CWSource as s '
'WHERE s.cw_name=e.asource AND '
'NOT EXISTS(SELECT 1 FROM cw_source_relation as cs '
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid) '
@@ -192,35 +194,35 @@
for row in cursor.fetchall():
sys.stderr.write(msg % row[0])
if fix:
- session.system_sql('INSERT INTO cw_source_relation (eid_from, eid_to) '
+ cnx.system_sql('INSERT INTO cw_source_relation (eid_from, eid_to) '
'SELECT e.eid, s.cw_eid FROM entities as e, cw_CWSource as s '
'WHERE s.cw_name=e.asource AND NOT EXISTS(SELECT 1 FROM cw_source_relation as cs '
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid)')
notify_fixed(True)
# inconsistencies for 'is'
msg = ' %s #%s is missing relation "is" (autofix will create the relation)\n'
- cursor = session.system_sql('SELECT e.type, e.eid FROM entities as e, cw_CWEType as s '
+ cursor = cnx.system_sql('SELECT e.type, e.eid FROM entities as e, cw_CWEType as s '
'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_relation as cs '
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid) '
'ORDER BY e.eid')
for row in cursor.fetchall():
sys.stderr.write(msg % row)
if fix:
- session.system_sql('INSERT INTO is_relation (eid_from, eid_to) '
+ cnx.system_sql('INSERT INTO is_relation (eid_from, eid_to) '
'SELECT e.eid, s.cw_eid FROM entities as e, cw_CWEType as s '
'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_relation as cs '
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid)')
notify_fixed(True)
# inconsistencies for 'is_instance_of'
msg = ' %s #%s is missing relation "is_instance_of" (autofix will create the relation)\n'
- cursor = session.system_sql('SELECT e.type, e.eid FROM entities as e, cw_CWEType as s '
+ cursor = cnx.system_sql('SELECT e.type, e.eid FROM entities as e, cw_CWEType as s '
'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_instance_of_relation as cs '
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid) '
'ORDER BY e.eid')
for row in cursor.fetchall():
sys.stderr.write(msg % row)
if fix:
- session.system_sql('INSERT INTO is_instance_of_relation (eid_from, eid_to) '
+ cnx.system_sql('INSERT INTO is_instance_of_relation (eid_from, eid_to) '
'SELECT e.eid, s.cw_eid FROM entities as e, cw_CWEType as s '
'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_instance_of_relation as cs '
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid)')
@@ -232,7 +234,7 @@
continue
table = SQL_PREFIX + eschema.type
column = SQL_PREFIX + 'eid'
- cursor = session.system_sql('SELECT %s FROM %s;' % (column, table))
+ cursor = cnx.system_sql('SELECT %s FROM %s;' % (column, table))
for row in cursor.fetchall():
eid = row[0]
# eids is full since we have fetched everything from the entities table,
@@ -240,7 +242,7 @@
if not eid in eids or not eids[eid]:
sys.stderr.write(msg % (eid, eschema.type))
if fix:
- session.system_sql('DELETE FROM %s WHERE %s=%s;' % (table, column, eid))
+ cnx.system_sql('DELETE FROM %s WHERE %s=%s;' % (table, column, eid))
notify_fixed(fix)
@@ -256,7 +258,7 @@
notify_fixed(fix)
-def check_relations(schema, session, eids, fix=1):
+def check_relations(schema, cnx, eids, fix=1):
"""check that eids referenced by relations are registered in the repo system
table
"""
@@ -270,42 +272,42 @@
column = SQL_PREFIX + str(rschema)
sql = 'SELECT cw_eid,%s FROM %s WHERE %s IS NOT NULL;' % (
column, table, column)
- cursor = session.system_sql(sql)
+ cursor = cnx.system_sql(sql)
for row in cursor.fetchall():
parent_eid, eid = row
- if not has_eid(session, cursor, eid, eids):
+ if not has_eid(cnx, cursor, eid, eids):
bad_inlined_msg(rschema, parent_eid, eid, fix)
if fix:
sql = 'UPDATE %s SET %s=NULL WHERE %s=%s;' % (
table, column, column, eid)
- session.system_sql(sql)
+ cnx.system_sql(sql)
continue
try:
- cursor = session.system_sql('SELECT eid_from FROM %s_relation;' % rschema)
+ cursor = cnx.system_sql('SELECT eid_from FROM %s_relation;' % rschema)
except Exception as ex:
# usually because table doesn't exist
print 'ERROR', ex
continue
for row in cursor.fetchall():
eid = row[0]
- if not has_eid(session, cursor, eid, eids):
+ if not has_eid(cnx, cursor, eid, eids):
bad_related_msg(rschema, 'subject', eid, fix)
if fix:
sql = 'DELETE FROM %s_relation WHERE eid_from=%s;' % (
rschema, eid)
- session.system_sql(sql)
- cursor = session.system_sql('SELECT eid_to FROM %s_relation;' % rschema)
+ cnx.system_sql(sql)
+ cursor = cnx.system_sql('SELECT eid_to FROM %s_relation;' % rschema)
for row in cursor.fetchall():
eid = row[0]
- if not has_eid(session, cursor, eid, eids):
+ if not has_eid(cnx, cursor, eid, eids):
bad_related_msg(rschema, 'object', eid, fix)
if fix:
sql = 'DELETE FROM %s_relation WHERE eid_to=%s;' % (
rschema, eid)
- session.system_sql(sql)
+ cnx.system_sql(sql)
-def check_mandatory_relations(schema, session, eids, fix=1):
+def check_mandatory_relations(schema, cnx, eids, fix=1):
"""check entities missing some mandatory relation"""
print 'Checking mandatory relations'
msg = '%s #%s is missing mandatory %s relation %s (autofix will delete the entity)'
@@ -325,7 +327,7 @@
rql = 'Any X WHERE NOT X %s Y, X is %s' % (rschema, etype)
else:
rql = 'Any X WHERE NOT Y %s X, X is %s' % (rschema, etype)
- for entity in session.execute(rql).entities():
+ for entity in cnx.execute(rql).entities():
sys.stderr.write(msg % (entity.cw_etype, entity.eid, role, rschema))
if fix:
#if entity.cw_describe()['source']['uri'] == 'system': XXX
@@ -333,7 +335,7 @@
notify_fixed(fix)
-def check_mandatory_attributes(schema, session, eids, fix=1):
+def check_mandatory_attributes(schema, cnx, eids, fix=1):
"""check for entities stored in the system source missing some mandatory
attribute
"""
@@ -346,40 +348,40 @@
if rdef.cardinality[0] in '1+':
rql = 'Any X WHERE X %s NULL, X is %s, X cw_source S, S name "system"' % (
rschema, rdef.subject)
- for entity in session.execute(rql).entities():
+ for entity in cnx.execute(rql).entities():
sys.stderr.write(msg % (entity.cw_etype, entity.eid, rschema))
if fix:
entity.cw_delete()
notify_fixed(fix)
-def check_metadata(schema, session, eids, fix=1):
+def check_metadata(schema, cnx, eids, fix=1):
"""check entities has required metadata
FIXME: rewrite using RQL queries ?
"""
print 'Checking metadata'
- cursor = session.system_sql("SELECT DISTINCT type FROM entities;")
+ cursor = cnx.system_sql("SELECT DISTINCT type FROM entities;")
eidcolumn = SQL_PREFIX + 'eid'
msg = ' %s with eid %s has no %s (autofix will set it to now)'
for etype, in cursor.fetchall():
- if etype not in session.vreg.schema:
+ if etype not in cnx.vreg.schema:
sys.stderr.write('entities table references unknown type %s\n' %
etype)
if fix:
- session.system_sql("DELETE FROM entities WHERE type = %(type)s",
+ cnx.system_sql("DELETE FROM entities WHERE type = %(type)s",
{'type': etype})
continue
table = SQL_PREFIX + etype
for rel, default in ( ('creation_date', datetime.now()),
('modification_date', datetime.now()), ):
column = SQL_PREFIX + rel
- cursor = session.system_sql("SELECT %s FROM %s WHERE %s is NULL"
+ cursor = cnx.system_sql("SELECT %s FROM %s WHERE %s is NULL"
% (eidcolumn, table, column))
for eid, in cursor.fetchall():
sys.stderr.write(msg % (etype, eid, rel))
if fix:
- session.system_sql("UPDATE %s SET %s=%%(v)s WHERE %s=%s ;"
+ cnx.system_sql("UPDATE %s SET %s=%%(v)s WHERE %s=%s ;"
% (table, column, eidcolumn, eid),
{'v': default})
notify_fixed(fix)
@@ -390,22 +392,23 @@
using given user and password to locally connect to the repository
(no running cubicweb server needed)
"""
- session = repo._get_session(cnx.sessionid, setcnxset=True)
# yo, launch checks
+ srvcnx = cnx._cnx
if checks:
eids_cache = {}
- with session.security_enabled(read=False, write=False): # ensure no read security
+ with srvcnx.security_enabled(read=False, write=False): # ensure no read security
for check in checks:
check_func = globals()['check_%s' % check]
- check_func(repo.schema, session, eids_cache, fix=fix)
+ with srvcnx.ensure_cnx_set:
+ check_func(repo.schema, srvcnx, eids_cache, fix=fix)
if fix:
- session.commit()
+ srvcnx.commit()
else:
print
if not fix:
print 'WARNING: Diagnostic run, nothing has been corrected'
if reindex:
- session.rollback()
- session.set_cnxset()
- reindex_entities(repo.schema, session, withpb=withpb)
- session.commit()
+ srvcnx.rollback()
+ with srvcnx.ensure_cnx_set:
+ reindex_entities(repo.schema, srvcnx, withpb=withpb)
+ srvcnx.commit()
--- a/server/serverctl.py Wed Mar 05 12:35:37 2014 +0100
+++ b/server/serverctl.py Mon Mar 17 12:13:14 2014 +0100
@@ -132,8 +132,8 @@
return cnx
def repo_cnx(config):
- """return a in-memory repository and a db api connection it"""
- from cubicweb.dbapi import in_memory_repo_cnx
+ """return a in-memory repository and a repoapi connection to it"""
+ from cubicweb import repoapi
from cubicweb.server.utils import manager_userpasswd
try:
login = config.default_admin_config['login']
@@ -142,7 +142,9 @@
login, pwd = manager_userpasswd()
while True:
try:
- return in_memory_repo_cnx(config, login, password=pwd)
+ repo = repoapi.get_repository(config=config)
+ cnx = repoapi.connect(repo, login, password=pwd)
+ return repo, cnx
except AuthenticationError:
print '-> Error: wrong user/password.'
# reset cubes else we'll have an assertion error on next retry
@@ -482,43 +484,43 @@
config = ServerConfiguration.config_for(appid)
config.quick_start = True
repo, cnx = repo_cnx(config)
- req = cnx.request()
- used = set(n for n, in req.execute('Any SN WHERE S is CWSource, S name SN'))
- cubes = repo.get_cubes()
- while True:
- type = raw_input('source type (%s): '
- % ', '.join(sorted(SOURCE_TYPES)))
- if type not in SOURCE_TYPES:
- print '-> unknown source type, use one of the available types.'
- continue
- sourcemodule = SOURCE_TYPES[type].module
- if not sourcemodule.startswith('cubicweb.'):
- # module names look like cubes.mycube.themodule
- sourcecube = SOURCE_TYPES[type].module.split('.', 2)[1]
- # if the source adapter is coming from an external component,
- # ensure it's specified in used cubes
- if not sourcecube in cubes:
- print ('-> this source type require the %s cube which is '
- 'not used by the instance.')
+ with cnx:
+ used = set(n for n, in cnx.execute('Any SN WHERE S is CWSource, S name SN'))
+ cubes = repo.get_cubes()
+ while True:
+ type = raw_input('source type (%s): '
+ % ', '.join(sorted(SOURCE_TYPES)))
+ if type not in SOURCE_TYPES:
+ print '-> unknown source type, use one of the available types.'
continue
- break
- while True:
- sourceuri = raw_input('source identifier (a unique name used to '
- 'tell sources apart): ').strip()
- if not sourceuri:
- print '-> mandatory.'
- else:
- sourceuri = unicode(sourceuri, sys.stdin.encoding)
- if sourceuri in used:
- print '-> uri already used, choose another one.'
+ sourcemodule = SOURCE_TYPES[type].module
+ if not sourcemodule.startswith('cubicweb.'):
+ # module names look like cubes.mycube.themodule
+ sourcecube = SOURCE_TYPES[type].module.split('.', 2)[1]
+ # if the source adapter is coming from an external component,
+ # ensure it's specified in used cubes
+ if not sourcecube in cubes:
+ print ('-> this source type require the %s cube which is '
+ 'not used by the instance.')
+ continue
+ break
+ while True:
+ sourceuri = raw_input('source identifier (a unique name used to '
+ 'tell sources apart): ').strip()
+ if not sourceuri:
+ print '-> mandatory.'
else:
- break
- # XXX configurable inputlevel
- sconfig = ask_source_config(config, type, inputlevel=self.config.config_level)
- cfgstr = unicode(generate_source_config(sconfig), sys.stdin.encoding)
- req.create_entity('CWSource', name=sourceuri,
- type=unicode(type), config=cfgstr)
- cnx.commit()
+ sourceuri = unicode(sourceuri, sys.stdin.encoding)
+ if sourceuri in used:
+ print '-> uri already used, choose another one.'
+ else:
+ break
+ # XXX configurable inputlevel
+ sconfig = ask_source_config(config, type, inputlevel=self.config.config_level)
+ cfgstr = unicode(generate_source_config(sconfig), sys.stdin.encoding)
+ cnx.create_entity('CWSource', name=sourceuri,
+ type=unicode(type), config=cfgstr)
+ cnx.commit()
class GrantUserOnInstanceCommand(Command):
@@ -978,8 +980,9 @@
config = ServerConfiguration.config_for(appid)
config.repairing = self.config.force
repo, cnx = repo_cnx(config)
- check(repo, cnx,
- self.config.checks, self.config.reindex, self.config.autofix)
+ with cnx:
+ check(repo, cnx,
+ self.config.checks, self.config.reindex, self.config.autofix)
class RebuildFTICommand(Command):
@@ -1001,9 +1004,9 @@
etypes = args or None
config = ServerConfiguration.config_for(appid)
repo, cnx = repo_cnx(config)
- session = repo._get_session(cnx.sessionid, setcnxset=True)
- reindex_entities(repo.schema, session, etypes=etypes)
- cnx.commit()
+ with cnx:
+ reindex_entities(repo.schema, cnx._cnx, etypes=etypes)
+ cnx.commit()
class SynchronizeSourceCommand(Command):
@@ -1074,7 +1077,7 @@
diff_tool = args.pop(0)
config = ServerConfiguration.config_for(appid)
repo, cnx = repo_cnx(config)
- session = repo._get_session(cnx.sessionid, setcnxset=True)
+ cnx.close()
fsschema = config.load_schema(expand_cubes=True)
schema_diff(fsschema, repo.schema, permissionshandler, diff_tool, ignore=('eid',))