--- a/.hgtags Tue Aug 21 13:25:29 2018 +0200
+++ b/.hgtags Tue Aug 28 11:24:50 2018 +0200
@@ -68,3 +68,4 @@
116cdd8c102ab0ae6295fb4886b0882e75e4d8f7 8.0.0
0887c30255a1a1808d74a63b16e896d457f8ef32 8.0.1
2c5d79c6459c6fabe0eb8723fc5041ac0dac7a9a 8.1.0
+e7abf863e1130e14cd4d65e53467a199d267b4fd 8.1.1
--- a/CHANGELOG Tue Aug 21 13:25:29 2018 +0200
+++ b/CHANGELOG Tue Aug 28 11:24:50 2018 +0200
@@ -1,7 +1,12 @@
Changelog
=========
-8.1.1 - in progress
+8.1.2 - in progress
+-------------------
+
+ * obshashrange: improved robusness of the cache under heavy load
+
+8.1.1 -- 2018-08-21
-------------------
* clone: fix possible crash when using clone bundle and forcing cache warming
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/hammerclient.py Tue Aug 28 11:24:50 2018 +0200
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+import os
+import sys
+import subprocess
+
+if len(sys.argv) < 2:
+ execname = os.path.basename(sys.argv[0])
+ print >> sys.stderr, "usage: %s CLIENT_ID" % execname
+
+client_id = sys.argv[1]
+
+subprocess.check_call(['hg', 'branch', "--force", "hammer-branch-%s" % client_id])
+
+while True:
+ subprocess.check_call([
+ 'hg', 'commit',
+ "--config", "ui.allowemptycommit=yes",
+ "--message", "hammer-%s" % client_id,
+ ])
+ nodeid = subprocess.check_output([
+ 'hg', 'log', '--rev', '.', '--template', '{node}'
+ ])
+ subprocess.check_call([
+ 'hg', 'debugobsolete', ''.join(reversed(nodeid)), nodeid
+ ])
+ subprocess.check_call(['hg', 'pull'])
+ subprocess.check_call(['hg', 'push', '--force'])
--- a/debian/changelog Tue Aug 21 13:25:29 2018 +0200
+++ b/debian/changelog Tue Aug 28 11:24:50 2018 +0200
@@ -1,3 +1,9 @@
+mercurial-evolve (8.1.1) unstable; urgency=medium
+
+ * new upstream release
+
+ -- Pierre-Yves David <pierre-yves.david@ens-lyon.org> Tue, 21 Aug 2018 15:28:29 +0200
+
mercurial-evolve (8.0.1-1) unstable; urgency=medium
* New upstream release
--- a/hgext3rd/evolve/metadata.py Tue Aug 21 13:25:29 2018 +0200
+++ b/hgext3rd/evolve/metadata.py Tue Aug 28 11:24:50 2018 +0200
@@ -5,7 +5,7 @@
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
-__version__ = '8.1.1.dev'
+__version__ = '8.1.2.dev'
testedwith = '4.3.2 4.4.2 4.5.2 4.6.2 4.7'
minimumhgversion = '4.3'
buglink = 'https://bz.mercurial-scm.org/'
--- a/hgext3rd/evolve/obsdiscovery.py Tue Aug 21 13:25:29 2018 +0200
+++ b/hgext3rd/evolve/obsdiscovery.py Tue Aug 28 11:24:50 2018 +0200
@@ -328,6 +328,11 @@
### sqlite caching
_sqliteschema = [
+ """CREATE TABLE obshashrange(rev INTEGER NOT NULL,
+ idx INTEGER NOT NULL,
+ obshash BLOB NOT NULL,
+ PRIMARY KEY(rev, idx));""",
+ "CREATE INDEX range_index ON obshashrange(rev, idx);",
"""CREATE TABLE meta(schemaversion INTEGER NOT NULL,
tiprev INTEGER NOT NULL,
tipnode BLOB NOT NULL,
@@ -335,11 +340,6 @@
obssize BLOB NOT NULL,
obskey BLOB NOT NULL
);""",
- """CREATE TABLE obshashrange(rev INTEGER NOT NULL,
- idx INTEGER NOT NULL,
- obshash BLOB NOT NULL,
- PRIMARY KEY(rev, idx));""",
- "CREATE INDEX range_index ON obshashrange(rev, idx);",
]
_queryexist = "SELECT name FROM sqlite_master WHERE type='table' AND name='meta';"
_clearmeta = """DELETE FROM meta;"""
@@ -432,10 +432,17 @@
value = self._data.get(rangeid)
if value is None and self._con is not None:
nrange = (rangeid[0], rangeid[1])
- obshash = self._con.execute(_queryobshash, nrange).fetchone()
- if obshash is not None:
- value = obshash[0]
- self._data[rangeid] = value
+ try:
+ obshash = self._con.execute(_queryobshash, nrange).fetchone()
+ if obshash is not None:
+ value = obshash[0]
+ self._data[rangeid] = value
+ except (sqlite3.DatabaseError, sqlite3.OperationalError):
+ # something is wrong with the sqlite db
+ # Since this is a cache, we ignore it.
+ if '_con' in vars(self):
+ del self._con
+ self._new.clear()
return value
def __setitem__(self, rangeid, obshash):
@@ -473,14 +480,19 @@
if con is not None:
# always reset for now, the code detecting affect is buggy
# so we need to reset more broadly than we would like.
- if repo.stablerange._con is None:
- con.execute(_reset)
- self._data.clear()
- else:
- ranges = repo.stablerange.contains(repo, affected)
- con.executemany(_delete, ranges)
- for r in ranges:
- self._data.pop(r, None)
+ try:
+ if repo.stablerange._con is None:
+ con.execute(_reset)
+ self._data.clear()
+ else:
+ ranges = repo.stablerange.contains(repo, affected)
+ con.executemany(_delete, ranges)
+ for r in ranges:
+ self._data.pop(r, None)
+ except (sqlite3.DatabaseError, sqlite3.OperationalError) as exc:
+ repo.ui.log('evoext-cache', 'error while updating obshashrange cache: %s' % exc)
+ del self._updating
+ return
# rewarm key revisions
#
@@ -524,7 +536,7 @@
util.makedirs(self._vfs.dirname(self._path))
except OSError:
return None
- con = sqlite3.connect(self._path)
+ con = sqlite3.connect(self._path, timeout=30, isolation_level="IMMEDIATE")
con.text_factory = str
return con
@@ -568,6 +580,21 @@
repo.ui.warn(msg)
def _save(self, repo):
+ if not self._new:
+ return
+ try:
+ return self._trysave(repo)
+ except (sqlite3.DatabaseError, sqlite3.OperationalError, sqlite3.IntegrityError) as exc:
+ # Catch error that may arise under stress
+ #
+ # operational error catch read-only and locked database
+ # IntegrityError catch Unique constraint error that may arise
+ if '_con' in vars(self):
+ del self._con
+ self._new.clear()
+ repo.ui.log('evoext-cache', 'error while saving new data: %s' % exc)
+
+ def _trysave(self, repo):
if self._con is None:
util.unlinkpath(self._path, ignoremissing=True)
if '_con' in vars(self):
@@ -582,30 +609,34 @@
for req in _sqliteschema:
con.execute(req)
- con.execute(_newmeta, self._fullcachekey)
+ meta = [self._schemaversion] + list(self.emptykey)
+ con.execute(_newmeta, meta)
+ self._ondiskcachekey = self.emptykey
else:
con = self._con
- if self._ondiskcachekey is not None:
- meta = con.execute(_querymeta).fetchone()
- if meta[1:] != self._ondiskcachekey:
- # drifting is currently an issue because this means another
- # process might have already added the cache line we are about
- # to add. This will confuse sqlite
- msg = _('obshashrange cache: skipping write, '
- 'database drifted under my feet\n')
- data = (meta[2], meta[1], self._ondiskcachekey[0], self._ondiskcachekey[1])
- repo.ui.warn(msg)
- return
- data = ((rangeid[0], rangeid[1], self.get(rangeid)) for rangeid in self._new)
- con.executemany(_updateobshash, data)
- cachekey = self._fullcachekey
- con.execute(_clearmeta) # remove the older entry
- con.execute(_newmeta, cachekey)
- con.commit()
- self._new.clear()
- self._ondiskcachekey = self._cachekey
- self._valid = True
-
+ with con:
+ meta = con.execute(_querymeta).fetchone()
+ if meta[1:] != self._ondiskcachekey:
+ # drifting is currently an issue because this means another
+ # process might have already added the cache line we are about
+ # to add. This will confuse sqlite
+ msg = _('obshashrange cache: skipping write, '
+ 'database drifted under my feet\n')
+ repo.ui.warn(msg)
+ self._new.clear()
+ self._valid = False
+ if '_con' in vars(self):
+ del self._con
+ self._valid = False
+ return
+ data = ((rangeid[0], rangeid[1], self.get(rangeid)) for rangeid in self._new)
+ con.executemany(_updateobshash, data)
+ cachekey = self._fullcachekey
+ con.execute(_clearmeta) # remove the older entry
+ con.execute(_newmeta, cachekey)
+ self._new.clear()
+ self._valid = True
+ self._ondiskcachekey = self._cachekey
@eh.wrapfunction(obsolete.obsstore, '_addmarkers')
def _addmarkers(orig, obsstore, *args, **kwargs):
obsstore.rangeobshashcache.clear()
--- a/hgext3rd/evolve/stablerangecache.py Tue Aug 21 13:25:29 2018 +0200
+++ b/hgext3rd/evolve/stablerangecache.py Tue Aug 28 11:24:50 2018 +0200
@@ -17,6 +17,7 @@
from mercurial import (
error,
localrepo,
+ node as nodemod,
util,
)
@@ -87,10 +88,6 @@
#############################
_sqliteschema = [
- """CREATE TABLE meta(schemaversion INTEGER NOT NULL,
- tiprev INTEGER NOT NULL,
- tipnode BLOB NOT NULL
- );""",
"""CREATE TABLE range(rev INTEGER NOT NULL,
idx INTEGER NOT NULL,
PRIMARY KEY(rev, idx));""",
@@ -106,6 +103,10 @@
"CREATE INDEX subranges_index ON subranges (suprev, supidx);",
"CREATE INDEX superranges_index ON subranges (subrev, subidx);",
"CREATE INDEX range_index ON range (rev, idx);",
+ """CREATE TABLE meta(schemaversion INTEGER NOT NULL,
+ tiprev INTEGER NOT NULL,
+ tipnode BLOB NOT NULL
+ );""",
]
_newmeta = "INSERT INTO meta (schemaversion, tiprev, tipnode) VALUES (?,?,?);"
_updatemeta = "UPDATE meta SET tiprev = ?, tipnode = ?;"
@@ -177,11 +178,19 @@
cache = self._subrangescache
if rangeid not in cache and rangeid[0] <= self._ondisktiprev and self._con is not None:
value = None
- result = self._con.execute(_queryrange, rangeid).fetchone()
- if result is not None: # database know about this node (skip in the future?)
- value = self._con.execute(_querysubranges, rangeid).fetchall()
- # in memory caching of the value
- cache[rangeid] = value
+ try:
+ result = self._con.execute(_queryrange, rangeid).fetchone()
+ if result is not None: # database know about this node (skip in the future?)
+ value = self._con.execute(_querysubranges, rangeid).fetchall()
+ # in memory caching of the value
+ cache[rangeid] = value
+ except (sqlite3.DatabaseError, sqlite3.OperationalError):
+ # something is wrong with the sqlite db
+ # Since this is a cache, we ignore it.
+ if '_con' in vars(self):
+ del self._con
+ self._unsavedsubranges.clear()
+
return cache.get(rangeid)
def _setsub(self, rangeid, value):
@@ -194,7 +203,7 @@
util.makedirs(self._vfs.dirname(self._path))
except OSError:
return None
- con = sqlite3.connect(self._path)
+ con = sqlite3.connect(self._path, timeout=30, isolation_level="IMMEDIATE")
con.text_factory = str
return con
@@ -223,6 +232,21 @@
return con
def _save(self, repo):
+ if not self._unsavedsubranges:
+ return
+ try:
+ return self._trysave(repo)
+ except (sqlite3.DatabaseError, sqlite3.OperationalError, sqlite3.IntegrityError) as exc:
+ # Catch error that may arise under stress
+ #
+ # operational error catch read-only and locked database
+ # IntegrityError catch Unique constraint error that may arise
+ if '_con' in vars(self):
+ del self._con
+ self._unsavedsubranges.clear()
+ repo.ui.log('evoext-cache', 'error while saving new data: %s' % exc)
+
+ def _trysave(self, repo):
repo = repo.unfiltered()
repo.depthcache.save(repo)
repo.stablesort.save(repo)
@@ -238,19 +262,20 @@
con = self._db()
if con is None:
return
- con.execute('BEGIN IMMEDIATE;')
with con:
for req in _sqliteschema:
con.execute(req)
meta = [self._schemaversion,
- self._tiprev,
- self._tipnode,
+ nodemod.nullrev,
+ nodemod.nullid,
]
con.execute(_newmeta, meta)
+ self._ondisktiprev = nodemod.nullrev
+ self._ondisktipnode = nodemod.nullid
else:
con = self._con
- con.execute('BEGIN IMMEDIATE;')
+ with con:
meta = con.execute(_querymeta).fetchone()
if meta[2] != self._ondisktipnode or meta[1] != self._ondisktiprev:
# drifting is currently an issue because this means another
@@ -258,19 +283,19 @@
# to add. This will confuse sqlite
msg = _('stable-range cache: skipping write, '
'database drifted under my feet\n')
- hint = _('(disk: %s-%s vs mem: %s%s)\n')
- data = (meta[2], meta[1], self._ondisktiprev, self._ondisktipnode)
+ hint = _('(disk: %s-%s vs mem: %s-%s)\n')
+ data = (nodemod.hex(meta[2]), meta[1],
+ nodemod.hex(self._ondisktipnode), self._ondisktiprev)
repo.ui.warn(msg)
repo.ui.warn(hint % data)
- con.execute('ROLLBACK;')
+ self._unsavedsubranges.clear()
return
- meta = [self._tiprev,
- self._tipnode,
- ]
- con.execute(_updatemeta, meta)
-
- self._saverange(con, repo)
- con.commit()
+ else:
+ self._saverange(con, repo)
+ meta = [self._tiprev,
+ self._tipnode,
+ ]
+ con.execute(_updatemeta, meta)
self._ondisktiprev = self._tiprev
self._ondisktipnode = self._tipnode
self._unsavedsubranges.clear()
--- a/tests/test-discovery-obshashrange.t Tue Aug 21 13:25:29 2018 +0200
+++ b/tests/test-discovery-obshashrange.t Tue Aug 28 11:24:50 2018 +0200
@@ -443,8 +443,8 @@
* @0000000000000000000000000000000000000000 (*)> updated evo-ext-obscache in *.???? seconds (0r, 1o) (glob)
* @0000000000000000000000000000000000000000 (*)> -R ../server debugobsolete bbbbbbb2222222222bbbbbbbbbbbbb2222222222 bebd167eb94d257ace0e814aeb98e6972ed2970d exited 0 after *.?? seconds (glob)
* @0000000000000000000000000000000000000000 (*)> -R server serve --stdio (glob)
- 1970/01/01 00:00:00 * @0000000000000000000000000000000000000000 (*)> obshashcache clean - new markers affect 2 changeset and cached ranges (glob)
- * @0000000000000000000000000000000000000000 (*)> updated evo-ext-obshashrange in *.???? seconds (0r, 2o) (glob)
+ 1970/01/01 00:00:00 * @0000000000000000000000000000000000000000 (*)> obshashcache clean - new markers affect 3 changeset and cached ranges (glob)
+ * @0000000000000000000000000000000000000000 (*)> updated evo-ext-obshashrange in *.???? seconds (0r, 4o) (glob)
* @0000000000000000000000000000000000000000 (*)> -R server serve --stdio exited 0 after *.?? seconds (glob)
* @0000000000000000000000000000000000000000 (*)> -R ../server blackbox (glob)
$ rm ../server/.hg/blackbox.log