test-compat: merge mercurial-4.4 into mercurial-4.3 mercurial-4.3
authorPierre-Yves David <pierre-yves.david@octobus.net>
Tue, 28 Aug 2018 11:25:32 +0200
branchmercurial-4.3
changeset 4033 b609412b42b0
parent 4002 21eab0a99841 (current diff)
parent 4032 fc065ec30351 (diff)
child 4077 9c025ec4af88
test-compat: merge mercurial-4.4 into mercurial-4.3
tests/test-discovery-obshashrange.t
--- a/.hgtags	Tue Aug 21 14:09:33 2018 +0200
+++ b/.hgtags	Tue Aug 28 11:25:32 2018 +0200
@@ -68,3 +68,4 @@
 116cdd8c102ab0ae6295fb4886b0882e75e4d8f7 8.0.0
 0887c30255a1a1808d74a63b16e896d457f8ef32 8.0.1
 2c5d79c6459c6fabe0eb8723fc5041ac0dac7a9a 8.1.0
+e7abf863e1130e14cd4d65e53467a199d267b4fd 8.1.1
--- a/CHANGELOG	Tue Aug 21 14:09:33 2018 +0200
+++ b/CHANGELOG	Tue Aug 28 11:25:32 2018 +0200
@@ -1,7 +1,12 @@
 Changelog
 =========
 
-8.1.1 - in progress
+8.1.2 - in progress
+-------------------
+
+  * obshashrange: improved robusness of the cache under heavy load
+
+8.1.1 -- 2018-08-21
 -------------------
 
   * clone: fix possible crash when using clone bundle and forcing cache warming
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/contrib/hammerclient.py	Tue Aug 28 11:25:32 2018 +0200
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+import os
+import sys
+import subprocess
+
+if len(sys.argv) < 2:
+    execname = os.path.basename(sys.argv[0])
+    print >> sys.stderr, "usage: %s CLIENT_ID" % execname
+
+client_id = sys.argv[1]
+
+subprocess.check_call(['hg', 'branch', "--force", "hammer-branch-%s" % client_id])
+
+while True:
+    subprocess.check_call([
+        'hg', 'commit',
+        "--config", "ui.allowemptycommit=yes",
+        "--message", "hammer-%s" % client_id,
+    ])
+    nodeid = subprocess.check_output([
+        'hg', 'log', '--rev', '.', '--template', '{node}'
+    ])
+    subprocess.check_call([
+        'hg', 'debugobsolete', ''.join(reversed(nodeid)), nodeid
+    ])
+    subprocess.check_call(['hg', 'pull'])
+    subprocess.check_call(['hg', 'push', '--force'])
--- a/debian/changelog	Tue Aug 21 14:09:33 2018 +0200
+++ b/debian/changelog	Tue Aug 28 11:25:32 2018 +0200
@@ -1,3 +1,9 @@
+mercurial-evolve (8.1.1) unstable; urgency=medium
+
+  * new upstream release
+
+ -- Pierre-Yves David <pierre-yves.david@ens-lyon.org>  Tue, 21 Aug 2018 15:28:29 +0200
+
 mercurial-evolve (8.0.1-1) unstable; urgency=medium
 
   * New upstream release
--- a/hgext3rd/evolve/metadata.py	Tue Aug 21 14:09:33 2018 +0200
+++ b/hgext3rd/evolve/metadata.py	Tue Aug 28 11:25:32 2018 +0200
@@ -5,7 +5,7 @@
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-__version__ = '8.1.1.dev'
+__version__ = '8.1.2.dev'
 testedwith = '4.3.2 4.4.2 4.5.2 4.6.2 4.7'
 minimumhgversion = '4.3'
 buglink = 'https://bz.mercurial-scm.org/'
--- a/hgext3rd/evolve/obsdiscovery.py	Tue Aug 21 14:09:33 2018 +0200
+++ b/hgext3rd/evolve/obsdiscovery.py	Tue Aug 28 11:25:32 2018 +0200
@@ -328,6 +328,11 @@
 ### sqlite caching
 
 _sqliteschema = [
+    """CREATE TABLE obshashrange(rev     INTEGER NOT NULL,
+                                 idx     INTEGER NOT NULL,
+                                 obshash BLOB    NOT NULL,
+                                 PRIMARY KEY(rev, idx));""",
+    "CREATE INDEX range_index ON obshashrange(rev, idx);",
     """CREATE TABLE meta(schemaversion INTEGER NOT NULL,
                          tiprev        INTEGER NOT NULL,
                          tipnode       BLOB    NOT NULL,
@@ -335,11 +340,6 @@
                          obssize       BLOB    NOT NULL,
                          obskey        BLOB    NOT NULL
                         );""",
-    """CREATE TABLE obshashrange(rev     INTEGER NOT NULL,
-                                 idx     INTEGER NOT NULL,
-                                 obshash BLOB    NOT NULL,
-                                 PRIMARY KEY(rev, idx));""",
-    "CREATE INDEX range_index ON obshashrange(rev, idx);",
 ]
 _queryexist = "SELECT name FROM sqlite_master WHERE type='table' AND name='meta';"
 _clearmeta = """DELETE FROM meta;"""
@@ -432,10 +432,17 @@
         value = self._data.get(rangeid)
         if value is None and self._con is not None:
             nrange = (rangeid[0], rangeid[1])
-            obshash = self._con.execute(_queryobshash, nrange).fetchone()
-            if obshash is not None:
-                value = obshash[0]
-            self._data[rangeid] = value
+            try:
+                obshash = self._con.execute(_queryobshash, nrange).fetchone()
+                if obshash is not None:
+                    value = obshash[0]
+                self._data[rangeid] = value
+            except (sqlite3.DatabaseError, sqlite3.OperationalError):
+                # something is wrong with the sqlite db
+                # Since this is a cache, we ignore it.
+                if '_con' in vars(self):
+                    del self._con
+                self._new.clear()
         return value
 
     def __setitem__(self, rangeid, obshash):
@@ -473,14 +480,19 @@
                 if con is not None:
                     # always reset for now, the code detecting affect is buggy
                     # so we need to reset more broadly than we would like.
-                    if repo.stablerange._con is None:
-                        con.execute(_reset)
-                        self._data.clear()
-                    else:
-                        ranges = repo.stablerange.contains(repo, affected)
-                        con.executemany(_delete, ranges)
-                        for r in ranges:
-                            self._data.pop(r, None)
+                    try:
+                        if repo.stablerange._con is None:
+                            con.execute(_reset)
+                            self._data.clear()
+                        else:
+                            ranges = repo.stablerange.contains(repo, affected)
+                            con.executemany(_delete, ranges)
+                            for r in ranges:
+                                self._data.pop(r, None)
+                    except (sqlite3.DatabaseError, sqlite3.OperationalError) as exc:
+                        repo.ui.log('evoext-cache', 'error while updating obshashrange cache: %s' % exc)
+                        del self._updating
+                        return
 
                 # rewarm key revisions
                 #
@@ -524,7 +536,7 @@
             util.makedirs(self._vfs.dirname(self._path))
         except OSError:
             return None
-        con = sqlite3.connect(self._path)
+        con = sqlite3.connect(self._path, timeout=30, isolation_level="IMMEDIATE")
         con.text_factory = str
         return con
 
@@ -568,6 +580,21 @@
             repo.ui.warn(msg)
 
     def _save(self, repo):
+        if not self._new:
+            return
+        try:
+            return self._trysave(repo)
+        except (sqlite3.DatabaseError, sqlite3.OperationalError, sqlite3.IntegrityError) as exc:
+            # Catch error that may arise under stress
+            #
+            # operational error catch read-only and locked database
+            # IntegrityError catch Unique constraint error that may arise
+            if '_con' in vars(self):
+                del self._con
+            self._new.clear()
+            repo.ui.log('evoext-cache', 'error while saving new data: %s' % exc)
+
+    def _trysave(self, repo):
         if self._con is None:
             util.unlinkpath(self._path, ignoremissing=True)
             if '_con' in vars(self):
@@ -582,30 +609,34 @@
                 for req in _sqliteschema:
                     con.execute(req)
 
-                con.execute(_newmeta, self._fullcachekey)
+                meta = [self._schemaversion] + list(self.emptykey)
+                con.execute(_newmeta, meta)
+                self._ondiskcachekey = self.emptykey
         else:
             con = self._con
-            if self._ondiskcachekey is not None:
-                meta = con.execute(_querymeta).fetchone()
-                if meta[1:] != self._ondiskcachekey:
-                    # drifting is currently an issue because this means another
-                    # process might have already added the cache line we are about
-                    # to add. This will confuse sqlite
-                    msg = _('obshashrange cache: skipping write, '
-                            'database drifted under my feet\n')
-                    data = (meta[2], meta[1], self._ondiskcachekey[0], self._ondiskcachekey[1])
-                    repo.ui.warn(msg)
-                    return
-        data = ((rangeid[0], rangeid[1], self.get(rangeid)) for rangeid in self._new)
-        con.executemany(_updateobshash, data)
-        cachekey = self._fullcachekey
-        con.execute(_clearmeta) # remove the older entry
-        con.execute(_newmeta, cachekey)
-        con.commit()
-        self._new.clear()
-        self._ondiskcachekey = self._cachekey
-        self._valid = True
-
+        with con:
+            meta = con.execute(_querymeta).fetchone()
+            if meta[1:] != self._ondiskcachekey:
+                # drifting is currently an issue because this means another
+                # process might have already added the cache line we are about
+                # to add. This will confuse sqlite
+                msg = _('obshashrange cache: skipping write, '
+                        'database drifted under my feet\n')
+                repo.ui.warn(msg)
+                self._new.clear()
+                self._valid = False
+                if '_con' in vars(self):
+                    del self._con
+                self._valid = False
+                return
+            data = ((rangeid[0], rangeid[1], self.get(rangeid)) for rangeid in self._new)
+            con.executemany(_updateobshash, data)
+            cachekey = self._fullcachekey
+            con.execute(_clearmeta) # remove the older entry
+            con.execute(_newmeta, cachekey)
+            self._new.clear()
+            self._valid = True
+            self._ondiskcachekey = self._cachekey
 @eh.wrapfunction(obsolete.obsstore, '_addmarkers')
 def _addmarkers(orig, obsstore, *args, **kwargs):
     obsstore.rangeobshashcache.clear()
--- a/hgext3rd/evolve/stablerangecache.py	Tue Aug 21 14:09:33 2018 +0200
+++ b/hgext3rd/evolve/stablerangecache.py	Tue Aug 28 11:25:32 2018 +0200
@@ -17,6 +17,7 @@
 from mercurial import (
     error,
     localrepo,
+    node as nodemod,
     util,
 )
 
@@ -87,10 +88,6 @@
 #############################
 
 _sqliteschema = [
-    """CREATE TABLE meta(schemaversion INTEGER NOT NULL,
-                         tiprev        INTEGER NOT NULL,
-                         tipnode       BLOB    NOT NULL
-                        );""",
     """CREATE TABLE range(rev INTEGER  NOT NULL,
                           idx INTEGER NOT NULL,
                           PRIMARY KEY(rev, idx));""",
@@ -106,6 +103,10 @@
     "CREATE INDEX subranges_index ON subranges (suprev, supidx);",
     "CREATE INDEX superranges_index ON subranges (subrev, subidx);",
     "CREATE INDEX range_index ON range (rev, idx);",
+    """CREATE TABLE meta(schemaversion INTEGER NOT NULL,
+                         tiprev        INTEGER NOT NULL,
+                         tipnode       BLOB    NOT NULL
+                        );""",
 ]
 _newmeta = "INSERT INTO meta (schemaversion, tiprev, tipnode) VALUES (?,?,?);"
 _updatemeta = "UPDATE meta SET tiprev = ?, tipnode = ?;"
@@ -177,11 +178,19 @@
         cache = self._subrangescache
         if rangeid not in cache and rangeid[0] <= self._ondisktiprev and self._con is not None:
             value = None
-            result = self._con.execute(_queryrange, rangeid).fetchone()
-            if result is not None: # database know about this node (skip in the future?)
-                value = self._con.execute(_querysubranges, rangeid).fetchall()
-            # in memory caching of the value
-            cache[rangeid] = value
+            try:
+                result = self._con.execute(_queryrange, rangeid).fetchone()
+                if result is not None: # database know about this node (skip in the future?)
+                    value = self._con.execute(_querysubranges, rangeid).fetchall()
+                # in memory caching of the value
+                cache[rangeid] = value
+            except (sqlite3.DatabaseError, sqlite3.OperationalError):
+                # something is wrong with the sqlite db
+                # Since this is a cache, we ignore it.
+                if '_con' in vars(self):
+                    del self._con
+                self._unsavedsubranges.clear()
+
         return cache.get(rangeid)
 
     def _setsub(self, rangeid, value):
@@ -194,7 +203,7 @@
             util.makedirs(self._vfs.dirname(self._path))
         except OSError:
             return None
-        con = sqlite3.connect(self._path)
+        con = sqlite3.connect(self._path, timeout=30, isolation_level="IMMEDIATE")
         con.text_factory = str
         return con
 
@@ -223,6 +232,21 @@
         return con
 
     def _save(self, repo):
+        if not self._unsavedsubranges:
+            return
+        try:
+            return self._trysave(repo)
+        except (sqlite3.DatabaseError, sqlite3.OperationalError, sqlite3.IntegrityError) as exc:
+            # Catch error that may arise under stress
+            #
+            # operational error catch read-only and locked database
+            # IntegrityError catch Unique constraint error that may arise
+            if '_con' in vars(self):
+                del self._con
+            self._unsavedsubranges.clear()
+            repo.ui.log('evoext-cache', 'error while saving new data: %s' % exc)
+
+    def _trysave(self, repo):
         repo = repo.unfiltered()
         repo.depthcache.save(repo)
         repo.stablesort.save(repo)
@@ -238,19 +262,20 @@
             con = self._db()
             if con is None:
                 return
-            con.execute('BEGIN IMMEDIATE;')
             with con:
                 for req in _sqliteschema:
                     con.execute(req)
 
                 meta = [self._schemaversion,
-                        self._tiprev,
-                        self._tipnode,
+                        nodemod.nullrev,
+                        nodemod.nullid,
                 ]
                 con.execute(_newmeta, meta)
+                self._ondisktiprev = nodemod.nullrev
+                self._ondisktipnode = nodemod.nullid
         else:
             con = self._con
-            con.execute('BEGIN IMMEDIATE;')
+        with con:
             meta = con.execute(_querymeta).fetchone()
             if meta[2] != self._ondisktipnode or meta[1] != self._ondisktiprev:
                 # drifting is currently an issue because this means another
@@ -258,19 +283,19 @@
                 # to add. This will confuse sqlite
                 msg = _('stable-range cache: skipping write, '
                         'database drifted under my feet\n')
-                hint = _('(disk: %s-%s vs mem: %s%s)\n')
-                data = (meta[2], meta[1], self._ondisktiprev, self._ondisktipnode)
+                hint = _('(disk: %s-%s vs mem: %s-%s)\n')
+                data = (nodemod.hex(meta[2]), meta[1],
+                        nodemod.hex(self._ondisktipnode), self._ondisktiprev)
                 repo.ui.warn(msg)
                 repo.ui.warn(hint % data)
-                con.execute('ROLLBACK;')
+                self._unsavedsubranges.clear()
                 return
-            meta = [self._tiprev,
-                    self._tipnode,
-            ]
-            con.execute(_updatemeta, meta)
-
-        self._saverange(con, repo)
-        con.commit()
+            else:
+                self._saverange(con, repo)
+                meta = [self._tiprev,
+                        self._tipnode,
+                ]
+                con.execute(_updatemeta, meta)
         self._ondisktiprev = self._tiprev
         self._ondisktipnode = self._tipnode
         self._unsavedsubranges.clear()
--- a/tests/test-discovery-obshashrange.t	Tue Aug 21 14:09:33 2018 +0200
+++ b/tests/test-discovery-obshashrange.t	Tue Aug 28 11:25:32 2018 +0200
@@ -394,8 +394,8 @@
   * @0000000000000000000000000000000000000000 (*)> updated evo-ext-obscache in *.???? seconds (0r, 1o) (glob)
   * @0000000000000000000000000000000000000000 (*)> -R ../server debugobsolete bbbbbbb2222222222bbbbbbbbbbbbb2222222222 bebd167eb94d257ace0e814aeb98e6972ed2970d exited 0 after *.?? seconds (glob)
   * @0000000000000000000000000000000000000000 (*)> serve --stdio (glob)
-  * @0000000000000000000000000000000000000000 (*)> obshashcache clean - new markers affect 2 changeset and cached ranges (glob)
-  * @0000000000000000000000000000000000000000 (*)> updated evo-ext-obshashrange in *.???? seconds (0r, 2o) (glob)
+  * @0000000000000000000000000000000000000000 (*)> obshashcache clean - new markers affect 3 changeset and cached ranges (glob)
+  * @0000000000000000000000000000000000000000 (*)> updated evo-ext-obshashrange in *.???? seconds (0r, 4o) (glob)
   * @0000000000000000000000000000000000000000 (*)> -R server serve --stdio exited 0 after *.?? seconds (glob)
   * @0000000000000000000000000000000000000000 (*)> blackbox (glob)
   $ rm ../server/.hg/blackbox.log