importing fastop extension in this repo
authorPierre-Yves David <pierre-yves.david@logilab.fr>
Thu, 20 Feb 2014 12:56:57 -0800
changeset 801 f49d4774b999
parent 790 5af309865040 (current diff)
parent 800 ad2060da7ffa (diff)
child 802 80e078959129
importing fastop extension in this repo This repo is dedicated to experimentation on mutable history topic. Fastop is appropriate there.
README
--- a/README	Wed Feb 12 17:18:50 2014 -0800
+++ b/README	Thu Feb 20 12:56:57 2014 -0800
@@ -44,6 +44,7 @@
 
 3.3.0 --
 
+- added Augie Facklers `fastop` extension (usage not recommended yet)
 - add verbose hint about how to handle corner case by hand.
   This should help people until evolve is able to to it itself.
 - removed the qsync extension. The only user I knew about (logilab) is not
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/hgfastobs.py	Thu Feb 20 12:56:57 2014 -0800
@@ -0,0 +1,124 @@
+"""Extension to try and speed up transfer of obsolete markers.
+
+Mercurial 2.6 transfers obsolete markers in the dumbest way possible:
+it simply transfers all of them to the server on every
+operation. While this /works/, it's not ideal because it's a large
+amount of extra data for users to pull down (1.9M for the 17k obsolete
+markers in hg-crew as of this writing in late July 2013). It's also
+frustrating because this transfer takes a nontrivial amount of time.
+
+You can specify a strategy with the config knob
+obsolete.syncstrategy. Current strategies are "stock" and
+"boxfill". Default strategy is presently boxfill.
+
+:stock: use the default strategy of mercurial explaned above
+
+:boxfill: transmit obsolete markers which list any of transmitted changesets as
+          a successor (transitively), as well as any kill markers for dead
+          nodes descended from any of the precursors of outgoing.missing.
+
+TODO(durin42): consider better names for sync strategies.
+"""
+import sys
+
+from mercurial import base85
+from mercurial import commands
+from mercurial import extensions
+from mercurial import node
+from mercurial import obsolete
+from mercurial import exchange
+from mercurial import revset
+from mercurial.i18n import _
+
+_strategies = {
+    'stock': exchange._pushobsolete,
+    }
+
+def _strategy(name, default=False):
+    def inner(func):
+        _strategies[name] = func
+        if default:
+            _strategies[None] = func
+        return func
+    return inner
+
+def _pushobsoletewrapper(orig, pushop):
+    stratfn = _strategies[pushop.repo.ui.config('obsolete', 'syncstrategy')]
+    return stratfn(pushop)
+
+extensions.wrapfunction(exchange, '_pushobsolete', _pushobsoletewrapper)
+
+def _precursors(repo, s):
+    """Precursor of a changeset"""
+    cs = set()
+    nm = repo.changelog.nodemap
+    markerbysubj = repo.obsstore.precursors
+    for r in s:
+        for p in markerbysubj.get(repo[r].node(), ()):
+            pr = nm.get(p[0])
+            if pr is not None:
+                cs.add(pr)
+    return cs
+
+def _revsetprecursors(repo, subset, x):
+    s = revset.getset(repo, revset.baseset(range(len(repo))), x)
+    cs = _precursors(repo, s)
+    return revset.baseset([r for r in subset if r in cs])
+
+revset.symbols['_fastobs_precursors'] = _revsetprecursors
+
+
+@_strategy('boxfill', default=True)
+def boxfill(pushop):
+    """The "fill in the box" strategy from the 2.6 sprint.
+
+    See the notes[0] from the 2.6 sprint for what "fill in the box"
+    means here. It's a fairly subtle algorithm, which may have
+    surprising behavior at times, but was the least-bad option
+    proposed at the sprint.
+
+    [0]: https://bitbucket.org/durin42/2.6sprint-notes/src/tip/mercurial26-obsstore-rev.1398.txt
+    """
+    repo = pushop.repo
+    remote = pushop.remote
+    outgoing = pushop.outgoing
+    urepo = pushop.repo.unfiltered()
+    # need to collect obsolete markers which list any of
+    # outgoing.missing as a successor (transitively), as well as any
+    # kill markers for dead nodes descended from any of the precursors
+    # of outgoing.missing.
+    boxedges = urepo.revs(
+        '(descendants(_fastobs_precursors(%ln)) or '
+        ' descendants(%ln)) and hidden()',
+        outgoing.missing, outgoing.missing)
+    transmit = []
+    for node in outgoing.missing:
+        transmit.extend(obsolete.precursormarkers(urepo[node]))
+    for node in boxedges:
+        transmit.extend(obsolete.successormarkers(urepo[node]))
+    transmit = list(set(transmit))
+    xmit, total = len(transmit), len(repo.obsstore._all)
+    repo.ui.status(
+        'boxpush: about to transmit %d obsolete markers (%d markers total)\n'
+        % (xmit, total))
+    parts, size, chunk = [], 0, 0
+    def transmitmarks():
+            repo.ui.note(
+                'boxpush: sending a chunk of obsolete markers\n')
+            data = ''.join([obsolete._pack('>B', obsolete._fmversion)] + parts)
+            remote.pushkey('obsolete', 'dump-%d' % chunk, '',
+                           base85.b85encode(data))
+
+    for marker in transmit:
+        enc = obsolete._encodeonemarker(_markertuple(marker))
+        parts.append(enc)
+        size += len(enc)
+        if size > obsolete._maxpayload:
+            transmitmarks()
+            parts, size = [], 0
+            chunk += 1
+    if parts:
+        transmitmarks()
+
+def _markertuple(marker):
+    return marker._data
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test-boxpush.t	Thu Feb 20 12:56:57 2014 -0800
@@ -0,0 +1,44 @@
+  $ fastobs="$TESTDIR"/../hgfastobs.py
+  $ echo 'from mercurial import obsolete ; obsolete._enabled = True' > enableobs.py
+  $ cat >> $HGRCPATH <<EOF
+  > [obsolete]
+  > syncstrategy = boxfill
+  > [extensions]
+  > EOF
+  $ echo "enable-obsolete = $PWD/enableobs.py" >> $HGRCPATH
+  $ echo "fastobs = $fastobs" >> $HGRCPATH
+
+  $ hg init alice
+  $ hg init bob
+  $ hg init trent
+  $ cd alice
+  $ echo a > a
+  $ hg addr && hg ci -m 'add a'
+  adding a
+  $ echo aa >> a
+  $ hg ci -m 'edit a'
+  $ echo aa > a
+  $ hg ci --amend -m 'edit a'
+  $ hg debugobsolete
+  e772e827cd64564621e7e5af15c9f848e3b92c8e efa8cd969bc37e6a1330c29f4234fe9e9be681b3 0 {'date': '* 0', 'user': 'test'} (glob)
+  5ccfcbc00f2a19cd7affedce5ff087e68e67c6cc 0 {'date': '* 0', 'user': 'test'} (glob)
+  $ hg push ../trent
+  pushing to ../trent
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 1 files
+  boxpush: about to transmit 2 obsolete markers (2 markers total)
+  $ cd ../bob
+  $ hg pull ../trent
+  pulling from ../trent
+  requesting all changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 2 changesets with 2 changes to 1 files
+  (run 'hg update' to get a working copy)
+  $ hg debugobsolete | sort
+  5ccfcbc00f2a19cd7affedce5ff087e68e67c6cc 0 {'date': '* 0', 'user': 'test'} (glob)
+  e772e827cd64564621e7e5af15c9f848e3b92c8e efa8cd969bc37e6a1330c29f4234fe9e9be681b3 0 {'date': '* 0', 'user': 'test'} (glob)