hgfastobs.py
changeset 791 5f3b53d74b7f
child 792 36d0e71aa9e4
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/hgfastobs.py	Fri Jul 26 16:04:40 2013 -0400
@@ -0,0 +1,93 @@
+"""Extension to try and speed up transfer of obsolete markers.
+
+Mercurial 2.6 transfers obsolete markers in the dumbest way possible:
+it simply transfers all of them to the server on every
+operation. While this /works/, it's not ideal because it's a large
+amount of extra data for users to pull down (1.9M for the 17k obsolete
+markers in hg-crew as of this writing in late July 2013). It's also
+frustrating because this transfer takes a nontrivial amount of time.
+
+You can specify a strategy with the config knob
+obsolete.syncstrategy. Current strategies are "stock" and
+"boxfill". Default strategy is presently boxfill.
+
+TODO(durin42): consider better names for sync strategies.
+"""
+import sys
+
+from mercurial import commands
+from mercurial import extensions
+from mercurial import obsolete
+from mercurial import node
+from mercurial.i18n import _
+
+_strategies = {
+    'stock': obsolete.syncpush,
+    }
+
+def _strategy(name, default=False):
+    def inner(func):
+        _strategies[name] = func
+        if default:
+            _strategies[None] = func
+        return func
+    return inner
+
+def syncpushwrapper(orig, repo, remote):
+    stratfn = _strategies[repo.ui.config('obsolete', 'syncstrategy')]
+    return stratfn(repo, remote)
+
+extensions.wrapfunction(obsolete, 'syncpush', syncpushwrapper)
+
+def pushmarkerwrapper(orig, repo, *args):
+    if repo.ui.config('obsolete', 'syncstrategy') == 'stock':
+        return orig(repo, *args)
+    # We shouldn't need to do this, since we transmit markers
+    # effectively during push in localrepo. Just return success.
+    return 1
+
+def _getoutgoing():
+    f = sys._getframe(4)
+    return f.f_locals['outgoing']
+
+@_strategy('boxfill', default=True)
+def boxfill(repo, remote):
+    """The "fill in the box" strategy from the 2.6 sprint.
+
+    See the notes[0] from the 2.6 sprint for what "fill in the box"
+    means here. It's a fairly subtle algorithm, which may have
+    surprising behavior at times, but was the least-bad option
+    proposed at the sprint.
+
+    [0]: https://bitbucket.org/durin42/2.6sprint-notes/src/tip/mercurial26-obsstore-rev.1398.txt
+    """
+    outgoing = _getoutgoing()
+    urepo = repo.unfiltered()
+    # need to collect obsolete markers which list any of
+    # outgoing.missing as a successor (transitively), as well as any
+    # kill markers for dead nodes descended from any of the precursors
+    # of outgoing.missing.
+    boxedges = urepo.revs(
+        '(descendants(precursors(%ln)) or descendants(%ln)) and hidden()',
+        outgoing.missing, outgoing.missing)
+    transmit = []
+    for node in boxedges:
+        transmit.extend(obsolete.successormarkers(urepo[node]))
+    xmit, total = len(transmit), len(repo.obsstore._all)
+    repo.ui.status('about to transmit %d obsolete markers (%d markers total)\n'
+                   % (xmit, total))
+    parts, size, chunk = [], 0, 0
+    for marker in transmit:
+        enc = obsolete._encodeonemarker(_markertuple(marker))
+        parts.append(enc)
+        size += len(enc)
+        if size > obsolete._maxpayload:
+            repo.ui.note(
+                'obsolete boxpush: sending a chunk of obsolete markers\n')
+            data = ''.join([obsolete._pack('>B', _fmversion)], parts)
+            remote.pushkey('obsolete', 'dump%d' % chunk, base85.b85encode(data))
+            parts, size = [], 0
+            chunk += 1
+
+def _markertuple(marker):
+    return marker._data