hgfastobs.py
author Augie Fackler <raf@durin42.com>
Fri, 26 Jul 2013 16:08:56 -0400
changeset 792 36d0e71aa9e4
parent 791 5f3b53d74b7f
child 793 fa746ef46e8a
permissions -rw-r--r--
fastobs: clean up logging a little

"""Extension to try and speed up transfer of obsolete markers.

Mercurial 2.6 transfers obsolete markers in the dumbest way possible:
it simply transfers all of them to the server on every
operation. While this /works/, it's not ideal because it's a large
amount of extra data for users to pull down (1.9M for the 17k obsolete
markers in hg-crew as of this writing in late July 2013). It's also
frustrating because this transfer takes a nontrivial amount of time.

You can specify a strategy with the config knob
obsolete.syncstrategy. Current strategies are "stock" and
"boxfill". Default strategy is presently boxfill.

TODO(durin42): consider better names for sync strategies.
"""
import sys

from mercurial import commands
from mercurial import extensions
from mercurial import obsolete
from mercurial import node
from mercurial.i18n import _

_strategies = {
    'stock': obsolete.syncpush,
    }

def _strategy(name, default=False):
    def inner(func):
        _strategies[name] = func
        if default:
            _strategies[None] = func
        return func
    return inner

def syncpushwrapper(orig, repo, remote):
    stratfn = _strategies[repo.ui.config('obsolete', 'syncstrategy')]
    return stratfn(repo, remote)

extensions.wrapfunction(obsolete, 'syncpush', syncpushwrapper)

def pushmarkerwrapper(orig, repo, *args):
    if repo.ui.config('obsolete', 'syncstrategy') == 'stock':
        return orig(repo, *args)
    # We shouldn't need to do this, since we transmit markers
    # effectively during push in localrepo. Just return success.
    return 1

def _getoutgoing():
    f = sys._getframe(4)
    return f.f_locals['outgoing']

@_strategy('boxfill', default=True)
def boxfill(repo, remote):
    """The "fill in the box" strategy from the 2.6 sprint.

    See the notes[0] from the 2.6 sprint for what "fill in the box"
    means here. It's a fairly subtle algorithm, which may have
    surprising behavior at times, but was the least-bad option
    proposed at the sprint.

    [0]: https://bitbucket.org/durin42/2.6sprint-notes/src/tip/mercurial26-obsstore-rev.1398.txt
    """
    outgoing = _getoutgoing()
    urepo = repo.unfiltered()
    # need to collect obsolete markers which list any of
    # outgoing.missing as a successor (transitively), as well as any
    # kill markers for dead nodes descended from any of the precursors
    # of outgoing.missing.
    boxedges = urepo.revs(
        '(descendants(precursors(%ln)) or descendants(%ln)) and hidden()',
        outgoing.missing, outgoing.missing)
    transmit = []
    for node in boxedges:
        transmit.extend(obsolete.successormarkers(urepo[node]))
    xmit, total = len(transmit), len(repo.obsstore._all)
    repo.ui.status(
        'boxpush: about to transmit %d obsolete markers (%d markers total)\n'
        % (xmit, total))
    parts, size, chunk = [], 0, 0
    for marker in transmit:
        enc = obsolete._encodeonemarker(_markertuple(marker))
        parts.append(enc)
        size += len(enc)
        if size > obsolete._maxpayload:
            repo.ui.note(
                'boxpush: sending a chunk of obsolete markers\n')
            data = ''.join([obsolete._pack('>B', _fmversion)], parts)
            remote.pushkey('obsolete', 'dump%d' % chunk, base85.b85encode(data))
            parts, size = [], 0
            chunk += 1

def _markertuple(marker):
    return marker._data