discovery: split discovery related code in 'obsdiscovery'
More code splitting for more clarity.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hgext3rd/evolve/obsdiscovery.py Tue Mar 07 14:29:43 2017 +0100
@@ -0,0 +1,204 @@
+# Code dedicated to the discovery of obsolescence marker "over the wire"
+#
+# Copyright 2017 Pierre-Yves David <pierre-yves.david@ens-lyon.org>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+try:
+ import StringIO as io
+ StringIO = io.StringIO
+except ImportError:
+ import io
+ StringIO = io.StringIO
+
+from mercurial import (
+ error,
+ exchange,
+ localrepo,
+ node,
+ obsolete,
+ wireproto,
+)
+from mercurial.i18n import _
+
+from . import (
+ exthelper,
+ serveronly,
+ utility,
+)
+
+eh = exthelper.exthelper()
+obsexcmsg = utility.obsexcmsg
+
+
+from mercurial import dagutil
+from mercurial import setdiscovery
+
+@eh.addattr(localrepo.localpeer, 'evoext_obshash')
+def local_obshash(peer, nodes):
+ return serveronly._obshash(peer._repo, nodes)
+
+@eh.addattr(localrepo.localpeer, 'evoext_obshash1')
+def local_obshash1(peer, nodes):
+ return serveronly._obshash(peer._repo, nodes, version=1)
+
+@eh.addattr(wireproto.wirepeer, 'evoext_obshash')
+def peer_obshash(self, nodes):
+ d = self._call("evoext_obshash", nodes=wireproto.encodelist(nodes))
+ try:
+ return wireproto.decodelist(d)
+ except ValueError:
+ self._abort(error.ResponseError(_("unexpected response:"), d))
+
+@eh.addattr(wireproto.wirepeer, 'evoext_obshash1')
+def peer_obshash1(self, nodes):
+ d = self._call("evoext_obshash1", nodes=wireproto.encodelist(nodes))
+ try:
+ return wireproto.decodelist(d)
+ except ValueError:
+ self._abort(error.ResponseError(_("unexpected response:"), d))
+
+def findcommonobsmarkers(ui, local, remote, probeset,
+ initialsamplesize=100,
+ fullsamplesize=200):
+ # from discovery
+ roundtrips = 0
+ cl = local.changelog
+ dag = dagutil.revlogdag(cl)
+ missing = set()
+ common = set()
+ undecided = set(probeset)
+ totalnb = len(undecided)
+ ui.progress(_("comparing with other"), 0, total=totalnb)
+ _takefullsample = setdiscovery._takefullsample
+ if remote.capable('_evoext_obshash_1'):
+ getremotehash = remote.evoext_obshash1
+ localhash = serveronly._obsrelsethashtreefm1(local)
+ else:
+ getremotehash = remote.evoext_obshash
+ localhash = serveronly._obsrelsethashtreefm0(local)
+
+ while undecided:
+
+ ui.note(_("sampling from both directions\n"))
+ if len(undecided) < fullsamplesize:
+ sample = set(undecided)
+ else:
+ sample = _takefullsample(dag, undecided, size=fullsamplesize)
+
+ roundtrips += 1
+ ui.progress(_("comparing with other"), totalnb - len(undecided),
+ total=totalnb)
+ ui.debug("query %i; still undecided: %i, sample size is: %i\n"
+ % (roundtrips, len(undecided), len(sample)))
+ # indices between sample and externalized version must match
+ sample = list(sample)
+ remotehash = getremotehash(dag.externalizeall(sample))
+
+ yesno = [localhash[ix][1] == remotehash[si]
+ for si, ix in enumerate(sample)]
+
+ commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
+ common.update(dag.ancestorset(commoninsample, common))
+
+ missinginsample = [n for i, n in enumerate(sample) if not yesno[i]]
+ missing.update(dag.descendantset(missinginsample, missing))
+
+ undecided.difference_update(missing)
+ undecided.difference_update(common)
+
+ ui.progress(_("comparing with other"), None)
+ result = dag.headsetofconnecteds(common)
+ ui.debug("%d total queries\n" % roundtrips)
+
+ if not result:
+ return set([node.nullid])
+ return dag.externalizeall(result)
+
+@eh.wrapfunction(exchange, '_pushdiscoveryobsmarkers')
+def _pushdiscoveryobsmarkers(orig, pushop):
+ if (obsolete.isenabled(pushop.repo, obsolete.exchangeopt)
+ and pushop.repo.obsstore
+ and 'obsolete' in pushop.remote.listkeys('namespaces')):
+ repo = pushop.repo
+ obsexcmsg(repo.ui, "computing relevant nodes\n")
+ revs = list(repo.revs('::%ln', pushop.futureheads))
+ unfi = repo.unfiltered()
+ cl = unfi.changelog
+ if not pushop.remote.capable('_evoext_obshash_0'):
+ # do not trust core yet
+ # return orig(pushop)
+ nodes = [cl.node(r) for r in revs]
+ if nodes:
+ obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n"
+ % len(nodes))
+ pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
+ else:
+ obsexcmsg(repo.ui, "markers already in sync\n")
+ pushop.outobsmarkers = []
+ pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
+ return
+
+ common = []
+ obsexcmsg(repo.ui, "looking for common markers in %i nodes\n"
+ % len(revs))
+ commonrevs = list(unfi.revs('::%ln', pushop.outgoing.commonheads))
+ common = findcommonobsmarkers(pushop.ui, unfi, pushop.remote,
+ commonrevs)
+
+ revs = list(unfi.revs('%ld - (::%ln)', revs, common))
+ nodes = [cl.node(r) for r in revs]
+ if nodes:
+ obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n"
+ % len(nodes))
+ pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
+ else:
+ obsexcmsg(repo.ui, "markers already in sync\n")
+ pushop.outobsmarkers = []
+
+@eh.extsetup
+def _installobsmarkersdiscovery(ui):
+ olddisco = exchange.pushdiscoverymapping['obsmarker']
+
+ def newdisco(pushop):
+ _pushdiscoveryobsmarkers(olddisco, pushop)
+ exchange.pushdiscoverymapping['obsmarker'] = newdisco
+
+def _buildpullobsmarkersboundaries(pullop):
+ """small funtion returning the argument for pull markers call
+ may to contains 'heads' and 'common'. skip the key for None.
+
+ Its a separed functio to play around with strategy for that."""
+ repo = pullop.repo
+ remote = pullop.remote
+ unfi = repo.unfiltered()
+ revs = unfi.revs('::(%ln - null)', pullop.common)
+ common = [node.nullid]
+ if remote.capable('_evoext_obshash_0'):
+ obsexcmsg(repo.ui, "looking for common markers in %i nodes\n"
+ % len(revs))
+ common = findcommonobsmarkers(repo.ui, repo, remote, revs)
+ return {'heads': pullop.pulledsubset, 'common': common}
+
+@eh.command(
+ 'debugobsrelsethashtree',
+ [('', 'v0', None, 'hash on marker format "0"'),
+ ('', 'v1', None, 'hash on marker format "1" (default)')], _(''))
+def debugobsrelsethashtree(ui, repo, v0=False, v1=False):
+ """display Obsolete markers, Relevant Set, Hash Tree
+ changeset-node obsrelsethashtree-node
+
+ It computed form the "orsht" of its parent and markers
+ relevant to the changeset itself."""
+ if v0 and v1:
+ raise error.Abort('cannot only specify one format')
+ elif v0:
+ treefunc = serveronly._obsrelsethashtreefm0
+ else:
+ treefunc = serveronly._obsrelsethashtreefm1
+
+ for chg, obs in treefunc(repo):
+ ui.status('%s %s\n' % (node.hex(chg), node.hex(obs)))
--- a/hgext3rd/evolve/obsexchange.py Tue Mar 07 14:19:12 2017 +0100
+++ b/hgext3rd/evolve/obsexchange.py Tue Mar 07 14:29:43 2017 +0100
@@ -33,152 +33,13 @@
exthelper,
serveronly,
utility,
+ obsdiscovery,
)
eh = exthelper.exthelper()
+eh.merge(obsdiscovery.eh)
obsexcmsg = utility.obsexcmsg
-
-def obsexcprg(ui, *args, **kwargs):
- topic = 'obsmarkers exchange'
- if ui.configbool('experimental', 'verbose-obsolescence-exchange', False):
- topic = 'OBSEXC'
- ui.progress(topic, *args, **kwargs)
-
-@eh.wrapfunction(exchange, '_pushdiscoveryobsmarkers')
-def _pushdiscoveryobsmarkers(orig, pushop):
- if (obsolete.isenabled(pushop.repo, obsolete.exchangeopt)
- and pushop.repo.obsstore
- and 'obsolete' in pushop.remote.listkeys('namespaces')):
- repo = pushop.repo
- obsexcmsg(repo.ui, "computing relevant nodes\n")
- revs = list(repo.revs('::%ln', pushop.futureheads))
- unfi = repo.unfiltered()
- cl = unfi.changelog
- if not pushop.remote.capable('_evoext_obshash_0'):
- # do not trust core yet
- # return orig(pushop)
- nodes = [cl.node(r) for r in revs]
- if nodes:
- obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n"
- % len(nodes))
- pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
- else:
- obsexcmsg(repo.ui, "markers already in sync\n")
- pushop.outobsmarkers = []
- pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
- return
-
- common = []
- obsexcmsg(repo.ui, "looking for common markers in %i nodes\n"
- % len(revs))
- commonrevs = list(unfi.revs('::%ln', pushop.outgoing.commonheads))
- common = findcommonobsmarkers(pushop.ui, unfi, pushop.remote,
- commonrevs)
-
- revs = list(unfi.revs('%ld - (::%ln)', revs, common))
- nodes = [cl.node(r) for r in revs]
- if nodes:
- obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n"
- % len(nodes))
- pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes)
- else:
- obsexcmsg(repo.ui, "markers already in sync\n")
- pushop.outobsmarkers = []
-
-@eh.extsetup
-def _installobsmarkersdiscovery(ui):
- olddisco = exchange.pushdiscoverymapping['obsmarker']
-
- def newdisco(pushop):
- _pushdiscoveryobsmarkers(olddisco, pushop)
- exchange.pushdiscoverymapping['obsmarker'] = newdisco
-
-### Set discovery START
-
-from mercurial import dagutil
-from mercurial import setdiscovery
-
-@eh.addattr(localrepo.localpeer, 'evoext_obshash')
-def local_obshash(peer, nodes):
- return serveronly._obshash(peer._repo, nodes)
-
-@eh.addattr(localrepo.localpeer, 'evoext_obshash1')
-def local_obshash1(peer, nodes):
- return serveronly._obshash(peer._repo, nodes, version=1)
-
-@eh.addattr(wireproto.wirepeer, 'evoext_obshash')
-def peer_obshash(self, nodes):
- d = self._call("evoext_obshash", nodes=wireproto.encodelist(nodes))
- try:
- return wireproto.decodelist(d)
- except ValueError:
- self._abort(error.ResponseError(_("unexpected response:"), d))
-
-@eh.addattr(wireproto.wirepeer, 'evoext_obshash1')
-def peer_obshash1(self, nodes):
- d = self._call("evoext_obshash1", nodes=wireproto.encodelist(nodes))
- try:
- return wireproto.decodelist(d)
- except ValueError:
- self._abort(error.ResponseError(_("unexpected response:"), d))
-
-def findcommonobsmarkers(ui, local, remote, probeset,
- initialsamplesize=100,
- fullsamplesize=200):
- # from discovery
- roundtrips = 0
- cl = local.changelog
- dag = dagutil.revlogdag(cl)
- missing = set()
- common = set()
- undecided = set(probeset)
- totalnb = len(undecided)
- ui.progress(_("comparing with other"), 0, total=totalnb)
- _takefullsample = setdiscovery._takefullsample
- if remote.capable('_evoext_obshash_1'):
- getremotehash = remote.evoext_obshash1
- localhash = serveronly._obsrelsethashtreefm1(local)
- else:
- getremotehash = remote.evoext_obshash
- localhash = serveronly._obsrelsethashtreefm0(local)
-
- while undecided:
-
- ui.note(_("sampling from both directions\n"))
- if len(undecided) < fullsamplesize:
- sample = set(undecided)
- else:
- sample = _takefullsample(dag, undecided, size=fullsamplesize)
-
- roundtrips += 1
- ui.progress(_("comparing with other"), totalnb - len(undecided),
- total=totalnb)
- ui.debug("query %i; still undecided: %i, sample size is: %i\n"
- % (roundtrips, len(undecided), len(sample)))
- # indices between sample and externalized version must match
- sample = list(sample)
- remotehash = getremotehash(dag.externalizeall(sample))
-
- yesno = [localhash[ix][1] == remotehash[si]
- for si, ix in enumerate(sample)]
-
- commoninsample = set(n for i, n in enumerate(sample) if yesno[i])
- common.update(dag.ancestorset(commoninsample, common))
-
- missinginsample = [n for i, n in enumerate(sample) if not yesno[i]]
- missing.update(dag.descendantset(missinginsample, missing))
-
- undecided.difference_update(missing)
- undecided.difference_update(common)
-
- ui.progress(_("comparing with other"), None)
- result = dag.headsetofconnecteds(common)
- ui.debug("%d total queries\n" % roundtrips)
-
- if not result:
- return set([node.nullid])
- return dag.externalizeall(result)
-
+obsexcprg = utility.obsexcprg
_pushkeyescape = getattr(obsolete, '_pushkeyescape', None)
@@ -293,22 +154,6 @@
data = obsfile.read()
serveronly._pushobsmarkers(peer._repo, data)
-def _buildpullobsmarkersboundaries(pullop):
- """small funtion returning the argument for pull markers call
- may to contains 'heads' and 'common'. skip the key for None.
-
- Its a separed functio to play around with strategy for that."""
- repo = pullop.repo
- remote = pullop.remote
- unfi = repo.unfiltered()
- revs = unfi.revs('::(%ln - null)', pullop.common)
- common = [node.nullid]
- if remote.capable('_evoext_obshash_0'):
- obsexcmsg(repo.ui, "looking for common markers in %i nodes\n"
- % len(revs))
- common = findcommonobsmarkers(repo.ui, repo, remote, revs)
- return {'heads': pullop.pulledsubset, 'common': common}
-
@eh.uisetup
def addgetbundleargs(self):
wireproto.gboptsmap['evo_obscommon'] = 'nodes'
@@ -318,7 +163,7 @@
ret = orig(pullop, kwargs)
if ('obsmarkers' in kwargs and
pullop.remote.capable('_evoext_getbundle_obscommon')):
- boundaries = _buildpullobsmarkersboundaries(pullop)
+ boundaries = obsdiscovery._buildpullobsmarkersboundaries(pullop)
common = boundaries['common']
if common != [node.nullid]:
kwargs['evo_obscommon'] = common
@@ -339,7 +184,7 @@
return None # remote opted out of obsolescence marker exchange
tr = None
ui = pullop.repo.ui
- boundaries = _buildpullobsmarkersboundaries(pullop)
+ boundaries = obsdiscovery._buildpullobsmarkersboundaries(pullop)
if not set(boundaries['heads']) - set(boundaries['common']):
obsexcmsg(ui, "nothing to pull\n")
return None
@@ -393,25 +238,4 @@
def local_pullobsmarkers(self, heads=None, common=None):
return serveronly._getobsmarkersstream(self._repo, heads=heads,
common=common)
-
-@eh.command(
- 'debugobsrelsethashtree',
- [('', 'v0', None, 'hash on marker format "0"'),
- ('', 'v1', None, 'hash on marker format "1" (default)')], _(''))
-def debugobsrelsethashtree(ui, repo, v0=False, v1=False):
- """display Obsolete markers, Relevant Set, Hash Tree
- changeset-node obsrelsethashtree-node
-
- It computed form the "orsht" of its parent and markers
- relevant to the changeset itself."""
- if v0 and v1:
- raise error.Abort('cannot only specify one format')
- elif v0:
- treefunc = serveronly._obsrelsethashtreefm0
- else:
- treefunc = serveronly._obsrelsethashtreefm1
-
- for chg, obs in treefunc(repo):
- ui.status('%s %s\n' % (node.hex(chg), node.hex(obs)))
-
_bestformat = max(obsolete.formats.keys())
--- a/hgext3rd/evolve/utility.py Tue Mar 07 14:19:12 2017 +0100
+++ b/hgext3rd/evolve/utility.py Tue Mar 07 14:29:43 2017 +0100
@@ -12,3 +12,9 @@
message = 'OBSEXC: ' + message
if important or verbose:
ui.status(message)
+
+def obsexcprg(ui, *args, **kwargs):
+ topic = 'obsmarkers exchange'
+ if ui.configbool('experimental', 'verbose-obsolescence-exchange', False):
+ topic = 'OBSEXC'
+ ui.progress(topic, *args, **kwargs)