# HG changeset patch # User Pierre-Yves David # Date 1488893383 -3600 # Node ID f9d65d24b9f9e328184eedcb5f31b2fc6807d651 # Parent f3765c4a352a17155e61b54b1e1392101265fff3 discovery: split discovery related code in 'obsdiscovery' More code splitting for more clarity. diff -r f3765c4a352a -r f9d65d24b9f9 hgext3rd/evolve/obsdiscovery.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hgext3rd/evolve/obsdiscovery.py Tue Mar 07 14:29:43 2017 +0100 @@ -0,0 +1,204 @@ +# Code dedicated to the discovery of obsolescence marker "over the wire" +# +# Copyright 2017 Pierre-Yves David +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +try: + import StringIO as io + StringIO = io.StringIO +except ImportError: + import io + StringIO = io.StringIO + +from mercurial import ( + error, + exchange, + localrepo, + node, + obsolete, + wireproto, +) +from mercurial.i18n import _ + +from . import ( + exthelper, + serveronly, + utility, +) + +eh = exthelper.exthelper() +obsexcmsg = utility.obsexcmsg + + +from mercurial import dagutil +from mercurial import setdiscovery + +@eh.addattr(localrepo.localpeer, 'evoext_obshash') +def local_obshash(peer, nodes): + return serveronly._obshash(peer._repo, nodes) + +@eh.addattr(localrepo.localpeer, 'evoext_obshash1') +def local_obshash1(peer, nodes): + return serveronly._obshash(peer._repo, nodes, version=1) + +@eh.addattr(wireproto.wirepeer, 'evoext_obshash') +def peer_obshash(self, nodes): + d = self._call("evoext_obshash", nodes=wireproto.encodelist(nodes)) + try: + return wireproto.decodelist(d) + except ValueError: + self._abort(error.ResponseError(_("unexpected response:"), d)) + +@eh.addattr(wireproto.wirepeer, 'evoext_obshash1') +def peer_obshash1(self, nodes): + d = self._call("evoext_obshash1", nodes=wireproto.encodelist(nodes)) + try: + return wireproto.decodelist(d) + except ValueError: + self._abort(error.ResponseError(_("unexpected response:"), d)) + +def findcommonobsmarkers(ui, local, remote, probeset, + initialsamplesize=100, + fullsamplesize=200): + # from discovery + roundtrips = 0 + cl = local.changelog + dag = dagutil.revlogdag(cl) + missing = set() + common = set() + undecided = set(probeset) + totalnb = len(undecided) + ui.progress(_("comparing with other"), 0, total=totalnb) + _takefullsample = setdiscovery._takefullsample + if remote.capable('_evoext_obshash_1'): + getremotehash = remote.evoext_obshash1 + localhash = serveronly._obsrelsethashtreefm1(local) + else: + getremotehash = remote.evoext_obshash + localhash = serveronly._obsrelsethashtreefm0(local) + + while undecided: + + ui.note(_("sampling from both directions\n")) + if len(undecided) < fullsamplesize: + sample = set(undecided) + else: + sample = _takefullsample(dag, undecided, size=fullsamplesize) + + roundtrips += 1 + ui.progress(_("comparing with other"), totalnb - len(undecided), + total=totalnb) + ui.debug("query %i; still undecided: %i, sample size is: %i\n" + % (roundtrips, len(undecided), len(sample))) + # indices between sample and externalized version must match + sample = list(sample) + remotehash = getremotehash(dag.externalizeall(sample)) + + yesno = [localhash[ix][1] == remotehash[si] + for si, ix in enumerate(sample)] + + commoninsample = set(n for i, n in enumerate(sample) if yesno[i]) + common.update(dag.ancestorset(commoninsample, common)) + + missinginsample = [n for i, n in enumerate(sample) if not yesno[i]] + missing.update(dag.descendantset(missinginsample, missing)) + + undecided.difference_update(missing) + undecided.difference_update(common) + + ui.progress(_("comparing with other"), None) + result = dag.headsetofconnecteds(common) + ui.debug("%d total queries\n" % roundtrips) + + if not result: + return set([node.nullid]) + return dag.externalizeall(result) + +@eh.wrapfunction(exchange, '_pushdiscoveryobsmarkers') +def _pushdiscoveryobsmarkers(orig, pushop): + if (obsolete.isenabled(pushop.repo, obsolete.exchangeopt) + and pushop.repo.obsstore + and 'obsolete' in pushop.remote.listkeys('namespaces')): + repo = pushop.repo + obsexcmsg(repo.ui, "computing relevant nodes\n") + revs = list(repo.revs('::%ln', pushop.futureheads)) + unfi = repo.unfiltered() + cl = unfi.changelog + if not pushop.remote.capable('_evoext_obshash_0'): + # do not trust core yet + # return orig(pushop) + nodes = [cl.node(r) for r in revs] + if nodes: + obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n" + % len(nodes)) + pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes) + else: + obsexcmsg(repo.ui, "markers already in sync\n") + pushop.outobsmarkers = [] + pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes) + return + + common = [] + obsexcmsg(repo.ui, "looking for common markers in %i nodes\n" + % len(revs)) + commonrevs = list(unfi.revs('::%ln', pushop.outgoing.commonheads)) + common = findcommonobsmarkers(pushop.ui, unfi, pushop.remote, + commonrevs) + + revs = list(unfi.revs('%ld - (::%ln)', revs, common)) + nodes = [cl.node(r) for r in revs] + if nodes: + obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n" + % len(nodes)) + pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes) + else: + obsexcmsg(repo.ui, "markers already in sync\n") + pushop.outobsmarkers = [] + +@eh.extsetup +def _installobsmarkersdiscovery(ui): + olddisco = exchange.pushdiscoverymapping['obsmarker'] + + def newdisco(pushop): + _pushdiscoveryobsmarkers(olddisco, pushop) + exchange.pushdiscoverymapping['obsmarker'] = newdisco + +def _buildpullobsmarkersboundaries(pullop): + """small funtion returning the argument for pull markers call + may to contains 'heads' and 'common'. skip the key for None. + + Its a separed functio to play around with strategy for that.""" + repo = pullop.repo + remote = pullop.remote + unfi = repo.unfiltered() + revs = unfi.revs('::(%ln - null)', pullop.common) + common = [node.nullid] + if remote.capable('_evoext_obshash_0'): + obsexcmsg(repo.ui, "looking for common markers in %i nodes\n" + % len(revs)) + common = findcommonobsmarkers(repo.ui, repo, remote, revs) + return {'heads': pullop.pulledsubset, 'common': common} + +@eh.command( + 'debugobsrelsethashtree', + [('', 'v0', None, 'hash on marker format "0"'), + ('', 'v1', None, 'hash on marker format "1" (default)')], _('')) +def debugobsrelsethashtree(ui, repo, v0=False, v1=False): + """display Obsolete markers, Relevant Set, Hash Tree + changeset-node obsrelsethashtree-node + + It computed form the "orsht" of its parent and markers + relevant to the changeset itself.""" + if v0 and v1: + raise error.Abort('cannot only specify one format') + elif v0: + treefunc = serveronly._obsrelsethashtreefm0 + else: + treefunc = serveronly._obsrelsethashtreefm1 + + for chg, obs in treefunc(repo): + ui.status('%s %s\n' % (node.hex(chg), node.hex(obs))) diff -r f3765c4a352a -r f9d65d24b9f9 hgext3rd/evolve/obsexchange.py --- a/hgext3rd/evolve/obsexchange.py Tue Mar 07 14:19:12 2017 +0100 +++ b/hgext3rd/evolve/obsexchange.py Tue Mar 07 14:29:43 2017 +0100 @@ -33,152 +33,13 @@ exthelper, serveronly, utility, + obsdiscovery, ) eh = exthelper.exthelper() +eh.merge(obsdiscovery.eh) obsexcmsg = utility.obsexcmsg - -def obsexcprg(ui, *args, **kwargs): - topic = 'obsmarkers exchange' - if ui.configbool('experimental', 'verbose-obsolescence-exchange', False): - topic = 'OBSEXC' - ui.progress(topic, *args, **kwargs) - -@eh.wrapfunction(exchange, '_pushdiscoveryobsmarkers') -def _pushdiscoveryobsmarkers(orig, pushop): - if (obsolete.isenabled(pushop.repo, obsolete.exchangeopt) - and pushop.repo.obsstore - and 'obsolete' in pushop.remote.listkeys('namespaces')): - repo = pushop.repo - obsexcmsg(repo.ui, "computing relevant nodes\n") - revs = list(repo.revs('::%ln', pushop.futureheads)) - unfi = repo.unfiltered() - cl = unfi.changelog - if not pushop.remote.capable('_evoext_obshash_0'): - # do not trust core yet - # return orig(pushop) - nodes = [cl.node(r) for r in revs] - if nodes: - obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n" - % len(nodes)) - pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes) - else: - obsexcmsg(repo.ui, "markers already in sync\n") - pushop.outobsmarkers = [] - pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes) - return - - common = [] - obsexcmsg(repo.ui, "looking for common markers in %i nodes\n" - % len(revs)) - commonrevs = list(unfi.revs('::%ln', pushop.outgoing.commonheads)) - common = findcommonobsmarkers(pushop.ui, unfi, pushop.remote, - commonrevs) - - revs = list(unfi.revs('%ld - (::%ln)', revs, common)) - nodes = [cl.node(r) for r in revs] - if nodes: - obsexcmsg(repo.ui, "computing markers relevant to %i nodes\n" - % len(nodes)) - pushop.outobsmarkers = repo.obsstore.relevantmarkers(nodes) - else: - obsexcmsg(repo.ui, "markers already in sync\n") - pushop.outobsmarkers = [] - -@eh.extsetup -def _installobsmarkersdiscovery(ui): - olddisco = exchange.pushdiscoverymapping['obsmarker'] - - def newdisco(pushop): - _pushdiscoveryobsmarkers(olddisco, pushop) - exchange.pushdiscoverymapping['obsmarker'] = newdisco - -### Set discovery START - -from mercurial import dagutil -from mercurial import setdiscovery - -@eh.addattr(localrepo.localpeer, 'evoext_obshash') -def local_obshash(peer, nodes): - return serveronly._obshash(peer._repo, nodes) - -@eh.addattr(localrepo.localpeer, 'evoext_obshash1') -def local_obshash1(peer, nodes): - return serveronly._obshash(peer._repo, nodes, version=1) - -@eh.addattr(wireproto.wirepeer, 'evoext_obshash') -def peer_obshash(self, nodes): - d = self._call("evoext_obshash", nodes=wireproto.encodelist(nodes)) - try: - return wireproto.decodelist(d) - except ValueError: - self._abort(error.ResponseError(_("unexpected response:"), d)) - -@eh.addattr(wireproto.wirepeer, 'evoext_obshash1') -def peer_obshash1(self, nodes): - d = self._call("evoext_obshash1", nodes=wireproto.encodelist(nodes)) - try: - return wireproto.decodelist(d) - except ValueError: - self._abort(error.ResponseError(_("unexpected response:"), d)) - -def findcommonobsmarkers(ui, local, remote, probeset, - initialsamplesize=100, - fullsamplesize=200): - # from discovery - roundtrips = 0 - cl = local.changelog - dag = dagutil.revlogdag(cl) - missing = set() - common = set() - undecided = set(probeset) - totalnb = len(undecided) - ui.progress(_("comparing with other"), 0, total=totalnb) - _takefullsample = setdiscovery._takefullsample - if remote.capable('_evoext_obshash_1'): - getremotehash = remote.evoext_obshash1 - localhash = serveronly._obsrelsethashtreefm1(local) - else: - getremotehash = remote.evoext_obshash - localhash = serveronly._obsrelsethashtreefm0(local) - - while undecided: - - ui.note(_("sampling from both directions\n")) - if len(undecided) < fullsamplesize: - sample = set(undecided) - else: - sample = _takefullsample(dag, undecided, size=fullsamplesize) - - roundtrips += 1 - ui.progress(_("comparing with other"), totalnb - len(undecided), - total=totalnb) - ui.debug("query %i; still undecided: %i, sample size is: %i\n" - % (roundtrips, len(undecided), len(sample))) - # indices between sample and externalized version must match - sample = list(sample) - remotehash = getremotehash(dag.externalizeall(sample)) - - yesno = [localhash[ix][1] == remotehash[si] - for si, ix in enumerate(sample)] - - commoninsample = set(n for i, n in enumerate(sample) if yesno[i]) - common.update(dag.ancestorset(commoninsample, common)) - - missinginsample = [n for i, n in enumerate(sample) if not yesno[i]] - missing.update(dag.descendantset(missinginsample, missing)) - - undecided.difference_update(missing) - undecided.difference_update(common) - - ui.progress(_("comparing with other"), None) - result = dag.headsetofconnecteds(common) - ui.debug("%d total queries\n" % roundtrips) - - if not result: - return set([node.nullid]) - return dag.externalizeall(result) - +obsexcprg = utility.obsexcprg _pushkeyescape = getattr(obsolete, '_pushkeyescape', None) @@ -293,22 +154,6 @@ data = obsfile.read() serveronly._pushobsmarkers(peer._repo, data) -def _buildpullobsmarkersboundaries(pullop): - """small funtion returning the argument for pull markers call - may to contains 'heads' and 'common'. skip the key for None. - - Its a separed functio to play around with strategy for that.""" - repo = pullop.repo - remote = pullop.remote - unfi = repo.unfiltered() - revs = unfi.revs('::(%ln - null)', pullop.common) - common = [node.nullid] - if remote.capable('_evoext_obshash_0'): - obsexcmsg(repo.ui, "looking for common markers in %i nodes\n" - % len(revs)) - common = findcommonobsmarkers(repo.ui, repo, remote, revs) - return {'heads': pullop.pulledsubset, 'common': common} - @eh.uisetup def addgetbundleargs(self): wireproto.gboptsmap['evo_obscommon'] = 'nodes' @@ -318,7 +163,7 @@ ret = orig(pullop, kwargs) if ('obsmarkers' in kwargs and pullop.remote.capable('_evoext_getbundle_obscommon')): - boundaries = _buildpullobsmarkersboundaries(pullop) + boundaries = obsdiscovery._buildpullobsmarkersboundaries(pullop) common = boundaries['common'] if common != [node.nullid]: kwargs['evo_obscommon'] = common @@ -339,7 +184,7 @@ return None # remote opted out of obsolescence marker exchange tr = None ui = pullop.repo.ui - boundaries = _buildpullobsmarkersboundaries(pullop) + boundaries = obsdiscovery._buildpullobsmarkersboundaries(pullop) if not set(boundaries['heads']) - set(boundaries['common']): obsexcmsg(ui, "nothing to pull\n") return None @@ -393,25 +238,4 @@ def local_pullobsmarkers(self, heads=None, common=None): return serveronly._getobsmarkersstream(self._repo, heads=heads, common=common) - -@eh.command( - 'debugobsrelsethashtree', - [('', 'v0', None, 'hash on marker format "0"'), - ('', 'v1', None, 'hash on marker format "1" (default)')], _('')) -def debugobsrelsethashtree(ui, repo, v0=False, v1=False): - """display Obsolete markers, Relevant Set, Hash Tree - changeset-node obsrelsethashtree-node - - It computed form the "orsht" of its parent and markers - relevant to the changeset itself.""" - if v0 and v1: - raise error.Abort('cannot only specify one format') - elif v0: - treefunc = serveronly._obsrelsethashtreefm0 - else: - treefunc = serveronly._obsrelsethashtreefm1 - - for chg, obs in treefunc(repo): - ui.status('%s %s\n' % (node.hex(chg), node.hex(obs))) - _bestformat = max(obsolete.formats.keys()) diff -r f3765c4a352a -r f9d65d24b9f9 hgext3rd/evolve/utility.py --- a/hgext3rd/evolve/utility.py Tue Mar 07 14:19:12 2017 +0100 +++ b/hgext3rd/evolve/utility.py Tue Mar 07 14:29:43 2017 +0100 @@ -12,3 +12,9 @@ message = 'OBSEXC: ' + message if important or verbose: ui.status(message) + +def obsexcprg(ui, *args, **kwargs): + topic = 'obsmarkers exchange' + if ui.configbool('experimental', 'verbose-obsolescence-exchange', False): + topic = 'OBSEXC' + ui.progress(topic, *args, **kwargs)