exchange: dispatch all code in 'serveronly' to the appropriate submodule
All code related to exchange is now in either 'evolve.obsexchange' or
'evolve.obsdiscovery', and the 'serveronly' extension is using their 'exthelper'
to set itself up. As a side effect, the 'serveronly' extensions now enable
better exchange when pushing from that server too.
--- a/hgext3rd/evolve/obsdiscovery.py Wed Mar 08 15:35:16 2017 -0800
+++ b/hgext3rd/evolve/obsdiscovery.py Tue Mar 07 15:45:21 2017 +0100
@@ -14,19 +14,22 @@
import io
StringIO = io.StringIO
+import hashlib
+
from mercurial import (
error,
exchange,
+ extensions,
localrepo,
node,
obsolete,
wireproto,
)
+from mercurial.hgweb import hgweb_mod
from mercurial.i18n import _
from . import (
exthelper,
- serveronly,
utility,
)
@@ -39,11 +42,11 @@
@eh.addattr(localrepo.localpeer, 'evoext_obshash')
def local_obshash(peer, nodes):
- return serveronly._obshash(peer._repo, nodes)
+ return _obshash(peer._repo, nodes)
@eh.addattr(localrepo.localpeer, 'evoext_obshash1')
def local_obshash1(peer, nodes):
- return serveronly._obshash(peer._repo, nodes, version=1)
+ return _obshash(peer._repo, nodes, version=1)
@eh.addattr(wireproto.wirepeer, 'evoext_obshash')
def peer_obshash(self, nodes):
@@ -76,10 +79,10 @@
_takefullsample = setdiscovery._takefullsample
if remote.capable('_evoext_obshash_1'):
getremotehash = remote.evoext_obshash1
- localhash = serveronly._obsrelsethashtreefm1(local)
+ localhash = _obsrelsethashtreefm1(local)
else:
getremotehash = remote.evoext_obshash
- localhash = serveronly._obsrelsethashtreefm0(local)
+ localhash = _obsrelsethashtreefm0(local)
while undecided:
@@ -196,9 +199,98 @@
if v0 and v1:
raise error.Abort('cannot only specify one format')
elif v0:
- treefunc = serveronly._obsrelsethashtreefm0
+ treefunc = _obsrelsethashtreefm0
else:
- treefunc = serveronly._obsrelsethashtreefm1
+ treefunc = _obsrelsethashtreefm1
for chg, obs in treefunc(repo):
ui.status('%s %s\n' % (node.hex(chg), node.hex(obs)))
+
+def _obsrelsethashtreefm0(repo):
+ return _obsrelsethashtree(repo, obsolete._fm0encodeonemarker)
+
+def _obsrelsethashtreefm1(repo):
+ return _obsrelsethashtree(repo, obsolete._fm1encodeonemarker)
+
+def _obsrelsethashtree(repo, encodeonemarker):
+ cache = []
+ unfi = repo.unfiltered()
+ markercache = {}
+ repo.ui.progress(_("preparing locally"), 0, total=len(unfi))
+ for i in unfi:
+ ctx = unfi[i]
+ entry = 0
+ sha = hashlib.sha1()
+ # add data from p1
+ for p in ctx.parents():
+ p = p.rev()
+ if p < 0:
+ p = node.nullid
+ else:
+ p = cache[p][1]
+ if p != node.nullid:
+ entry += 1
+ sha.update(p)
+ tmarkers = repo.obsstore.relevantmarkers([ctx.node()])
+ if tmarkers:
+ bmarkers = []
+ for m in tmarkers:
+ if m not in markercache:
+ markercache[m] = encodeonemarker(m)
+ bmarkers.append(markercache[m])
+ bmarkers.sort()
+ for m in bmarkers:
+ entry += 1
+ sha.update(m)
+ if entry:
+ cache.append((ctx.node(), sha.digest()))
+ else:
+ cache.append((ctx.node(), node.nullid))
+ repo.ui.progress(_("preparing locally"), i, total=len(unfi))
+ repo.ui.progress(_("preparing locally"), None)
+ return cache
+
+def _obshash(repo, nodes, version=0):
+ if version == 0:
+ hashs = _obsrelsethashtreefm0(repo)
+ elif version == 1:
+ hashs = _obsrelsethashtreefm1(repo)
+ else:
+ assert False
+ nm = repo.changelog.nodemap
+ revs = [nm.get(n) for n in nodes]
+ return [r is None and node.nullid or hashs[r][1] for r in revs]
+
+def srv_obshash(repo, proto, nodes):
+ return wireproto.encodelist(_obshash(repo, wireproto.decodelist(nodes)))
+
+def srv_obshash1(repo, proto, nodes):
+ return wireproto.encodelist(_obshash(repo, wireproto.decodelist(nodes),
+ version=1))
+
+def _obshash_capabilities(orig, repo, proto):
+ """wrapper to advertise new capability"""
+ caps = orig(repo, proto)
+ advertise = repo.ui.configbool('__temporary__', 'advertiseobsolete', True)
+ if obsolete.isenabled(repo, obsolete.exchangeopt) and advertise:
+ caps = caps.split()
+ caps.append('_evoext_obshash_0')
+ caps.append('_evoext_obshash_1')
+ caps.sort()
+ caps = ' '.join(caps)
+ return caps
+
+@eh.extsetup
+def obshash_extsetup(ui):
+ hgweb_mod.perms['evoext_obshash'] = 'pull'
+ hgweb_mod.perms['evoext_obshash1'] = 'pull'
+
+ wireproto.commands['evoext_obshash'] = (srv_obshash, 'nodes')
+ wireproto.commands['evoext_obshash1'] = (srv_obshash1, 'nodes')
+ extensions.wrapfunction(wireproto, 'capabilities', _obshash_capabilities)
+ # wrap command content
+ oldcap, args = wireproto.commands['capabilities']
+
+ def newcap(repo, proto):
+ return _obshash_capabilities(oldcap, repo, proto)
+ wireproto.commands['capabilities'] = (newcap, args)
--- a/hgext3rd/evolve/obsexchange.py Wed Mar 08 15:35:16 2017 -0800
+++ b/hgext3rd/evolve/obsexchange.py Tue Mar 07 15:45:21 2017 +0100
@@ -20,13 +20,17 @@
from mercurial import (
error,
exchange,
+ extensions,
httppeer,
localrepo,
+ lock as lockmod,
node,
obsolete,
+ pushkey,
util,
wireproto,
)
+from mercurial.hgweb import hgweb_mod
from mercurial.i18n import _
from . import (
@@ -41,10 +45,33 @@
obsexcmsg = utility.obsexcmsg
obsexcprg = utility.obsexcprg
-### adds support for the 'evo_obscommon' argument to getbundle
-#
-# This argument use the data recovered from the discovery to request only a
-# subpart of the obsolete subtree.
+
+_bestformat = max(obsolete.formats.keys())
+
+##################################
+### Control evolve advertising ###
+##################################
+
+# This is useful to not enable evolution on all repositories on an install
+
+def _nslist(orig, repo):
+ rep = orig(repo)
+ if not repo.ui.configbool('__temporary__', 'advertiseobsolete', True):
+ rep.pop('obsolete')
+ return rep
+
+@eh.extsetup
+def extsetup_advertise(ui):
+ extensions.wrapfunction(pushkey, '_nslist', _nslist)
+ pushkey._namespaces['namespaces'] = (lambda *x: False, pushkey._nslist)
+
+#####################################################
+### Support for subset specification in getbundle ###
+#####################################################
+
+# Adds support for the 'evo_obscommon' argument to getbundle This argument use
+# the data recovered from the discovery to request only a subpart of the
+# obsolete subtree.
@eh.uisetup
def addgetbundleargs(self):
@@ -61,6 +88,128 @@
kwargs['evo_obscommon'] = common
return ret
+def _getbundleobsmarkerpart(orig, bundler, repo, source, **kwargs):
+ if 'evo_obscommon' not in kwargs:
+ return orig(bundler, repo, source, **kwargs)
+
+ heads = kwargs.get('heads')
+ if kwargs.get('obsmarkers', False):
+ if heads is None:
+ heads = repo.heads()
+ obscommon = kwargs.get('evo_obscommon', ())
+ assert obscommon
+ obsset = repo.unfiltered().set('::%ln - ::%ln', heads, obscommon)
+ subset = [c.node() for c in obsset]
+ markers = repo.obsstore.relevantmarkers(subset)
+ exchange.buildobsmarkerspart(bundler, markers)
+
+# manual wrap up in extsetup because of the wireproto.commands mapping
+def _obscommon_capabilities(orig, repo, proto):
+ """wrapper to advertise new capability"""
+ caps = orig(repo, proto)
+ advertise = repo.ui.configbool('__temporary__', 'advertiseobsolete', True)
+ if obsolete.isenabled(repo, obsolete.exchangeopt) and advertise:
+ caps = caps.split()
+ caps.append('_evoext_getbundle_obscommon')
+ caps.sort()
+ caps = ' '.join(caps)
+ return caps
+
+@eh.extsetup
+def extsetup_obscommon(ui):
+ wireproto.gboptsmap['evo_obscommon'] = 'nodes'
+
+ # wrap module content
+ origfunc = exchange.getbundle2partsmapping['obsmarkers']
+
+ def newfunc(*args, **kwargs):
+ return _getbundleobsmarkerpart(origfunc, *args, **kwargs)
+ exchange.getbundle2partsmapping['obsmarkers'] = newfunc
+
+ extensions.wrapfunction(wireproto, 'capabilities', _obscommon_capabilities)
+ # wrap command content
+ oldcap, args = wireproto.commands['capabilities']
+
+ def newcap(repo, proto):
+ return _obscommon_capabilities(oldcap, repo, proto)
+ wireproto.commands['capabilities'] = (newcap, args)
+
+def _pushobsmarkers(repo, data):
+ tr = lock = None
+ try:
+ lock = repo.lock()
+ tr = repo.transaction('pushkey: obsolete markers')
+ new = repo.obsstore.mergemarkers(tr, data)
+ if new is not None:
+ obsexcmsg(repo.ui, "%i obsolescence markers added\n" % new, True)
+ tr.close()
+ finally:
+ lockmod.release(tr, lock)
+ repo.hook('evolve_pushobsmarkers')
+
+def srv_pushobsmarkers(repo, proto):
+ """wireprotocol command"""
+ fp = StringIO()
+ proto.redirect()
+ proto.getfile(fp)
+ data = fp.getvalue()
+ fp.close()
+ _pushobsmarkers(repo, data)
+ return wireproto.pushres(0)
+
+def _getobsmarkersstream(repo, heads=None, common=None):
+ """Get a binary stream for all markers relevant to `::<heads> - ::<common>`
+ """
+ revset = ''
+ args = []
+ repo = repo.unfiltered()
+ if heads is None:
+ revset = 'all()'
+ elif heads:
+ revset += "(::%ln)"
+ args.append(heads)
+ else:
+ assert False, 'pulling no heads?'
+ if common:
+ revset += ' - (::%ln)'
+ args.append(common)
+ nodes = [c.node() for c in repo.set(revset, *args)]
+ markers = repo.obsstore.relevantmarkers(nodes)
+ obsdata = StringIO()
+ for chunk in obsolete.encodemarkers(markers, True):
+ obsdata.write(chunk)
+ obsdata.seek(0)
+ return obsdata
+
+# The wireproto.streamres API changed, handling chunking and compression
+# directly. Handle either case.
+if util.safehasattr(wireproto.abstractserverproto, 'groupchunks'):
+ # We need to handle chunking and compression directly
+ def streamres(d, proto):
+ return wireproto.streamres(proto.groupchunks(d))
+else:
+ # Leave chunking and compression to streamres
+ def streamres(d, proto):
+ return wireproto.streamres(reader=d, v1compressible=True)
+
+def srv_pullobsmarkers(repo, proto, others):
+ """serves a binary stream of markers.
+
+ Serves relevant to changeset between heads and common. The stream is prefix
+ by a -string- representation of an integer. This integer is the size of the
+ stream."""
+ opts = wireproto.options('', ['heads', 'common'], others)
+ for k, v in opts.iteritems():
+ if k in ('heads', 'common'):
+ opts[k] = wireproto.decodelist(v)
+ obsdata = _getobsmarkersstream(repo, **opts)
+ finaldata = StringIO()
+ obsdata = obsdata.getvalue()
+ finaldata.write('%20i' % len(obsdata))
+ finaldata.write(obsdata)
+ finaldata.seek(0)
+ return streamres(finaldata, proto)
+
###############################################
### Support for old legacy exchange methods ###
###############################################
@@ -119,6 +268,7 @@
True)
remote.evoext_pushobsmarkers_0(obsdata)
obsexcprg(repo.ui, None)
+
else:
# XXX core could be able do the same things but without the debug
# and progress output.
@@ -206,6 +356,8 @@
if 'obsmarkers' in pullop.stepsdone:
return None
wirepull = pullop.remote.capable('_evoext_pullobsmarkers_0')
+ if 'obsolete' not in pullop.remote.listkeys('namespaces'):
+ return None # remote opted out of obsolescence marker exchange
if not wirepull:
return orig(pullop)
tr = None
@@ -268,4 +420,33 @@
def local_pullobsmarkers(self, heads=None, common=None):
return serveronly._getobsmarkersstream(self._repo, heads=heads,
common=common)
-_bestformat = max(obsolete.formats.keys())
+
+def _legacypush_capabilities(orig, repo, proto):
+ """wrapper to advertise new capability"""
+ caps = orig(repo, proto)
+ advertise = repo.ui.configbool('__temporary__', 'advertiseobsolete', True)
+ if obsolete.isenabled(repo, obsolete.exchangeopt) and advertise:
+ caps = caps.split()
+ caps.append('_evoext_pushobsmarkers_0')
+ caps.append('_evoext_pullobsmarkers_0')
+ caps.sort()
+ caps = ' '.join(caps)
+ return caps
+
+@eh.extsetup
+def extsetup(ui):
+ # XXX old bundle version, still supported ???
+ localrepo.moderncaps.add('_evoext_b2x_obsmarkers_0')
+ # legacy standalone method
+ hgweb_mod.perms['evoext_pushobsmarkers_0'] = 'push'
+ hgweb_mod.perms['evoext_pullobsmarkers_0'] = 'pull'
+ wireproto.commands['evoext_pushobsmarkers_0'] = (srv_pushobsmarkers, '')
+ wireproto.commands['evoext_pullobsmarkers_0'] = (srv_pullobsmarkers, '*')
+
+ extensions.wrapfunction(wireproto, 'capabilities', _legacypush_capabilities)
+ # wrap command content
+ oldcap, args = wireproto.commands['capabilities']
+
+ def newcap(repo, proto):
+ return _legacypush_capabilities(oldcap, repo, proto)
+ wireproto.commands['capabilities'] = (newcap, args)
--- a/hgext3rd/evolve/serveronly.py Wed Mar 08 15:35:16 2017 -0800
+++ b/hgext3rd/evolve/serveronly.py Tue Mar 07 15:45:21 2017 +0100
@@ -10,43 +10,14 @@
from __future__ import absolute_import
-import hashlib
-import struct
-from cStringIO import StringIO
import sys
import os
-from mercurial import (
- error,
- exchange,
- extensions,
- localrepo,
- lock as lockmod,
- node,
- obsolete,
- util,
- wireproto
-)
-from mercurial.hgweb import hgweb_mod
-from mercurial.i18n import _
-_pack = struct.pack
-
-gboptslist = gboptsmap = None
-try:
- gboptslist = getattr(wireproto, 'gboptslist', None)
- gboptsmap = getattr(wireproto, 'gboptsmap', None)
-except (ImportError, AttributeError):
- raise error.Abort('Your Mercurial is too old for this version of Evolve\n'
- 'requires version 3.0.1 or above')
-
-# Start of simple4server specific content
-
-from mercurial import pushkey
-
try:
from . import (
+ exthelper,
metadata,
- utility,
+ obsexchange,
)
except ValueError as exc:
if exc.message != 'Attempted relative import in non-package':
@@ -54,223 +25,24 @@
# extension imported using direct path
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from evolve import (
+ exthelper,
metadata,
- utility,
+ obsexchange,
)
-
__version__ = metadata.__version__
testedwith = metadata.testedwith
minimumhgversion = metadata.minimumhgversion
buglink = metadata.buglink
-obsexcmsg = utility.obsexcmsg
-
-# specific content also include the wrapping int extsetup
-def _nslist(orig, repo):
- rep = orig(repo)
- if not repo.ui.configbool('__temporary__', 'advertiseobsolete', True):
- rep.pop('obsolete')
- return rep
-
-# End of simple4server specific content
-
-
-def _pushobsmarkers(repo, data):
- tr = lock = None
- try:
- lock = repo.lock()
- tr = repo.transaction('pushkey: obsolete markers')
- new = repo.obsstore.mergemarkers(tr, data)
- if new is not None:
- obsexcmsg(repo.ui, "%i obsolescence markers added\n" % new, True)
- tr.close()
- finally:
- lockmod.release(tr, lock)
- repo.hook('evolve_pushobsmarkers')
-
-def srv_pushobsmarkers(repo, proto):
- """wireprotocol command"""
- fp = StringIO()
- proto.redirect()
- proto.getfile(fp)
- data = fp.getvalue()
- fp.close()
- _pushobsmarkers(repo, data)
- return wireproto.pushres(0)
-
-def _getobsmarkersstream(repo, heads=None, common=None):
- """Get a binary stream for all markers relevant to `::<heads> - ::<common>`
- """
- revset = ''
- args = []
- repo = repo.unfiltered()
- if heads is None:
- revset = 'all()'
- elif heads:
- revset += "(::%ln)"
- args.append(heads)
- else:
- assert False, 'pulling no heads?'
- if common:
- revset += ' - (::%ln)'
- args.append(common)
- nodes = [c.node() for c in repo.set(revset, *args)]
- markers = repo.obsstore.relevantmarkers(nodes)
- obsdata = StringIO()
- for chunk in obsolete.encodemarkers(markers, True):
- obsdata.write(chunk)
- obsdata.seek(0)
- return obsdata
-
-# The wireproto.streamres API changed, handling chunking and compression
-# directly. Handle either case.
-if util.safehasattr(wireproto.abstractserverproto, 'groupchunks'):
- # We need to handle chunking and compression directly
- def streamres(d, proto):
- return wireproto.streamres(proto.groupchunks(d))
-else:
- # Leave chunking and compression to streamres
- def streamres(d, proto):
- return wireproto.streamres(reader=d, v1compressible=True)
-
-def srv_pullobsmarkers(repo, proto, others):
- """serves a binary stream of markers.
-
- Serves relevant to changeset between heads and common. The stream is prefix
- by a -string- representation of an integer. This integer is the size of the
- stream."""
- opts = wireproto.options('', ['heads', 'common'], others)
- for k, v in opts.iteritems():
- if k in ('heads', 'common'):
- opts[k] = wireproto.decodelist(v)
- obsdata = _getobsmarkersstream(repo, **opts)
- finaldata = StringIO()
- obsdata = obsdata.getvalue()
- finaldata.write('%20i' % len(obsdata))
- finaldata.write(obsdata)
- finaldata.seek(0)
- return streamres(finaldata, proto)
-
-def _obsrelsethashtreefm0(repo):
- return _obsrelsethashtree(repo, obsolete._fm0encodeonemarker)
-
-def _obsrelsethashtreefm1(repo):
- return _obsrelsethashtree(repo, obsolete._fm1encodeonemarker)
+eh = exthelper.exthelper()
+eh.merge(obsexchange.eh)
+uisetup = eh.final_uisetup
+extsetup = eh.final_extsetup
+reposetup = eh.final_reposetup
+cmdtable = eh.cmdtable
-def _obsrelsethashtree(repo, encodeonemarker):
- cache = []
- unfi = repo.unfiltered()
- markercache = {}
- repo.ui.progress(_("preparing locally"), 0, total=len(unfi))
- for i in unfi:
- ctx = unfi[i]
- entry = 0
- sha = hashlib.sha1()
- # add data from p1
- for p in ctx.parents():
- p = p.rev()
- if p < 0:
- p = node.nullid
- else:
- p = cache[p][1]
- if p != node.nullid:
- entry += 1
- sha.update(p)
- tmarkers = repo.obsstore.relevantmarkers([ctx.node()])
- if tmarkers:
- bmarkers = []
- for m in tmarkers:
- if m not in markercache:
- markercache[m] = encodeonemarker(m)
- bmarkers.append(markercache[m])
- bmarkers.sort()
- for m in bmarkers:
- entry += 1
- sha.update(m)
- if entry:
- cache.append((ctx.node(), sha.digest()))
- else:
- cache.append((ctx.node(), node.nullid))
- repo.ui.progress(_("preparing locally"), i, total=len(unfi))
- repo.ui.progress(_("preparing locally"), None)
- return cache
-
-def _obshash(repo, nodes, version=0):
- if version == 0:
- hashs = _obsrelsethashtreefm0(repo)
- elif version == 1:
- hashs = _obsrelsethashtreefm1(repo)
- else:
- assert False
- nm = repo.changelog.nodemap
- revs = [nm.get(n) for n in nodes]
- return [r is None and node.nullid or hashs[r][1] for r in revs]
-
-def srv_obshash(repo, proto, nodes):
- return wireproto.encodelist(_obshash(repo, wireproto.decodelist(nodes)))
-
-def srv_obshash1(repo, proto, nodes):
- return wireproto.encodelist(_obshash(repo, wireproto.decodelist(nodes),
- version=1))
-
-def capabilities(orig, repo, proto):
- """wrapper to advertise new capability"""
- caps = orig(repo, proto)
- advertise = repo.ui.configbool('__temporary__', 'advertiseobsolete', True)
- if obsolete.isenabled(repo, obsolete.exchangeopt) and advertise:
- caps = caps.split()
- caps.append('_evoext_pushobsmarkers_0')
- caps.append('_evoext_pullobsmarkers_0')
- caps.append('_evoext_obshash_0')
- caps.append('_evoext_obshash_1')
- caps.append('_evoext_getbundle_obscommon')
- caps.sort()
- return ' '.join(caps)
-
-def _getbundleobsmarkerpart(orig, bundler, repo, source, **kwargs):
- if 'evo_obscommon' not in kwargs:
- return orig(bundler, repo, source, **kwargs)
-
- heads = kwargs.get('heads')
- if kwargs.get('obsmarkers', False):
- if heads is None:
- heads = repo.heads()
- obscommon = kwargs.get('evo_obscommon', ())
- assert obscommon
- obsset = repo.unfiltered().set('::%ln - ::%ln', heads, obscommon)
- subset = [c.node() for c in obsset]
- markers = repo.obsstore.relevantmarkers(subset)
- exchange.buildobsmarkerspart(bundler, markers)
-
-def extsetup(ui):
- localrepo.moderncaps.add('_evoext_b2x_obsmarkers_0')
- gboptsmap['evo_obscommon'] = 'nodes'
- hgweb_mod.perms['evoext_pushobsmarkers_0'] = 'push'
- hgweb_mod.perms['evoext_pullobsmarkers_0'] = 'pull'
- hgweb_mod.perms['evoext_obshash'] = 'pull'
- hgweb_mod.perms['evoext_obshash1'] = 'pull'
- wireproto.commands['evoext_pushobsmarkers_0'] = (srv_pushobsmarkers, '')
- wireproto.commands['evoext_pullobsmarkers_0'] = (srv_pullobsmarkers, '*')
- # wrap module content
- origfunc = exchange.getbundle2partsmapping['obsmarkers']
-
- def newfunc(*args, **kwargs):
- return _getbundleobsmarkerpart(origfunc, *args, **kwargs)
- exchange.getbundle2partsmapping['obsmarkers'] = newfunc
- extensions.wrapfunction(wireproto, 'capabilities', capabilities)
- # wrap command content
- oldcap, args = wireproto.commands['capabilities']
-
- def newcap(repo, proto):
- return capabilities(oldcap, repo, proto)
- wireproto.commands['capabilities'] = (newcap, args)
- wireproto.commands['evoext_obshash'] = (srv_obshash, 'nodes')
- wireproto.commands['evoext_obshash1'] = (srv_obshash1, 'nodes')
- # specific simple4server content
- extensions.wrapfunction(pushkey, '_nslist', _nslist)
- pushkey._namespaces['namespaces'] = (lambda *x: False, pushkey._nslist)
-
+@eh.reposetup
def reposetup(ui, repo):
evolveopts = ui.configlist('experimental', 'evolution')
if not evolveopts: