evolve: comparison hgext/obsolete.py

equal deleted inserted replaced

-:adeebf3b9ac3
+:aab826129142
 from mercurial.i18n import _
 import base64
 import json
+import struct
+from mercurial import util, base85
+_pack = struct.pack
+_unpack = struct.unpack
 from mercurial import util
 from mercurial import context
 from mercurial import revset
 from mercurial import scmutil
 def _precursors(repo, s):
 """Precursor of a changeset"""
 cs = set()
 nm = repo.changelog.nodemap
-markerbysubj = repo.obsoletestore.subjects
+markerbysubj = repo.obsstore.successors
 for r in s:
 for p in markerbysubj.get(repo[r].node(), ()):
-pr = nm.get(p['object'])
+pr = nm.get(p[0])
 if pr is not None:
 cs.add(pr)
 return cs
 def revsetprecursors(repo, subset, x):
 def _allprecursors(repo, s):  # XXX we need a better naming
 """transitive precursors of a subset"""
 toproceed = [repo[r].node() for r in s]
 seen = set()
-allsubjects = repo.obsoletestore.subjects
+allsubjects = repo.obsstore.successors
 while toproceed:
 nc = toproceed.pop()
 for mark in allsubjects.get(nc, ()):
-np = mark['object']
+np = mark[0]
 if np not in seen:
 seen.add(np)
 toproceed.append(np)
 nm = repo.changelog.nodemap
 cs = set()
 def _successors(repo, s):
 """Successors of a changeset"""
 cs = set()
 nm = repo.changelog.nodemap
-markerbyobj = repo.obsoletestore.objects
+markerbyobj = repo.obsstore.precursors
 for r in s:
 for p in markerbyobj.get(repo[r].node(), ()):
-for sub in p['subjects']:
+for sub in p[1]:
 sr = nm.get(sub)
 if sr is not None:
 cs.add(sr)
 return cs
 def _allsuccessors(repo, s):  # XXX we need a better naming
 """transitive successors of a subset"""
 toproceed = [repo[r].node() for r in s]
 seen = set()
-allobjects = repo.obsoletestore.objects
+allobjects = repo.obsstore.precursors
 while toproceed:
 nc = toproceed.pop()
 for mark in allobjects.get(nc, ()):
-for sub in mark['subjects']:
+for sub in mark[1]:
 if sub not in seen:
 seen.add(sub)
 toproceed.append(sub)
 nm = repo.changelog.nodemap
 cs = set()
 pass  # rebase not found
 # Pushkey mechanism for mutable
 #########################################
-def pushobsolete(repo, key, old, raw):
+def listmarkers(repo):
-"""push obsolete relation through pushkey"""
+"""List markers over pushkey"""
-assert key == "markers"
+if not repo.obsstore:
-l = repo.lock()
+return {}
+data = repo.obsstore._writemarkers()
+return {'dump': base85.b85encode(data)}
+def pushmarker(repo, key, old, new):
+"""Push markers over pushkey"""
+if key != 'dump':
+repo.ui.warn(_('unknown key: %r') % key)
+return 0
+if old:
+repo.ui.warn(_('unexpected old value') % key)
+return 0
+data = base85.b85decode(new)
+lock = repo.lock()
 try:
-tmp = StringIO()
+repo.obsstore.mergemarkers(data)
-tmp.write(raw)
-tmp.seek(0)
-repo.obsoletestore.load(tmp)
-repo.obsoletestore._dirty = True # XXX meh
 return 1
 finally:
-l.release()
+lock.release()
-def listobsolete(repo):
+pushkey.register('obsolete', pushmarker, listmarkers)
-"""dump all obsolete relation in
-XXX this have be improved"""
-tmp = StringIO()
-repo.obsoletestore.save(tmp)
-return {'markers': base64.b64encode(tmp.getvalue())}
-pushkey.register('obsolete', pushobsolete, listobsolete)
 ### Discovery wrapping
 #############################
 class blist(list, object):
 for branch, nodes in oldbm.iteritems():
 nodes = list(nodes)
 new = set()
 while nodes:
 n = nodes.pop()
-if n in repo.obsoletestore.objects:
+if n in repo.obsstore.precursors:
-markers = repo.obsoletestore.objects[n]
+markers = repo.obsstore.precursors[n]
 for mark in markers:
-for newernode in mark['subjects']:
+for newernode in mark[1]:
 if newernode is not None:
 nodes.append(newernode)
 else:
 new.add(n)
 if new:
 @command('debugconvertobsolete', [], '')
 def cmddebugconvertobsolete(ui, repo):
 """import markers from an .hg/obsolete-relations file"""
 cnt = 0
 l = repo.lock()
+some = False
 try:
 repo._importoldobsolete = True
-store = repo.obsoletestore
+store = repo.obsstore
+### very first format
 try:
 f = repo.opener('obsolete-relations')
 try:
+some = True
 for line in f:
 subhex, objhex = line.split()
-sub = bin(subhex)
+suc = bin(subhex)
-obj = bin(objhex)
+prec = bin(objhex)
-newmarker = {
+sucs = (suc==nullid) and [] or [suc]
-'subjects': (sub==nullid) and [] or [sub],
+meta = {
-'object': obj,
+'date':  '%i %i' % util.makedate(),
-'date':  util.makedate(),
 'user': ui.username(),
-'reason': 'import from older format.',
 }
-store.new(newmarker)
+store.create(prec, sucs, 0, meta)
-store._dirty = True
 cnt += 1
 finally:
 f.close()
 util.unlink(repo.join('obsolete-relations'))
 except IOError:
-ui.warn('nothing to do\n')
 pass
+### second (json) format
+data = repo.sopener.tryread('obsoletemarkers')
+if data:
+some = True
+for oldmark in json.loads(data):
+del oldmark['id'] # dropped for now
+del oldmark['reason'] # unused until then
+oldmark['subjects'] = [bin(n) for n in oldmark['subjects']]
+oldmark['object'] = bin(oldmark['object'])
+oldmark['date'] = '%i %i' % tuple(oldmark['date'])
+store.create(oldmark.pop('object'),
+oldmark.pop('subjects'),
+0, oldmark)
+cnt += 1
+util.unlink(repo.sjoin('obsoletemarkers'))
 finally:
 del repo._importoldobsolete
 l.release()
+if not some:
+ui.warn('nothing to do\n')
 ui.status('%i obsolete marker converted\n' % cnt)
 @command('debugsuccessors', [], '')
 def cmddebugsuccessors(ui, repo):
 """dump obsolete changesets and their successors
 Each line matches an existing marker, the first identifier is the
 obsolete changeset identifier, followed by it successors.
 """
 lock = repo.lock()
 try:
-allsuccessors = repo.obsoletestore.objects
+allsuccessors = repo.obsstore.precursors
 for old in sorted(allsuccessors):
-successors = [sorted(m['subjects']) for m in allsuccessors[old]]
+successors = [sorted(m[1]) for m in allsuccessors[old]]
 for i, group in enumerate(sorted(successors)):
 ui.write('%s' % short(old))
 for new in group:
 ui.write(' %s' % short(new))
 ui.write('\n')
 oldnode = old.node()
 new = orig(ui, repo, commitfunc, old, *args, **kwargs)
 if new != oldnode:
 lock = repo.lock()
 try:
-newmarker = {
+meta = {
 'subjects':  [new],
 'object': oldnode,
 'date':  util.makedate(),
 'user': ui.username(),
 'reason': 'commit --amend',
 }
-repo.obsoletestore.new(newmarker)
+repo.obsstore.create(oldnode, [new], 0, meta)
 repo._clearobsoletecache()
 repo._turn_extinct_secret()
 finally:
 lock.release()
 return new
 def newerversion(repo, obs):
 """Return the newer version of an obsolete changeset"""
 toproceed = set([(obs,)])
 # XXX known optimization available
 newer = set()
-objectrels = repo.obsoletestore.objects
+objectrels = repo.obsstore.precursors
 while toproceed:
 current = toproceed.pop()
 assert len(current) <= 1, 'splitting not handled yet. %r' % current
 if current:
 n, = current
 if n in objectrels:
 markers = objectrels[n]
 for mark in markers:
-toproceed.add(tuple(mark['subjects']))
+toproceed.add(tuple(mark[1]))
 else:
 newer.add(tuple(current))
 else:
 newer.add(())
 return sorted(newer)
 else:
 a.update(str(marker[key]))
 a.update('\0')
 return a.digest()
-class obsoletestore(object):
+# mercurial backport
-"""Store obsolete relations
+def encodemeta(meta):
-Relation are stored in three mapping. All mapping have "obsolete markers"
+"""Return encoded metadata string to string mapping.
-as values::
+Assume no ':' in key and no '\0' in both key and value."""
-{'id': "unique id of the obsolete marker"
+for key, value in meta.iteritems():
-'subjects': "0-N newer version of changeset in "object" (as ordered list)
+if ':' in key or '\0' in key:
-'object': "old and obsolete version"
+raise ValueError("':' and '\0' are forbidden in metadata key'")
-'date': "When was this marker created ?"
+if '\0' in value:
-'user': "Who did that ?"
+raise ValueError("':' are forbidden in metadata value'")
-'reason': "Why was it done"
+return '\0'.join(['%s:%s' % (k, meta[k]) for k in sorted(meta)])
-}
+def decodemeta(data):
-Three keys exists
+"""Return string to string dictionary from encoded version."""
+d = {}
-:self._markers: "id" -> marker
+for l in data.split('\0'):
+if l:
-:self.subjects: "subject" -> set(marker)
+key, value = l.split(':')
+d[key] = value
-:self.objects: "object" -> set(marker)
+return d
+# data used for parsing and writing
+_fmversion = 0
+_fmfixed   = '>BIB20s'
+_fmnode = '20s'
+_fmfsize = struct.calcsize(_fmfixed)
+_fnodesize = struct.calcsize(_fmnode)
+def _readmarkers(data):
+"""Read and enumerate markers from raw data"""
+off = 0
+diskversion = _unpack('>B', data[off:off + 1])[0]
+off += 1
+if diskversion != _fmversion:
+raise util.Abort(_('parsing obsolete marker: unknown version %r')
+% diskversion)
+# Loop on markers
+l = len(data)
+while off + _fmfsize <= l:
+# read fixed part
+cur = data[off:off + _fmfsize]
+off += _fmfsize
+nbsuc, mdsize, flags, pre = _unpack(_fmfixed, cur)
+# read replacement
+sucs = ()
+if nbsuc:
+s = (_fnodesize * nbsuc)
+cur = data[off:off + s]
+sucs = _unpack(_fmnode * nbsuc, cur)
+off += s
+# read metadata
+# (metadata will be decoded on demand)
+metadata = data[off:off + mdsize]
+if len(metadata) != mdsize:
+raise util.Abort(_('parsing obsolete marker: metadata is too '
+'short, %d bytes expected, got %d')
+% (len(metadata), mdsize))
+off += mdsize
+yield (pre, sucs, flags, metadata)
+class obsstore(object):
+"""Store obsolete markers
+Markers can be accessed with two mappings:
+- precursors: old -> set(new)
+- successors: new -> set(old)
 """
 def __init__(self):
-self._markers = {}
+self._all = []
-self.subjects = {}
+# new markers to serialize
-self.objects = {}
+self._new = []
-self._dirty = False # should be on repo
+self.precursors = {}
+self.successors = {}
-def new(self, marker):
-"""Add a *new* marker to the store. computing it's ID"""
+def __iter__(self):
-mid = marker['id'] = markerid(marker)
+return iter(self._all)
-self._insert(marker)
-self._dirty = True
+def __nonzero__(self):
-return mid
+return bool(self._all)
-def _insert(self, marker):
+def create(self, prec, succs=(), flag=0, metadata=None):
-if marker['id'] not in self._markers:
+"""obsolete: add a new obsolete marker
-self._markers[marker['id']] = marker
-add2set(self.objects, marker['object'], marker)
+* ensuring it is hashable
-for subj in marker['subjects']:
+* check mandatory metadata
-add2set(self.subjects, subj, marker)
+* encode metadata
+"""
-def save(self, stream):
+if metadata is None:
-markers = []
+metadata = {}
-for mark in self._markers.itervalues():
+if len(prec) != 20:
-jmark = mark.copy()
+raise ValueError(prec)
-jmark['id'] = hex(jmark['id'])
+for succ in succs:
-jmark['subjects'] = [hex(n) for n in jmark['subjects']]
+if len(succ) != 20:
-jmark['object'] = hex(jmark['object'])
+raise ValueError(prec)
-markers.append(jmark)
+marker = (str(prec), tuple(succs), int(flag), encodemeta(metadata))
-json.dump(markers, stream, indent=4)
+self.add(marker)
-def load(self, stream):
+def add(self, marker):
-for mark in json.load(stream):
+"""Add a new marker to the store
-mark['id'] = bin(mark['id'])
-mark['subjects'] = [bin(n) for n in mark['subjects']]
+This marker still needs to be written to disk"""
-mark['object'] = bin(mark['object'])
+self._new.append(marker)
-self._insert(mark)
+self._load(marker)
-def writeobsolete(repo):
+def loadmarkers(self, data):
-"""wire obsolete data on disk"""
+"""Load all markers in data, mark them as known."""
-f = repo.sopener('obsoletemarkers', 'w', atomictemp=True)
+for marker in _readmarkers(data):
-try:
+self._load(marker)
-repo.obsoletestore.save(f)
-repo._dirty = False
+def mergemarkers(self, data):
-finally:
+other = set(_readmarkers(data))
-f.close()
+local = set(self._all)
+new = other - local
+for marker in new:
+self.add(marker)
+def flushmarkers(self, stream):
+"""Write all markers to a stream
+After this operation, "new" markers are considered "known"."""
+self._writemarkers(stream)
+self._new[:] = []
+def _load(self, marker):
+self._all.append(marker)
+pre, sucs = marker[:2]
+self.precursors.setdefault(pre, set()).add(marker)
+for suc in sucs:
+self.successors.setdefault(suc, set()).add(marker)
+def _writemarkers(self, stream=None):
+# Kept separate from flushmarkers(), it will be reused for
+# markers exchange.
+if stream is None:
+final = []
+w = final.append
+else:
+w = stream.write
+w(_pack('>B', _fmversion))
+for marker in self._all:
+pre, sucs, flags, metadata = marker
+nbsuc = len(sucs)
+format = _fmfixed + (_fmnode * nbsuc)
+data = [nbsuc, len(metadata), flags, pre]
+data.extend(sucs)
+w(_pack(format, *data))
+w(metadata)
+if stream is None:
+return ''.join(final)
 ### repo subclassing
 #############################
 ### Public method
 def obsoletedby(self, node):
 """return the set of node that make <node> obsolete (obj)"""
 others = set()
-for marker in self.obsoletestore.objects.get(node, []):
+for marker in self.obsstore.precursors.get(node, []):
-others.update(marker['subjects'])
+others.update(marker[1])
 return others
 def obsolete(self, node):
 """return the set of node that <node> make obsolete (sub)"""
-return set(marker['object'] for marker in self.obsoletestore.subjects.get(node, []))
+return set(marker[0] for marker in self.obsstore.successors.get(node, []))
-@util.propertycache
+@storecache('obsstore')
-def obsoletestore(self):
+def obsstore(self):
 if not getattr(self, '_importoldobsolete', False):
-try:
+data = repo.opener.tryread('obsolete-relations')
-f = self.opener('obsolete-relations')
+if not data:
-f.close()
+data = repo.sopener.tryread('obsoletemarkers')
+if data:
 raise util.Abort('old format of obsolete marker detected!\n'
 'run `hg debugconvertobsolete` once.')
-except IOError:
+store = obsstore()
-pass
+data = self.sopener.tryread('obsstore')
-store = obsoletestore()
+if data:
-try:
+store.loadmarkers(data)
-f = self.sopener('obsoletemarkers')
-store.load(f)
-except IOError:
-pass
 return store
 @util.propertycache
 def _obsoleteset(self):
 """the set of obsolete revision"""
 obs = set()
 nm = self.changelog.nodemap
-for obj in self.obsoletestore.objects:
+for obj in self.obsstore.precursors:
 try: # /!\api change in Hg 2.2 (e8d37b78acfb22ae2c1fb126c2)/!\
 rev = nm.get(obj)
 except TypeError:  #XXX to remove while breaking Hg 2.1 support
 rev = nm.get(obj, None)
 if rev is not None:
 self.ui.warn(
 _("%(sub)s try to obsolete immutable changeset %(obj)s\n")
 % {'sub': short(sub), 'obj': short(obj)})
 lock = self.lock()
 try:
-newmarker = {
+meta = {
-'subjects': (sub==nullid) and [] or [sub],
-'object': obj,
 'date':  util.makedate(),
 'user': ui.username(),
 'reason': 'unknown',
 }
-mid = self.obsoletestore.new(newmarker)
+subs = (sub == nullid) and [] or [sub]
+mid = self.obsstore.create(obj, subs, 0, meta)
 self._clearobsoletecache()
 self._turn_extinct_secret()
 return mid
 finally:
 lock.release()
 def lock(self, *args, **kwargs):
 l = olock(*args, **kwargs)
 if not getattr(l.releasefn, 'obspatched', False):
 oreleasefn = l.releasefn
 def releasefn(*args, **kwargs):
-if self.obsoletestore._dirty:
+if 'obsstore' in vars(self) and self.obsstore._new:
-writeobsolete(self)
+f = self.sopener('obsstore', 'wb', atomictemp=True)
+try:
+self.obsstore.flushmarkers(f)
+f.close()
+except: # re-raises
+f.discard()
+raise
 oreleasefn(*args, **kwargs)
 releasefn.obspatched = True
 l.releasefn = releasefn
 return l
-def _readobsrels(self):
-"""Read obsolete relation on disk"""
-# XXX handle lock
-try:
-f = self.opener('obsolete-relations')
-try:
-return _obsdeserialise(f)
-finally:
-f.close()
-except IOError:
-return {}
 ### pull // push support
 def pull(self, remote, *args, **kwargs):
 """wrapper around push that push obsolete relation"""
 l = repo.lock()
 try:
 result = opull(remote, *args, **kwargs)
-if 'obsolete' in remote.listkeys('namespaces'):
+remoteobs = remote.listkeys('obsolete')
-tmp = StringIO()
+if 'dump' in remoteobs:
-rels = remote.listkeys('obsolete')['markers']
+data = base85.b85decode(remoteobs['dump'])
-tmp.write(base64.b64decode(rels))
+self.obsstore.mergemarkers(data)
-tmp.seek(0)
-repo.obsoletestore.load(tmp)
-repo.obsoletestore._dirty = True # XXX meh
 self._clearobsoletecache()
 self._turn_extinct_secret()
 return result
 finally:
 l.release()
 def push(self, remote, *args, **opts):
 """wrapper around pull that pull obsolete relation"""
 self._turn_extinct_secret()
 result = opush(remote, *args, **opts)
-if 'obsolete' in remote.listkeys('namespaces'):
+if 'obsolete' in self.listkeys('namespaces') and self.obsstore:
-tmp = StringIO()
+data = self.obsstore._writemarkers()
-self.obsoletestore.save(tmp)
+r = remote.pushkey('obsolete', 'dump', '',
-remote.pushkey('obsolete', 'markers', '', tmp.getvalue())
+base85.b85encode(data))
+if not r:
+self.ui.warn(_('failed to push obsolete markers!\n'))
 self._turn_extinct_secret()
 return result
 ### rollback support
 # /!\ api change in  Hg 2.2 (97efd26eb9576f39590812ea9) /!\
 if util.safehasattr(repo, '_journalfiles'): # Hg 2.2
 def _journalfiles(self):
-return o_journalfiles() + (self.sjoin('journal.obsoletemarkers'),)
+return o_journalfiles() + (self.sjoin('journal.obsstore'),)
 def _writejournal(self, desc):
 """wrapped version of _writejournal that save obsolete data"""
 o_writejournal(desc)
-filename = 'obsoletemarkers'
+filename = 'obsstore'
 filepath = self.sjoin(filename)
 if os.path.exists(filepath):
 journalname = 'journal.' + filename
 journalpath = self.sjoin(journalname)
 util.copyfile(filepath, journalpath)
 else: # XXX removing this bloc will break Hg 2.1 support
 def _writejournal(self, desc):
 """wrapped version of _writejournal that save obsolete data"""
 entries = list(o_writejournal(desc))
-filename = 'obsoletemarkers'
+filename = 'obsstore'
 filepath = self.sjoin(filename)
 if  os.path.exists(filepath):
 journalname = 'journal.' + filename
 journalpath = self.sjoin(journalname)
 util.copyfile(filepath, journalpath)
 def _rollback(self, dryrun, force):
 """wrapped version of _rollback that restore obsolete data"""
 ret = o_rollback(dryrun, force)
 if not (ret or dryrun): #rollback did not failed
-src = self.sjoin('undo.obsoletemarkers')
+src = self.sjoin('undo.obsstore')
-dst = self.sjoin('obsoletemarkers')
+dst = self.sjoin('obsstore')
 if os.path.exists(src):
 util.rename(src, dst)
 elif os.path.exists(dst):
 # If no state was saved because the file did not existed before.
 os.unlink(dst)
 # invalidate cache
-self.__dict__.pop('obsoletestore', None)
+self.__dict__.pop('obsstore', None)
 return ret
 @storecache('00changelog.i')
 def changelog(self):
 # << copy pasted from mercurial source

changeset 342	aab826129142
parent 334	fb83210bce32
child 335	c27a465daef2
child 343	6b92f8d5ae58