pullbundle: add caching logic
We now only generate a bundle once (and store it to disk). If we need it again,
we use it directly from disk.
--- a/hgext3rd/evolve/stablerange.py Sun Sep 23 23:41:08 2018 +0200
+++ b/hgext3rd/evolve/stablerange.py Sun Sep 23 00:08:02 2018 +0200
@@ -195,7 +195,6 @@
assert standard_start < rangedepth
slicepoint = standard_start
return slicepoint
-
class stablerangebasic(abstractstablerange):
"""a very dummy implementation of stablerange
--- a/hgext3rd/pullbundle.py Sun Sep 23 23:41:08 2018 +0200
+++ b/hgext3rd/pullbundle.py Sun Sep 23 00:08:02 2018 +0200
@@ -123,7 +123,7 @@
subranges = canonicalsubranges(repo, stablerange, rangeid)
idx = 0
- slices =[]
+ slices = []
nodes.reverse()
for rangeid in subranges:
size = rangelength(rangeid)
@@ -158,7 +158,7 @@
cut += cursor
if cursor == 1:
break
- cursor //=2
+ cursor //= 2
# 2. optimise, bottom part
if skip != cut:
@@ -216,10 +216,8 @@
if repo.ui.verbose or source == 'bundle':
repo.ui.status(_("%d changesets found\n") % len(nodes))
-def makeonecgpart(newpart, repo, rangeid, outgoing, version, source,
- bundlecaps, filematcher, cgversions):
- # same as upstream code
-
+def _makenewstream(newpart, repo, outgoing, version, source,
+ bundlecaps, filematcher, cgversions):
old = changegroup._changegroupinfo
try:
changegroup._changegroupinfo = _changegroupinfo
@@ -229,12 +227,76 @@
finally:
changegroup._changegroupinfo = old
+ nbchanges = len(outgoing.missing)
+ pversion = None
+ if cgversions:
+ pversion = version
+ return (cgstream, nbchanges, pversion)
+
+def _makepartfromstream(newpart, repo, cgstream, nbchanges, version):
+ # same as upstream code
+
part = newpart('changegroup', data=cgstream)
- if cgversions:
+ if version:
part.addparam('version', version)
- part.addparam('nbchanges', '%d' % len(outgoing.missing),
+ part.addparam('nbchanges', '%d' % nbchanges,
mandatory=False)
if 'treemanifest' in repo.requirements:
part.addparam('treemanifest', '1')
+
+# cache management
+
+def cachedir(repo):
+ return repo.cachevfs.join('pullbundles')
+
+def getcache(repo, bundlename):
+ cdir = cachedir(repo)
+ bundlepath = os.path.join(cdir, bundlename)
+ try:
+ fd = open(bundlepath, 'rb')
+ return util.filechunkiter(fd)
+ except IOError as exc:
+ if exc.errno != errno.ENOENT:
+ raise
+ return None
+
+def cachewriter(repo, bundlename, stream):
+ cdir = cachedir(repo)
+ bundlepath = os.path.join(cdir, bundlename)
+ try:
+ os.makedirs(cdir)
+ except OSError as exc:
+ if exc.errno == errno.EEXIST:
+ pass
+ with util.atomictempfile(bundlepath) as cachefile:
+ for chunk in stream:
+ cachefile.write(chunk)
+ yield chunk
+
+BUNDLEMASK = "%s-%s-%010iskip-%010isize.hg"
+
+def makeonecgpart(newpart, repo, rangeid, outgoing, version, source,
+ bundlecaps, filematcher, cgversions):
+ bundlename = cachedata = None
+ if rangeid is not None:
+ nbchanges = repo.stablerange.rangelength(repo, rangeid)
+ headnode = nodemod.hex(repo.changelog.node(rangeid[0]))
+ # XXX do we need to use cgversion in there?
+ bundlename = BUNDLEMASK % (version, headnode, rangeid[1], nbchanges)
+ cachedata = getcache(repo, bundlename)
+ if cachedata is None:
+ partdata = _makenewstream(newpart, repo, outgoing, version, source,
+ bundlecaps, filematcher, cgversions)
+ if bundlename is not None:
+ cgstream = cachewriter(repo, bundlename, partdata[0])
+ partdata = (cgstream,) + partdata[1:]
+ else:
+ if repo.ui.verbose or source == 'bundle':
+ repo.ui.status(_("%d changesets found in caches\n") % nbchanges)
+ pversion = None
+ if cgversions:
+ pversion = version
+ partdata = (cachedata, nbchanges, pversion)
+ return _makepartfromstream(newpart, repo, *partdata)
--- a/tests/test-pullbundle.t Sun Sep 23 23:41:08 2018 +0200
+++ b/tests/test-pullbundle.t Sun Sep 23 00:08:02 2018 +0200
@@ -221,6 +221,20 @@
new changesets 1ea73414a91b:f864bc82f6a2
(run 'hg update' to get a working copy)
+ $ touch oldbundles
+ $ ls -1 server/.hg/cache/pullbundles > newbundles
+ $ diff -u oldbundles newbundles
+ --- oldbundles * (glob)
+ +++ newbundles * (glob)
+ @@ -0,0 +1,6 @@
+ +02-467b6e370e816747e27de0d0b9237f4090a33656-0000001152skip-0000000064size.hg
+ +02-540f762640ee62ca597ece26af725e6357e82805-0000000000skip-0000001024size.hg
+ +02-63ded94ceab180ac2fa13e1f0beeb4d2265998a3-0000001232skip-0000000002size.hg
+ +02-7f3a79522d6e904d52aea07c71e6cb612667e8f4-0000001216skip-0000000016size.hg
+ +02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg
+ +02-f864bc82f6a2f2ecb49b83722e0895f9d657b0dd-0000001234skip-0000000001size.hg
+ [1]
+
pull the other missing entries (multiple heads pulled)
------------------------------------------------------
@@ -626,6 +640,36 @@
new changesets 17185c1c22f1:1dded5aafa0f
(run 'hg heads' to see heads, 'hg merge' to merge)
+ $ mv newbundles oldbundles
+ $ ls -1 server/.hg/cache/pullbundles > newbundles
+ $ diff -u oldbundles newbundles
+ --- oldbundles * (glob)
+ +++ newbundles * (glob)
+ @@ -1,6 +1,22 @@
+ +02-0f376356904fc8c1c6ceaac27990f2fd79b1f8c1-0000001856skip-0000000008size.hg
+ +02-1dded5aafa0f8d548f6357cc2f8882dcc4489fbf-0000001516skip-0000000002size.hg
+ +02-2dc4f1ab9029719714b8e0dde8e3725a5bb28472-0000001408skip-0000000064size.hg
+ +02-2f0e261a08964bc1c607c0eda4978364c22a9b94-0000001504skip-0000000008size.hg
+ 02-467b6e370e816747e27de0d0b9237f4090a33656-0000001152skip-0000000064size.hg
+ +02-4a6d0f7d07d060b026d9fc690cd89cd26af96e42-0000001264skip-0000000016size.hg
+ 02-540f762640ee62ca597ece26af725e6357e82805-0000000000skip-0000001024size.hg
+ +02-5eaa75df73c454c1afff722301a4c73e897de94d-0000001256skip-0000000008size.hg
+ 02-63ded94ceab180ac2fa13e1f0beeb4d2265998a3-0000001232skip-0000000002size.hg
+ 02-7f3a79522d6e904d52aea07c71e6cb612667e8f4-0000001216skip-0000000016size.hg
+ +02-b2d350c94c26edbb783aaa21fc24f1fc65c30e74-0000001536skip-0000000256size.hg
+ +02-bbd293bd171fd5b711d428db46940a72eca7a40f-0000001280skip-0000000128size.hg
+ +02-ca970a853ea24846035ccb324cc8de49ef768748-0000001252skip-0000000004size.hg
+ +02-d1807e3513890ac71c2e8d10e9dc9a5b58b15d4b-0000001251skip-0000000001size.hg
+ +02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001235skip-0000000016size.hg
+ +02-da87a81c5310760f414a933e6550b7e8e60cf241-0000001792skip-0000000064size.hg
+ +02-dba2fddbf3c28198659046674a512afd616a1519-0000001472skip-0000000032size.hg
+ +02-e469a7aa5cce57653b6b02ff46c80b2d94d62629-0000000899skip-0000000029size.hg
+ +02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001280skip-0000000256size.hg
+ 02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg
+ 02-f864bc82f6a2f2ecb49b83722e0895f9d657b0dd-0000001234skip-0000000001size.hg
+ +02-fb6c210a224903e81e5a8d2ee099cb0c9526ba8c-0000001512skip-0000000004size.hg
+ [1]
+
Same Pullin with a different client
-----------------------------------
$ hg -R server log -G -T '{rev}:{node}\n' -r '0+1234+(::1234 and (merge() or branchpoint()))'
@@ -639,12 +683,12 @@
$ hg -R client2 pull server -r 1234 --debug --config devel.bundle2.debug=yes --traceback| grep -v 'add changeset'
pulling from server
query 1; heads
- 1024 changesets found
- 128 changesets found
- 64 changesets found
- 16 changesets found
- 2 changesets found
- 1 changesets found
+ 1024 changesets found in caches
+ 128 changesets found in caches
+ 64 changesets found in caches
+ 16 changesets found in caches
+ 2 changesets found in caches
+ 1 changesets found in caches
listing keys for "bookmarks"
bundle2-output-bundle: "HG20", 9 parts total
bundle2-output: start emission of HG20 stream
@@ -829,6 +873,10 @@
new changesets 1ea73414a91b:f864bc82f6a2
(run 'hg update' to get a working copy)
+ $ mv newbundles oldbundles
+ $ ls -1 server/.hg/cache/pullbundles > newbundles
+ $ diff -u oldbundles newbundles
+
$ hg -R server log -G -T '{rev}:{node}\n' -r '1234+head()+(only(head(), 1234) and (merge() or branchpoint()))'
o 2130:0f376356904fc8c1c6ceaac27990f2fd79b1f8c1
:
@@ -848,22 +896,22 @@
query 1; heads
searching for changes
all local heads known remotely
- 16 changesets found
- 29 changesets found
- 256 changesets found
- 256 changesets found
- 64 changesets found
- 8 changesets found
- 1 changesets found
- 4 changesets found
- 8 changesets found
- 16 changesets found
- 128 changesets found
- 64 changesets found
- 32 changesets found
- 8 changesets found
- 4 changesets found
- 2 changesets found
+ 16 changesets found in caches
+ 29 changesets found in caches
+ 256 changesets found in caches
+ 256 changesets found in caches
+ 64 changesets found in caches
+ 8 changesets found in caches
+ 1 changesets found in caches
+ 4 changesets found in caches
+ 8 changesets found in caches
+ 16 changesets found in caches
+ 128 changesets found in caches
+ 64 changesets found in caches
+ 32 changesets found in caches
+ 8 changesets found in caches
+ 4 changesets found in caches
+ 2 changesets found in caches
listing keys for "bookmarks"
bundle2-output-bundle: "HG20", 19 parts total
bundle2-output: start emission of HG20 stream
@@ -1232,6 +1280,10 @@
new changesets 17185c1c22f1:1dded5aafa0f
(run 'hg heads' to see heads, 'hg merge' to merge)
+ $ mv newbundles oldbundles
+ $ ls -1 server/.hg/cache/pullbundles > newbundles
+ $ diff -u oldbundles newbundles
+
different pull with a different client
--------------------------------------
@@ -1252,9 +1304,9 @@
$ hg -R client3 pull server -r 1789 --debug --config devel.bundle2.debug=yes | grep -v 'add changeset'
pulling from server
query 1; heads
- 1024 changesets found
+ 1024 changesets found in caches
227 changesets found
- 29 changesets found
+ 29 changesets found in caches
128 changesets found
64 changesets found
32 changesets found
@@ -1504,6 +1556,40 @@
new changesets 1ea73414a91b:44e80141ad53
(run 'hg update' to get a working copy)
+ $ mv newbundles oldbundles
+ $ ls -1 server/.hg/cache/pullbundles > newbundles
+ $ diff -u oldbundles newbundles
+ --- oldbundles * (glob)
+ +++ newbundles * (glob)
+ @@ -2,19 +2,26 @@
+ 02-1dded5aafa0f8d548f6357cc2f8882dcc4489fbf-0000001516skip-0000000002size.hg
+ 02-2dc4f1ab9029719714b8e0dde8e3725a5bb28472-0000001408skip-0000000064size.hg
+ 02-2f0e261a08964bc1c607c0eda4978364c22a9b94-0000001504skip-0000000008size.hg
+ +02-44e80141ad530a2aa085e9bd9b5311b57eff72ff-0000001522skip-0000000001size.hg
+ 02-467b6e370e816747e27de0d0b9237f4090a33656-0000001152skip-0000000064size.hg
+ 02-4a6d0f7d07d060b026d9fc690cd89cd26af96e42-0000001264skip-0000000016size.hg
+ 02-540f762640ee62ca597ece26af725e6357e82805-0000000000skip-0000001024size.hg
+ +02-59e60b258b18cde1e931cf30ce4ae62b49e37abd-0000001520skip-0000000002size.hg
+ 02-5eaa75df73c454c1afff722301a4c73e897de94d-0000001256skip-0000000008size.hg
+ 02-63ded94ceab180ac2fa13e1f0beeb4d2265998a3-0000001232skip-0000000002size.hg
+ +02-7a55a4d5ce324910842c893b56173cf2a847cb9d-0000001472skip-0000000032size.hg
+ 02-7f3a79522d6e904d52aea07c71e6cb612667e8f4-0000001216skip-0000000016size.hg
+ +02-a4ab7df9d74053fb819c8a1c6a48ad605cc05f8a-0000001504skip-0000000016size.hg
+ 02-b2d350c94c26edbb783aaa21fc24f1fc65c30e74-0000001536skip-0000000256size.hg
+ 02-bbd293bd171fd5b711d428db46940a72eca7a40f-0000001280skip-0000000128size.hg
+ +02-c12927fef661d2463043347101b90067c2961333-0000001280skip-0000000128size.hg
+ 02-ca970a853ea24846035ccb324cc8de49ef768748-0000001252skip-0000000004size.hg
+ 02-d1807e3513890ac71c2e8d10e9dc9a5b58b15d4b-0000001251skip-0000000001size.hg
+ +02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001024skip-0000000227size.hg
+ 02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001235skip-0000000016size.hg
+ 02-da87a81c5310760f414a933e6550b7e8e60cf241-0000001792skip-0000000064size.hg
+ 02-dba2fddbf3c28198659046674a512afd616a1519-0000001472skip-0000000032size.hg
+ +02-dc714c3a5d080165292ba99b097567d0b95e5756-0000001408skip-0000000064size.hg
+ 02-e469a7aa5cce57653b6b02ff46c80b2d94d62629-0000000899skip-0000000029size.hg
+ 02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001280skip-0000000256size.hg
+ 02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg
+ [1]
+
$ hg -R server log -G -T '{rev}:{node}\n' -r '1789+head()+parents(roots(only(head(), 1789)))+(only(head(), 1789) and (merge() or branchpoint()))'
o 2130:0f376356904fc8c1c6ceaac27990f2fd79b1f8c1
:
@@ -1523,19 +1609,19 @@
1 changesets found
4 changesets found
8 changesets found
- 256 changesets found
- 64 changesets found
- 8 changesets found
- 1 changesets found
- 4 changesets found
- 8 changesets found
- 16 changesets found
- 128 changesets found
- 64 changesets found
- 32 changesets found
- 8 changesets found
- 4 changesets found
- 2 changesets found
+ 256 changesets found in caches
+ 64 changesets found in caches
+ 8 changesets found in caches
+ 1 changesets found in caches
+ 4 changesets found in caches
+ 8 changesets found in caches
+ 16 changesets found in caches
+ 128 changesets found in caches
+ 64 changesets found in caches
+ 32 changesets found in caches
+ 8 changesets found in caches
+ 4 changesets found in caches
+ 2 changesets found in caches
listing keys for "bookmarks"
bundle2-output-bundle: "HG20", 19 parts total
bundle2-output: start emission of HG20 stream
@@ -1901,3 +1987,33 @@
updating the branch cache
new changesets c232505f58fd:1dded5aafa0f
(run 'hg heads' to see heads, 'hg merge' to merge)
+
+ $ mv newbundles oldbundles
+ $ ls -1 server/.hg/cache/pullbundles > newbundles
+ $ diff -u oldbundles newbundles
+ --- oldbundles * (glob)
+ +++ newbundles * (glob)
+ @@ -1,5 +1,6 @@
+ 02-0f376356904fc8c1c6ceaac27990f2fd79b1f8c1-0000001856skip-0000000008size.hg
+ 02-1dded5aafa0f8d548f6357cc2f8882dcc4489fbf-0000001516skip-0000000002size.hg
+ +02-1ed78f99f705cb819a02f1227c217728d008e461-0000001524skip-0000000004size.hg
+ 02-2dc4f1ab9029719714b8e0dde8e3725a5bb28472-0000001408skip-0000000064size.hg
+ 02-2f0e261a08964bc1c607c0eda4978364c22a9b94-0000001504skip-0000000008size.hg
+ 02-44e80141ad530a2aa085e9bd9b5311b57eff72ff-0000001522skip-0000000001size.hg
+ @@ -15,6 +16,7 @@
+ 02-b2d350c94c26edbb783aaa21fc24f1fc65c30e74-0000001536skip-0000000256size.hg
+ 02-bbd293bd171fd5b711d428db46940a72eca7a40f-0000001280skip-0000000128size.hg
+ 02-c12927fef661d2463043347101b90067c2961333-0000001280skip-0000000128size.hg
+ +02-c232505f58fdf70bcf5f6ab6a555f23ffc74f761-0000001523skip-0000000001size.hg
+ 02-ca970a853ea24846035ccb324cc8de49ef768748-0000001252skip-0000000004size.hg
+ 02-d1807e3513890ac71c2e8d10e9dc9a5b58b15d4b-0000001251skip-0000000001size.hg
+ 02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001024skip-0000000227size.hg
+ @@ -24,6 +26,7 @@
+ 02-dc714c3a5d080165292ba99b097567d0b95e5756-0000001408skip-0000000064size.hg
+ 02-e469a7aa5cce57653b6b02ff46c80b2d94d62629-0000000899skip-0000000029size.hg
+ 02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001280skip-0000000256size.hg
+ +02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001528skip-0000000008size.hg
+ 02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg
+ 02-f864bc82f6a2f2ecb49b83722e0895f9d657b0dd-0000001234skip-0000000001size.hg
+ 02-fb6c210a224903e81e5a8d2ee099cb0c9526ba8c-0000001512skip-0000000004size.hg
+ [1]