pullbundle: add caching logic
authorPierre-Yves David <pierre-yves.david@octobus.net>
Sun, 23 Sep 2018 00:08:02 +0200
changeset 4130 a1f6b8211016
parent 4129 bc4e62a1cb82
child 4131 788798a9a159
pullbundle: add caching logic We now only generate a bundle once (and store it to disk). If we need it again, we use it directly from disk.
hgext3rd/evolve/stablerange.py
hgext3rd/pullbundle.py
tests/test-pullbundle.t
--- a/hgext3rd/evolve/stablerange.py	Sun Sep 23 23:41:08 2018 +0200
+++ b/hgext3rd/evolve/stablerange.py	Sun Sep 23 00:08:02 2018 +0200
@@ -195,7 +195,6 @@
             assert standard_start < rangedepth
             slicepoint = standard_start
         return slicepoint
-
 class stablerangebasic(abstractstablerange):
     """a very dummy implementation of stablerange
 
--- a/hgext3rd/pullbundle.py	Sun Sep 23 23:41:08 2018 +0200
+++ b/hgext3rd/pullbundle.py	Sun Sep 23 00:08:02 2018 +0200
@@ -123,7 +123,7 @@
 
     subranges = canonicalsubranges(repo, stablerange, rangeid)
     idx = 0
-    slices =[]
+    slices = []
     nodes.reverse()
     for rangeid in subranges:
         size = rangelength(rangeid)
@@ -158,7 +158,7 @@
             cut += cursor
         if cursor == 1:
             break
-        cursor //=2
+        cursor //= 2
 
     # 2. optimise, bottom part
     if skip != cut:
@@ -216,10 +216,8 @@
     if repo.ui.verbose or source == 'bundle':
         repo.ui.status(_("%d changesets found\n") % len(nodes))
 
-def makeonecgpart(newpart, repo, rangeid, outgoing, version, source,
-                  bundlecaps, filematcher, cgversions):
-    # same as upstream code
-
+def _makenewstream(newpart, repo, outgoing, version, source,
+                   bundlecaps, filematcher, cgversions):
     old = changegroup._changegroupinfo
     try:
         changegroup._changegroupinfo = _changegroupinfo
@@ -229,12 +227,76 @@
     finally:
         changegroup._changegroupinfo = old
 
+    nbchanges = len(outgoing.missing)
+    pversion = None
+    if cgversions:
+        pversion = version
+    return (cgstream, nbchanges, pversion)
+
+def _makepartfromstream(newpart, repo, cgstream, nbchanges, version):
+    # same as upstream code
+
     part = newpart('changegroup', data=cgstream)
-    if cgversions:
+    if version:
         part.addparam('version', version)
 
-    part.addparam('nbchanges', '%d' % len(outgoing.missing),
+    part.addparam('nbchanges', '%d' % nbchanges,
                   mandatory=False)
 
     if 'treemanifest' in repo.requirements:
         part.addparam('treemanifest', '1')
+
+# cache management
+
+def cachedir(repo):
+    return repo.cachevfs.join('pullbundles')
+
+def getcache(repo, bundlename):
+    cdir = cachedir(repo)
+    bundlepath = os.path.join(cdir, bundlename)
+    try:
+        fd = open(bundlepath, 'rb')
+        return util.filechunkiter(fd)
+    except IOError as exc:
+        if exc.errno != errno.ENOENT:
+            raise
+        return None
+
+def cachewriter(repo, bundlename, stream):
+    cdir = cachedir(repo)
+    bundlepath = os.path.join(cdir, bundlename)
+    try:
+        os.makedirs(cdir)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST:
+            pass
+    with util.atomictempfile(bundlepath) as cachefile:
+        for chunk in stream:
+            cachefile.write(chunk)
+            yield chunk
+
+BUNDLEMASK = "%s-%s-%010iskip-%010isize.hg"
+
+def makeonecgpart(newpart, repo, rangeid, outgoing, version, source,
+                  bundlecaps, filematcher, cgversions):
+    bundlename = cachedata = None
+    if rangeid is not None:
+        nbchanges = repo.stablerange.rangelength(repo, rangeid)
+        headnode = nodemod.hex(repo.changelog.node(rangeid[0]))
+        # XXX do we need to use cgversion in there?
+        bundlename = BUNDLEMASK % (version, headnode, rangeid[1], nbchanges)
+        cachedata = getcache(repo, bundlename)
+    if cachedata is None:
+        partdata = _makenewstream(newpart, repo, outgoing, version, source,
+                                  bundlecaps, filematcher, cgversions)
+        if bundlename is not None:
+            cgstream = cachewriter(repo, bundlename, partdata[0])
+            partdata = (cgstream,) + partdata[1:]
+    else:
+        if repo.ui.verbose or source == 'bundle':
+            repo.ui.status(_("%d changesets found in caches\n") % nbchanges)
+        pversion = None
+        if cgversions:
+            pversion = version
+        partdata = (cachedata, nbchanges, pversion)
+    return _makepartfromstream(newpart, repo, *partdata)
--- a/tests/test-pullbundle.t	Sun Sep 23 23:41:08 2018 +0200
+++ b/tests/test-pullbundle.t	Sun Sep 23 00:08:02 2018 +0200
@@ -221,6 +221,20 @@
   new changesets 1ea73414a91b:f864bc82f6a2
   (run 'hg update' to get a working copy)
 
+  $ touch oldbundles
+  $ ls -1 server/.hg/cache/pullbundles > newbundles
+  $ diff -u oldbundles newbundles
+  --- oldbundles	* (glob)
+  +++ newbundles	* (glob)
+  @@ -0,0 +1,6 @@
+  +02-467b6e370e816747e27de0d0b9237f4090a33656-0000001152skip-0000000064size.hg
+  +02-540f762640ee62ca597ece26af725e6357e82805-0000000000skip-0000001024size.hg
+  +02-63ded94ceab180ac2fa13e1f0beeb4d2265998a3-0000001232skip-0000000002size.hg
+  +02-7f3a79522d6e904d52aea07c71e6cb612667e8f4-0000001216skip-0000000016size.hg
+  +02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg
+  +02-f864bc82f6a2f2ecb49b83722e0895f9d657b0dd-0000001234skip-0000000001size.hg
+  [1]
+
 pull the other missing entries (multiple heads pulled)
 ------------------------------------------------------
 
@@ -626,6 +640,36 @@
   new changesets 17185c1c22f1:1dded5aafa0f
   (run 'hg heads' to see heads, 'hg merge' to merge)
 
+  $ mv newbundles oldbundles
+  $ ls -1 server/.hg/cache/pullbundles > newbundles
+  $ diff -u oldbundles newbundles
+  --- oldbundles	* (glob)
+  +++ newbundles	* (glob)
+  @@ -1,6 +1,22 @@
+  +02-0f376356904fc8c1c6ceaac27990f2fd79b1f8c1-0000001856skip-0000000008size.hg
+  +02-1dded5aafa0f8d548f6357cc2f8882dcc4489fbf-0000001516skip-0000000002size.hg
+  +02-2dc4f1ab9029719714b8e0dde8e3725a5bb28472-0000001408skip-0000000064size.hg
+  +02-2f0e261a08964bc1c607c0eda4978364c22a9b94-0000001504skip-0000000008size.hg
+   02-467b6e370e816747e27de0d0b9237f4090a33656-0000001152skip-0000000064size.hg
+  +02-4a6d0f7d07d060b026d9fc690cd89cd26af96e42-0000001264skip-0000000016size.hg
+   02-540f762640ee62ca597ece26af725e6357e82805-0000000000skip-0000001024size.hg
+  +02-5eaa75df73c454c1afff722301a4c73e897de94d-0000001256skip-0000000008size.hg
+   02-63ded94ceab180ac2fa13e1f0beeb4d2265998a3-0000001232skip-0000000002size.hg
+   02-7f3a79522d6e904d52aea07c71e6cb612667e8f4-0000001216skip-0000000016size.hg
+  +02-b2d350c94c26edbb783aaa21fc24f1fc65c30e74-0000001536skip-0000000256size.hg
+  +02-bbd293bd171fd5b711d428db46940a72eca7a40f-0000001280skip-0000000128size.hg
+  +02-ca970a853ea24846035ccb324cc8de49ef768748-0000001252skip-0000000004size.hg
+  +02-d1807e3513890ac71c2e8d10e9dc9a5b58b15d4b-0000001251skip-0000000001size.hg
+  +02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001235skip-0000000016size.hg
+  +02-da87a81c5310760f414a933e6550b7e8e60cf241-0000001792skip-0000000064size.hg
+  +02-dba2fddbf3c28198659046674a512afd616a1519-0000001472skip-0000000032size.hg
+  +02-e469a7aa5cce57653b6b02ff46c80b2d94d62629-0000000899skip-0000000029size.hg
+  +02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001280skip-0000000256size.hg
+   02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg
+   02-f864bc82f6a2f2ecb49b83722e0895f9d657b0dd-0000001234skip-0000000001size.hg
+  +02-fb6c210a224903e81e5a8d2ee099cb0c9526ba8c-0000001512skip-0000000004size.hg
+  [1]
+
 Same Pullin with a different client
 -----------------------------------
   $ hg -R server log -G -T '{rev}:{node}\n' -r '0+1234+(::1234 and (merge() or branchpoint()))'
@@ -639,12 +683,12 @@
   $ hg -R client2 pull server -r 1234 --debug --config devel.bundle2.debug=yes --traceback| grep -v 'add changeset'
   pulling from server
   query 1; heads
-  1024 changesets found
-  128 changesets found
-  64 changesets found
-  16 changesets found
-  2 changesets found
-  1 changesets found
+  1024 changesets found in caches
+  128 changesets found in caches
+  64 changesets found in caches
+  16 changesets found in caches
+  2 changesets found in caches
+  1 changesets found in caches
   listing keys for "bookmarks"
   bundle2-output-bundle: "HG20", 9 parts total
   bundle2-output: start emission of HG20 stream
@@ -829,6 +873,10 @@
   new changesets 1ea73414a91b:f864bc82f6a2
   (run 'hg update' to get a working copy)
 
+  $ mv newbundles oldbundles
+  $ ls -1 server/.hg/cache/pullbundles > newbundles
+  $ diff -u oldbundles newbundles
+
   $ hg -R server log -G -T '{rev}:{node}\n' -r '1234+head()+(only(head(), 1234) and (merge() or branchpoint()))'
   o  2130:0f376356904fc8c1c6ceaac27990f2fd79b1f8c1
   :
@@ -848,22 +896,22 @@
   query 1; heads
   searching for changes
   all local heads known remotely
-  16 changesets found
-  29 changesets found
-  256 changesets found
-  256 changesets found
-  64 changesets found
-  8 changesets found
-  1 changesets found
-  4 changesets found
-  8 changesets found
-  16 changesets found
-  128 changesets found
-  64 changesets found
-  32 changesets found
-  8 changesets found
-  4 changesets found
-  2 changesets found
+  16 changesets found in caches
+  29 changesets found in caches
+  256 changesets found in caches
+  256 changesets found in caches
+  64 changesets found in caches
+  8 changesets found in caches
+  1 changesets found in caches
+  4 changesets found in caches
+  8 changesets found in caches
+  16 changesets found in caches
+  128 changesets found in caches
+  64 changesets found in caches
+  32 changesets found in caches
+  8 changesets found in caches
+  4 changesets found in caches
+  2 changesets found in caches
   listing keys for "bookmarks"
   bundle2-output-bundle: "HG20", 19 parts total
   bundle2-output: start emission of HG20 stream
@@ -1232,6 +1280,10 @@
   new changesets 17185c1c22f1:1dded5aafa0f
   (run 'hg heads' to see heads, 'hg merge' to merge)
 
+  $ mv newbundles oldbundles
+  $ ls -1 server/.hg/cache/pullbundles > newbundles
+  $ diff -u oldbundles newbundles
+
 different pull with a different client
 --------------------------------------
 
@@ -1252,9 +1304,9 @@
   $ hg -R client3 pull server -r 1789 --debug --config devel.bundle2.debug=yes | grep -v 'add changeset'
   pulling from server
   query 1; heads
-  1024 changesets found
+  1024 changesets found in caches
   227 changesets found
-  29 changesets found
+  29 changesets found in caches
   128 changesets found
   64 changesets found
   32 changesets found
@@ -1504,6 +1556,40 @@
   new changesets 1ea73414a91b:44e80141ad53
   (run 'hg update' to get a working copy)
 
+  $ mv newbundles oldbundles
+  $ ls -1 server/.hg/cache/pullbundles > newbundles
+  $ diff -u oldbundles newbundles
+  --- oldbundles	* (glob)
+  +++ newbundles	* (glob)
+  @@ -2,19 +2,26 @@
+   02-1dded5aafa0f8d548f6357cc2f8882dcc4489fbf-0000001516skip-0000000002size.hg
+   02-2dc4f1ab9029719714b8e0dde8e3725a5bb28472-0000001408skip-0000000064size.hg
+   02-2f0e261a08964bc1c607c0eda4978364c22a9b94-0000001504skip-0000000008size.hg
+  +02-44e80141ad530a2aa085e9bd9b5311b57eff72ff-0000001522skip-0000000001size.hg
+   02-467b6e370e816747e27de0d0b9237f4090a33656-0000001152skip-0000000064size.hg
+   02-4a6d0f7d07d060b026d9fc690cd89cd26af96e42-0000001264skip-0000000016size.hg
+   02-540f762640ee62ca597ece26af725e6357e82805-0000000000skip-0000001024size.hg
+  +02-59e60b258b18cde1e931cf30ce4ae62b49e37abd-0000001520skip-0000000002size.hg
+   02-5eaa75df73c454c1afff722301a4c73e897de94d-0000001256skip-0000000008size.hg
+   02-63ded94ceab180ac2fa13e1f0beeb4d2265998a3-0000001232skip-0000000002size.hg
+  +02-7a55a4d5ce324910842c893b56173cf2a847cb9d-0000001472skip-0000000032size.hg
+   02-7f3a79522d6e904d52aea07c71e6cb612667e8f4-0000001216skip-0000000016size.hg
+  +02-a4ab7df9d74053fb819c8a1c6a48ad605cc05f8a-0000001504skip-0000000016size.hg
+   02-b2d350c94c26edbb783aaa21fc24f1fc65c30e74-0000001536skip-0000000256size.hg
+   02-bbd293bd171fd5b711d428db46940a72eca7a40f-0000001280skip-0000000128size.hg
+  +02-c12927fef661d2463043347101b90067c2961333-0000001280skip-0000000128size.hg
+   02-ca970a853ea24846035ccb324cc8de49ef768748-0000001252skip-0000000004size.hg
+   02-d1807e3513890ac71c2e8d10e9dc9a5b58b15d4b-0000001251skip-0000000001size.hg
+  +02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001024skip-0000000227size.hg
+   02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001235skip-0000000016size.hg
+   02-da87a81c5310760f414a933e6550b7e8e60cf241-0000001792skip-0000000064size.hg
+   02-dba2fddbf3c28198659046674a512afd616a1519-0000001472skip-0000000032size.hg
+  +02-dc714c3a5d080165292ba99b097567d0b95e5756-0000001408skip-0000000064size.hg
+   02-e469a7aa5cce57653b6b02ff46c80b2d94d62629-0000000899skip-0000000029size.hg
+   02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001280skip-0000000256size.hg
+   02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg
+  [1]
+
   $ hg -R server log -G -T '{rev}:{node}\n' -r '1789+head()+parents(roots(only(head(), 1789)))+(only(head(), 1789) and (merge() or branchpoint()))'
   o  2130:0f376356904fc8c1c6ceaac27990f2fd79b1f8c1
   :
@@ -1523,19 +1609,19 @@
   1 changesets found
   4 changesets found
   8 changesets found
-  256 changesets found
-  64 changesets found
-  8 changesets found
-  1 changesets found
-  4 changesets found
-  8 changesets found
-  16 changesets found
-  128 changesets found
-  64 changesets found
-  32 changesets found
-  8 changesets found
-  4 changesets found
-  2 changesets found
+  256 changesets found in caches
+  64 changesets found in caches
+  8 changesets found in caches
+  1 changesets found in caches
+  4 changesets found in caches
+  8 changesets found in caches
+  16 changesets found in caches
+  128 changesets found in caches
+  64 changesets found in caches
+  32 changesets found in caches
+  8 changesets found in caches
+  4 changesets found in caches
+  2 changesets found in caches
   listing keys for "bookmarks"
   bundle2-output-bundle: "HG20", 19 parts total
   bundle2-output: start emission of HG20 stream
@@ -1901,3 +1987,33 @@
   updating the branch cache
   new changesets c232505f58fd:1dded5aafa0f
   (run 'hg heads' to see heads, 'hg merge' to merge)
+
+  $ mv newbundles oldbundles
+  $ ls -1 server/.hg/cache/pullbundles > newbundles
+  $ diff -u oldbundles newbundles
+  --- oldbundles	* (glob)
+  +++ newbundles	* (glob)
+  @@ -1,5 +1,6 @@
+   02-0f376356904fc8c1c6ceaac27990f2fd79b1f8c1-0000001856skip-0000000008size.hg
+   02-1dded5aafa0f8d548f6357cc2f8882dcc4489fbf-0000001516skip-0000000002size.hg
+  +02-1ed78f99f705cb819a02f1227c217728d008e461-0000001524skip-0000000004size.hg
+   02-2dc4f1ab9029719714b8e0dde8e3725a5bb28472-0000001408skip-0000000064size.hg
+   02-2f0e261a08964bc1c607c0eda4978364c22a9b94-0000001504skip-0000000008size.hg
+   02-44e80141ad530a2aa085e9bd9b5311b57eff72ff-0000001522skip-0000000001size.hg
+  @@ -15,6 +16,7 @@
+   02-b2d350c94c26edbb783aaa21fc24f1fc65c30e74-0000001536skip-0000000256size.hg
+   02-bbd293bd171fd5b711d428db46940a72eca7a40f-0000001280skip-0000000128size.hg
+   02-c12927fef661d2463043347101b90067c2961333-0000001280skip-0000000128size.hg
+  +02-c232505f58fdf70bcf5f6ab6a555f23ffc74f761-0000001523skip-0000000001size.hg
+   02-ca970a853ea24846035ccb324cc8de49ef768748-0000001252skip-0000000004size.hg
+   02-d1807e3513890ac71c2e8d10e9dc9a5b58b15d4b-0000001251skip-0000000001size.hg
+   02-d83212ecaa436c80d6113cf915ba35e2db787e79-0000001024skip-0000000227size.hg
+  @@ -24,6 +26,7 @@
+   02-dc714c3a5d080165292ba99b097567d0b95e5756-0000001408skip-0000000064size.hg
+   02-e469a7aa5cce57653b6b02ff46c80b2d94d62629-0000000899skip-0000000029size.hg
+   02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001280skip-0000000256size.hg
+  +02-e74670ea99533967c5d90da3ddbc0318cc1fd502-0000001528skip-0000000008size.hg
+   02-ee2deecf044fa5583f66188c9177b0f13332adc2-0000001024skip-0000000128size.hg
+   02-f864bc82f6a2f2ecb49b83722e0895f9d657b0dd-0000001234skip-0000000001size.hg
+   02-fb6c210a224903e81e5a8d2ee099cb0c9526ba8c-0000001512skip-0000000004size.hg
+  [1]