Cli test passing

app/lib/pulling.js

@@ -99,44 +99,73 @@ module.exports = {
    * @param dao An abstract layer to retrieve peers data (blocks).
    */
   pull: (conf, dao) => co(function *() {
-
-    let forks = [];
     let localCurrent = yield dao.localCurrent();
-    let peers = yield dao.remotePeers();
-    // Try to get new legit blocks for local blockchain
-    for (const peer of peers) {
+    const forks = [];
+
+    const applyCoroutine = (peer, blocks) => co(function*() {
       let shortPubkey = peer.pubkey.substr(0, 6);
-      let remoteNext;
-      if (localCurrent) {
-        remoteNext = yield dao.getRemoteBlock(peer, localCurrent.number + 1);
-      } else {
-        remoteNext = yield dao.getRemoteBlock(peer, 0);
-      }
-      if (remoteNext) {
+      if (blocks.length > 0) {
+        logger.debug("Applying from " + blocks[0].number);
         let isFork = localCurrent
-            && !(remoteNext.previousHash == localCurrent.hash && remoteNext.number == localCurrent.number + 1);
+            && !(blocks[0].previousHash == localCurrent.hash
+            && blocks[0].number == localCurrent.number + 1);
         if (!isFork) {
           logger.debug('Peer %s is on same blockchain', shortPubkey);
-          let appliedSuccessfully;
-          do {
-            yield dao.applyMainBranch(remoteNext);
+          yield dao.applyMainBranch(blocks);
           localCurrent = yield dao.localCurrent();
-            appliedSuccessfully = localCurrent.number == remoteNext.number && localCurrent.hash == remoteNext.hash;
-            remoteNext = yield dao.getRemoteBlock(peer, localCurrent.number + 1);
-          } while (appliedSuccessfully && remoteNext);
+          const appliedSuccessfully = localCurrent.number == blocks[blocks.length - 1].number
+                                  && localCurrent.hash == blocks[blocks.length - 1].hash;
+          return appliedSuccessfully;
         } else {
           logger.debug('Peer %s has forked', shortPubkey);
           let remoteCurrent = yield dao.remoteCurrent(peer);
           forks.push({
             peer: peer,
-            block: remoteNext,
+            block: blocks[0],
             current: remoteCurrent
           });
         }
+      }
+      return true;
+    });
+
+    const downloadCoroutine = (peer, number) => co(function*() {
+      return yield dao.getRemoteBlock(peer, number);
+    });
+
+    const downloadChuncks = (peer) => co(function*() {
+      let blocksToApply = [];
+      const currentBlock = yield dao.localCurrent();
+      let currentChunckStart;
+      if (currentBlock) {
+        currentChunckStart = currentBlock.number + 1;
       } else {
-        logger.debug('Peer %s do not have next block #%s', shortPubkey, localCurrent.number + 1);
+        currentChunckStart = 0;
+      }
+      let res;
+      do {
+        logger.debug("dl starts from " + currentChunckStart);
+        if (blocksToApply.length > 0)
+          logger.debug("apply starts from " + blocksToApply[0].number);
+        res = yield {
+          applied: applyCoroutine(peer, blocksToApply),
+          downloaded: downloadCoroutine(peer, currentChunckStart)
+        };
+        blocksToApply = res.downloaded;
+        currentChunckStart += res.downloaded.length;
+        if (!res.applied) {
+          logger.error("Blocks were not applied.")
         }
+      } while (res.downloaded.length > 0 && res.applied);
+    });
+
+    let peers = yield dao.remotePeers();
+    // Try to get new legit blocks for local blockchain
+    const downloadChuncksTasks = [];
+    for (const peer of peers) {
+      downloadChuncksTasks.push(downloadChuncks(peer));
     }
+    yield downloadChuncksTasks;
     // Filter forks: do not include mirror peers (non-member peers)
     let memberForks = [];
     for (const fork of forks) {

app/lib/sync.js

@@ -103,7 +103,13 @@ function Synchroniser (server, host, port, conf, interactive) {
       lastBlock: null,
 
       // Get the local blockchain current block
-      localCurrent: () => dal.getCurrentBlockOrNull(),
+      localCurrent: () => co(function*() {
+        if (cautious) {
+          return yield dal.getCurrentBlockOrNull();
+        } else {
+          return this.lastBlock;
+        }
+      }),
 
       // Get the remote blockchain (bc) current block
       remoteCurrent: (peer) => Q.nfcall(peer.blockchain.current),
@@ -118,29 +124,33 @@ function Synchroniser (server, host, port, conf, interactive) {
 
       // Get block of given peer with given block number
       getRemoteBlock: (thePeer, number) => co(function *() {
+        let blocks = [];
         if (number <= to) {
-          let block = null;
+          const nextChunck = Math.min(to - number + 1, CONST_BLOCKS_CHUNK);
+
           try {
-            block = yield Q.nfcall(thePeer.blockchain.block, number);
-            Transaction.statics.setIssuers(block.transactions);
+            blocks = yield Q.nfcall(thePeer.blockchain.blocks, nextChunck, number);
             watcher.downloadPercent(Math.floor(number / to * 100));
-            return block;
           } catch (e) {
             if (e.httpCode != 404) {
               throw e;
             }
           }
-          return block;
-        } else {
-          return null;
         }
+        return blocks;
       }),
 
-      applyMainBranch: (block) => co(function *() {
-        const addedBlock = yield server.BlockchainService.submitBlock(block, cautious, constants.FORK_ALLOWED);
+      applyMainBranch: (blocks) => co(function *() {
+        if (cautious) {
+          for (const block of blocks) {
+            const addedBlock = yield server.BlockchainService.submitBlock(block, true, constants.FORK_ALLOWED);
             server.streamPush(addedBlock);
-        this.lastBlock = block;
-        watcher.appliedPercent(Math.floor(block.number / to * 100));
+          }
+        } else {
+          yield server.BlockchainService.saveBlocksInMainBranch(blocks);
+        }
+        this.lastBlock = blocks[blocks.length - 1];
+        watcher.appliedPercent(Math.floor(blocks[blocks.length - 1].number / to * 100));
       }),
 
       // Eventually remove forks later on
@@ -150,9 +160,7 @@ function Synchroniser (server, host, port, conf, interactive) {
       isMemberPeer: (thePeer) => co(function *() {
         let idty = yield dal.getWrittenIdtyByPubkey(thePeer.pubkey);
         return (idty && idty.member) || false;
-      }),
-
-      downloadBlocks: (thePeer, fromNumber, count) => Q.nfcall(thePeer.blockchain.blocks, count, fromNumber)
+      })
     });
 
     const logInterval = setInterval(() => logRemaining(to), EVAL_REMAINING_INTERVAL);

app/service/PeeringService.js

@@ -449,17 +449,18 @@ function PeeringService(server) {
                 try {
                   block = yield Q.nfcall(thePeer.blockchain.block, number);
                   Transaction.statics.setIssuers(block.transactions);
-                  return block;
+                  return [block];
                 } catch (e) {
                   if (e.httpCode != 404) {
                     throw e;
                   }
                 }
-                return block;
+                return [];
               }),
 
               // Simulate the adding of a single new block on local blockchain
-              applyMainBranch: (block) => co(function *() {
+              applyMainBranch: (blocks) => co(function *() {
+                const block = blocks[0];
                 let addedBlock = yield server.BlockchainService.submitBlock(block, true, constants.FORK_ALLOWED);
                 server.streamPush(addedBlock);
               }),

test/integration/cli.js

@@ -25,14 +25,16 @@ describe("CLI", function() {
 
   it('sync 2200 blocks (fast)', () => co(function*() {
     yield execute(['reset', 'data']);
-    yield execute(['sync', 'duniter.org', '8999', '2200']);
+    yield execute(['sync', 'duniter.org', '8999', '2200', '--nocautious', '--nointeractive']);
     const res = yield execute(['export-bc', '--nostdout']);
+    res[res.length - 1].should.have.property('number').equal(2200);
     res.should.have.length(2200 + 1);
   }));
 
   it('sync 5 blocks (cautious)', () => co(function*() {
     yield execute(['sync', 'duniter.org', '8999', '2204', '--nointeractive']);
     const res = yield execute(['export-bc', '--nostdout']);
+    res[res.length - 1].should.have.property('number').equal(2204);
     res.should.have.length(2204 + 1);
   }));