From dfaed99eab01918757a3632cca5641aa768fd9d9 Mon Sep 17 00:00:00 2001
From: cgeek <cem.moreau@gmail.com>
Date: Tue, 24 Nov 2015 18:32:30 +0100
Subject: [PATCH] Fix #226 bad algorithm for memory cleaning of blocks

---
 app/lib/constants.js             |  3 +-
 app/lib/dal/fileDALs/BlockDAL.js | 63 +++++++++++++++-----------------
 server.js                        |  3 ++
 3 files changed, 35 insertions(+), 34 deletions(-)

diff --git a/app/lib/constants.js b/app/lib/constants.js
index 17df2d5aa..e4090714a 100644
--- a/app/lib/constants.js
+++ b/app/lib/constants.js
@@ -164,7 +164,8 @@ module.exports = {
   INVALIDATE_CORE_CACHE: true,
   WITH_SIGNATURES_AND_POW: true,
 
-  SAFE_FACTOR: 1.5,
+  MEMORY_CLEAN_INTERVAL: 60 * 60, // hourly
+  SAFE_FACTOR: 3,
   BLOCKS_COLLECT_THRESHOLD: 30, // Blocks to collect from memory and persist
 
   setUDID2Format: function () {
diff --git a/app/lib/dal/fileDALs/BlockDAL.js b/app/lib/dal/fileDALs/BlockDAL.js
index d7f3cd49b..8f04a1723 100644
--- a/app/lib/dal/fileDALs/BlockDAL.js
+++ b/app/lib/dal/fileDALs/BlockDAL.js
@@ -86,7 +86,6 @@ function BlockDAL(loki, rootFS, getLowerWindowBlock) {
     }
     return co(function *() {
       let filesBlocks = yield Q.all(_.range(start, Math.min(lowerInLoki.number, end + 1)).map((number) => rootFS.readJSON(pathOfBlock(number) + blockFileName(number) + '.json')));
-      yield migrateOldBlocks();
       return filesBlocks.concat(lokiBlocks);
     });
   };
@@ -184,40 +183,38 @@ function BlockDAL(loki, rootFS, getLowerWindowBlock) {
     current = previousBlock;
   };
 
-  function migrateOldBlocks() {
-    return co(function *() {
-      let number = yield getLowerWindowBlock();
-      logger.debug("Clean some blocks from memory to disk...");
-      logger.debug("Lower block = %s", number);
-      let lowerInLoki = collection.chain().simplesort('number').limit(1).data()[0];
-      if (!lowerInLoki) {
-        return;
+  this.migrateOldBlocks = () => co(function *() {
+    let number = yield getLowerWindowBlock();
+    logger.debug("Clean some blocks from memory to disk...");
+    logger.debug("Lower block = %s", number);
+    let lowerInLoki = collection.chain().simplesort('number').limit(1).data()[0];
+    if (!lowerInLoki) {
+      return;
+    }
+    logger.debug("LastUD in loki = %s", that.lastBlockWithDividend().number);
+    logger.debug("Lower in loki = %s", lowerInLoki.number);
+    let deadBlocksInLoki = number - lowerInLoki.number;
+    logger.debug("Dead blocks = %s", deadBlocksInLoki);
+    if (deadBlocksInLoki >= constants.BLOCKS_COLLECT_THRESHOLD) {
+      let blocksToPersist = blocksDB.branchResultset().find({
+        $and: [{
+          number: { $gte: lowerInLoki.number }
+        }, {
+          number: { $lte: number }
+        }]
+      }).simplesort('number').data();
+      logger.debug("To store in files = %s to %s", blocksToPersist[0].number, blocksToPersist[blocksToPersist.length - 1].number);
+      for (let i = 0; i < blocksToPersist.length; i++) {
+        let block = blocksToPersist[i];
+        yield rootFS.makeTree(pathOfBlock(block.number));
+        yield rootFS.writeJSON(pathOfBlock(block.number) + blockFileName(block.number) + '.json', block);
+        collection.remove(block);
       }
+      lowerInLoki = collection.chain().simplesort('number').limit(1).data()[0];
+      logger.debug("Lower in loki now = %s", lowerInLoki.number);
       logger.debug("LastUD in loki = %s", that.lastBlockWithDividend().number);
-      logger.debug("Lower in loki = %s", lowerInLoki.number);
-      let deadBlocksInLoki = number - lowerInLoki.number;
-      logger.debug("Dead blocks = %s", deadBlocksInLoki);
-      if (deadBlocksInLoki >= constants.BLOCKS_COLLECT_THRESHOLD) {
-        let blocksToPersist = blocksDB.branchResultset().find({
-          $and: [{
-            number: { $gte: lowerInLoki.number }
-          }, {
-            number: { $lte: number }
-          }]
-        }).simplesort('number').data();
-        logger.debug("To store in files = %s to %s", blocksToPersist[0].number, blocksToPersist[blocksToPersist.length - 1].number);
-        for (let i = 0; i < blocksToPersist.length; i++) {
-          let block = blocksToPersist[i];
-          yield rootFS.makeTree(pathOfBlock(block.number));
-          yield rootFS.writeJSON(pathOfBlock(block.number) + blockFileName(block.number) + '.json', block);
-          collection.remove(block);
-        }
-        lowerInLoki = collection.chain().simplesort('number').limit(1).data()[0];
-        logger.debug("Lower in loki now = %s", lowerInLoki.number);
-        logger.debug("LastUD in loki = %s", that.lastBlockWithDividend().number);
-      }
-    });
-  }
+    }
+  });
 
   function getView() {
     let view;
diff --git a/server.js b/server.js
index 50f6e8ec7..22fd7dec8 100644
--- a/server.js
+++ b/server.js
@@ -167,6 +167,9 @@ function Server (dbConf, overrideConf) {
       },
       function (next){
         that.PeeringService.regularSyncBlock(next);
+      },
+      function (next){
+        that.BlockchainService.regularCleanMemory(next);
       }
     ], done);
   };
-- 
GitLab