[fix] #763 Now trims the INDEX continuously

app/lib/computation/blockchainContext.js

@@ -351,7 +351,21 @@ function BlockchainContext() {
     // Create/Update nodes in wotb
     yield that.updateMembers(block);
 
-    yield dal.trimIndexes(block, conf);
+    const TAIL = yield dal.bindexDAL.tail();
+    const bindexSize = [
+      block.issuersCount,
+      block.issuersFrame,
+      conf.medianTimeBlocks,
+      conf.dtDiffEval
+    ].reduce((max, value) => {
+      return Math.max(max, value);
+    }, 0);
+    const MAX_BINDEX_SIZE = 2*bindexSize;
+    const currentSize = indexes.HEAD.number - TAIL.number + 1;
+    if (currentSize > MAX_BINDEX_SIZE) {
+      yield dal.trimIndexes(indexes.HEAD.number - MAX_BINDEX_SIZE);
+    }
+
     yield updateBlocksComputedVars(current, block);
     // Saves the block (DAL)
     yield dal.saveBlock(block);

app/lib/dal/fileDAL.js

@@ -571,9 +571,12 @@ function FileDAL(params) {
     }
   });
 
-  this.trimIndexes = (block, conf) => co(function*() {
-    // TODO: trim should be done on a fork window size
-    // yield that.cindexDAL.trimExpiredCerts();
+  this.trimIndexes = (maxNumber) => co(function*() {
+    yield that.bindexDAL.trimBlocks(maxNumber);
+    yield that.iindexDAL.trimRecords(maxNumber);
+    yield that.mindexDAL.trimRecords(maxNumber);
+    yield that.cindexDAL.trimExpiredCerts(maxNumber);
+    yield that.sindexDAL.trimConsumedSource(maxNumber);
     return true;
   });
 

app/lib/dal/sqliteDAL/AbstractIndex.js

+/**
+ * Created by cgeek on 22/08/15.
+ */
+
+const _ = require('underscore');
+const co = require('co');
+const indexer = require('../../dup/indexer');
+
+module.exports = AbstractIndex;
+
+function AbstractIndex() {
+
+  "use strict";
+
+  const that = this;
+
+  this.trimRecords = (belowNumber) => co(function*() {
+    const belowRecords = yield that.query('SELECT * FROM ' + that.table + ' WHERE CAST(written_on as int) < ?', [belowNumber]);
+    const reducedByPub = indexer.DUP_HELPERS.reduceBy(belowRecords, ['pub']);
+    for (const rec of reducedByPub) {
+      const recordsOfPub = yield that.query('SELECT * FROM ' + that.table + ' WHERE pub = ?', [rec.pub]);
+      const toReduce = _.filter(recordsOfPub, (rec) => parseInt(rec.written_on) < belowNumber);
+      if (toReduce.length) {
+        // Clean the records in the DB
+        yield that.exec('DELETE FROM ' + that.table + ' WHERE pub = \'' + rec.pub + '\'');
+        const nonReduced = _.filter(recordsOfPub, (rec) => parseInt(rec.written_on) >= belowNumber);
+        const reduced = indexer.DUP_HELPERS.reduce(toReduce);
+        // Persist
+        yield that.insertBatch([reduced].concat(nonReduced));
+      }
+    }
+  });
+}
\ No newline at end of file

app/lib/dal/sqliteDAL/index/BIndexDAL.js

@@ -86,6 +86,14 @@ function BIndexDAL(driver) {
     return headRecords[0];
   });
 
+  /**
+   * Get the last record available in bindex
+   */
+  this.tail = () => co(function*() {
+    const tailRecords = yield that.query('SELECT * FROM ' + that.table + ' ORDER BY number ASC LIMIT 1', []);
+    return tailRecords[0];
+  });
+
   /**
    * Get HEAD~n..m
    * @param n
@@ -97,4 +105,6 @@ function BIndexDAL(driver) {
   });
 
   this.removeBlock = (number) => that.exec('DELETE FROM ' + that.table + ' WHERE number = ' + number);
+
+  this.trimBlocks = (maxnumber) => that.exec('DELETE FROM ' + that.table + ' WHERE number < ' + maxnumber);
 }

app/lib/dal/sqliteDAL/index/CIndexDAL.js

@@ -6,6 +6,7 @@ const co = require('co');
 const constants = require('./../../../constants');
 const indexer = require('./../../../dup/indexer');
 const AbstractSQLite = require('./../AbstractSQLite');
+const AbstractIndex = require('./../AbstractIndex');
 
 module.exports = CIndexDAL;
 
@@ -14,6 +15,7 @@ function CIndexDAL(driver) {
   "use strict";
 
   AbstractSQLite.call(this, driver);
+  AbstractIndex.call(this, driver);
 
   const that = this;
 
@@ -64,8 +66,8 @@ function CIndexDAL(driver) {
     return indexer.DUP_HELPERS.reduceBy(reducables, ['issuer', 'receiver', 'created_on']);
   });
 
-  this.trimExpiredCerts = () => co(function*() {
-    const toDelete = yield that.sqlFind({ expired_on: { $gt: 0 }});
+  this.trimExpiredCerts = (belowNumber) => co(function*() {
+    const toDelete = yield that.query('SELECT * FROM ' + that.table + ' WHERE expired_on > ? AND CAST(written_on as int) < ?', [0, belowNumber]);
     for (const row of toDelete) {
       yield that.exec("DELETE FROM " + that.table + " " +
         "WHERE issuer like '" + row.issuer + "' " +

app/lib/dal/sqliteDAL/index/IIndexDAL.js

@@ -6,6 +6,7 @@ const co = require('co');
 const _ = require('underscore');
 const indexer = require('./../../../dup/indexer');
 const AbstractSQLite = require('./../AbstractSQLite');
+const AbstractIndex = require('./../AbstractIndex');
 
 module.exports = IIndexDAL;
 
@@ -14,6 +15,7 @@ function IIndexDAL(driver) {
   "use strict";
 
   AbstractSQLite.call(this, driver);
+  AbstractIndex.call(this);
 
   const that = this;
 

app/lib/dal/sqliteDAL/index/MIndexDAL.js

@@ -5,6 +5,7 @@
 const co = require('co');
 const indexer = require('./../../../dup/indexer');
 const AbstractSQLite = require('./../AbstractSQLite');
+const AbstractIndex = require('./../AbstractIndex');
 
 module.exports = MIndexDAL;
 
@@ -13,6 +14,7 @@ function MIndexDAL(driver) {
   "use strict";
 
   AbstractSQLite.call(this, driver);
+  AbstractIndex.call(this);
 
   const that = this;
 

app/lib/dal/sqliteDAL/index/SIndexDAL.js

@@ -96,4 +96,15 @@ function SIndexDAL(driver) {
     const filtered = _.filter(sources, (src) => !src.consumed);
     return _.sortBy(filtered, (row) => row.type == 'D' ? 0 : 1);
   });
+
+  this.trimConsumedSource = (belowNumber) => co(function*() {
+    const toDelete = yield that.query('SELECT * FROM ' + that.table + ' WHERE consumed AND CAST(written_on as int) < ?', [belowNumber]);
+    for (const row of toDelete) {
+      const sql = "DELETE FROM " + that.table + " " +
+        "WHERE identifier like '" + row.identifier + "' " +
+        (toDelete.tx ? 'AND tx like = \'' + toDelete.tx + '\'' : 'AND tx IS NULL ') +
+        "AND pos = " + row.pos;
+      yield that.exec(sql);
+    }
+  });
 }

test/dal/triming.js

+"use strict";
+const co = require('co');
+const _ = require('underscore');
+const should = require('should');
+const FileDAL = require('../../app/lib/dal/fileDAL');
+const dir = require('../../app/lib/system/directory');
+const constants = require('../../app/lib/constants');
+const indexer = require('../../app/lib/dup/indexer');
+const toolbox = require('../integration/tools/toolbox');
+const limiter = require('../../app/lib/system/limiter');
+
+let dal;
+
+describe("Triming", function(){
+
+  before(() => co(function *() {
+    dal = FileDAL(yield dir.getHomeParams(true, 'db0'));
+    yield dal.init();
+    limiter.noLimit();
+  }));
+
+  it('should be able to feed the bindex', () => co(function *() {
+    yield dal.bindexDAL.insertBatch([
+      { number: 121, version: 6, bsize: 0, hash: "HASH", issuer: "ISSUER", time: 0, membersCount: 3, issuersCount: 2, issuersFrame: 1, issuersFrameVar: 2, avgBlockSize: 0, medianTime: 1482500000, dividend: 100, mass: 300, unitBase: 2, powMin: 70, udTime: 0, diffNumber: 5, speed: 1.0 },
+      { number: 122, version: 6, bsize: 0, hash: "HASH", issuer: "ISSUER", time: 0, membersCount: 3, issuersCount: 2, issuersFrame: 1, issuersFrameVar: 2, avgBlockSize: 0, medianTime: 1482500000, dividend: 100, mass: 300, unitBase: 2, powMin: 70, udTime: 0, diffNumber: 5, speed: 1.0 },
+      { number: 123, version: 6, bsize: 0, hash: "HASH", issuer: "ISSUER", time: 0, membersCount: 3, issuersCount: 2, issuersFrame: 1, issuersFrameVar: 2, avgBlockSize: 0, medianTime: 1482500000, dividend: 100, mass: 300, unitBase: 2, powMin: 70, udTime: 0, diffNumber: 5, speed: 1.0 },
+      { number: 124, version: 6, bsize: 0, hash: "HASH", issuer: "ISSUER", time: 0, membersCount: 3, issuersCount: 2, issuersFrame: 1, issuersFrameVar: 2, avgBlockSize: 0, medianTime: 1482500000, dividend: 100, mass: 300, unitBase: 2, powMin: 70, udTime: 0, diffNumber: 5, speed: 1.0 },
+      { number: 125, version: 6, bsize: 0, hash: "HASH", issuer: "ISSUER", time: 0, membersCount: 3, issuersCount: 2, issuersFrame: 1, issuersFrameVar: 2, avgBlockSize: 0, medianTime: 1482500000, dividend: 100, mass: 300, unitBase: 2, powMin: 70, udTime: 0, diffNumber: 5, speed: 1.0 }
+    ]);
+  }));
+
+  it('should have bindex head(1) = 125', () => co(function *() {
+    const head = yield dal.bindexDAL.head(1);
+    head.should.have.property('number').equal(125);
+  }));
+
+  it('should have bindex range(1, 3) = 125, 124, 123', () => co(function *() {
+    const range = yield dal.bindexDAL.range(1,3);
+    range.should.have.length(3);
+    range[0].should.have.property('number').equal(125);
+    range[1].should.have.property('number').equal(124);
+    range[2].should.have.property('number').equal(123);
+  }));
+
+  it('should be able to feed the iindex', () => co(function *() {
+    yield dal.iindexDAL.insertBatch([
+      { op: 'CREATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', uid: 'cat', created_on: '121-H', written_on: '122-H', member: true,  wasMember: true, kick: false },
+      { op: 'UPDATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', uid: null,  created_on: '121-H', written_on: '123-H', member: null,  wasMember: null, kick: true },
+      { op: 'UPDATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', uid: null,  created_on: '121-H', written_on: '124-H', member: false, wasMember: null, kick: false },
+    ]);
+    let lignes = yield dal.iindexDAL.reducable('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+    lignes.should.have.length(3);
+    indexer.DUP_HELPERS.reduce(lignes).should.have.property('member').equal(false);
+  }));
+
+  it('should be able to trim the iindex', () => co(function *() {
+    // Triming
+    yield dal.trimIndexes(124);
+    const lignes = yield dal.iindexDAL.reducable('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+    lignes.should.have.length(2);
+    indexer.DUP_HELPERS.reduce(lignes).should.have.property('member').equal(false);
+  }));
+
+  it('triming again the iindex should have no effet', () => co(function *() {
+    // Triming
+    yield dal.trimIndexes(124);
+    const lignes = yield dal.iindexDAL.reducable('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+    lignes.should.have.length(2);
+    indexer.DUP_HELPERS.reduce(lignes).should.have.property('member').equal(false);
+  }));
+
+  it('should be able to feed the mindex', () => co(function *() {
+    yield dal.mindexDAL.insertBatch([
+      { op: 'CREATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', created_on: '121-H', written_on: '122-H', expires_on: 1000, expired_on: null },
+      { op: 'UPDATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', created_on: '121-H', written_on: '123-H', expires_on: 1200, expired_on: null },
+      { op: 'UPDATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', created_on: '121-H', written_on: '124-H', expires_on: null, expired_on: null },
+      { op: 'UPDATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', created_on: '121-H', written_on: '125-H', expires_on: 1400, expired_on: null },
+    ]);
+    const lignes = yield dal.mindexDAL.reducable('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+    lignes.should.have.length(4);
+    indexer.DUP_HELPERS.reduce(lignes).should.have.property('expires_on').equal(1400);
+  }));
+
+  it('should be able to trim the mindex', () => co(function *() {
+    // Triming
+    yield dal.trimIndexes(124);
+    const lignes = yield dal.mindexDAL.reducable('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+    lignes.should.have.length(3);
+    indexer.DUP_HELPERS.reduce(lignes).should.have.property('expires_on').equal(1400);
+  }));
+
+  it('should be able to feed the cindex', () => co(function *() {
+    yield dal.cindexDAL.insertBatch([
+      { op: 'CREATE', issuer: 'HgTT', receiver: 'DNan', created_on: '121-H', written_on: '126-H', expires_on: 1000, expired_on: null },
+      { op: 'UPDATE', issuer: 'HgTT', receiver: 'DNan', created_on: '121-H', written_on: '126-H', expires_on: null, expired_on: 3000 },
+      { op: 'CREATE', issuer: 'DNan', receiver: 'HgTT', created_on: '125-H', written_on: '126-H', expires_on: null, expired_on: null },
+    ]);
+    (yield dal.cindexDAL.sqlFind({ issuer: 'HgTT' })).should.have.length(2);
+    (yield dal.cindexDAL.sqlFind({ issuer: 'DNan' })).should.have.length(1);
+  }));
+
+  it('should be able to trim the cindex', () => co(function *() {
+    // Triming
+    yield dal.trimIndexes(127);
+    (yield dal.cindexDAL.sqlFind({ issuer: 'HgTT' })).should.have.length(0);
+    // { op: 'UPDATE', issuer: 'DNan', receiver: 'HgTT', created_on: '125-H', written_on: '126-H', expires_on: 3600, expired_on: null },/**/
+    (yield dal.cindexDAL.sqlFind({ issuer: 'DNan' })).should.have.length(1);
+  }));
+
+  it('should be able to feed the sindex', () => co(function *() {
+    yield dal.sindexDAL.insertBatch([
+      { op: 'CREATE', identifier: 'SOURCE_1', pos: 4, written_on: '126-H', written_time: 2000, consumed: false },
+      { op: 'UPDATE', identifier: 'SOURCE_1', pos: 4, written_on: '139-H', written_time: 4500, consumed: true },
+      { op: 'CREATE', identifier: 'SOURCE_2', pos: 4, written_on: '126-H', written_time: 2000, consumed: false },
+      { op: 'CREATE', identifier: 'SOURCE_3', pos: 4, written_on: '126-H', written_time: 2000, consumed: false },
+    ]);
+    (yield dal.sindexDAL.sqlFind({ identifier: 'SOURCE_1' })).should.have.length(2);
+    (yield dal.sindexDAL.sqlFind({ pos: 4 })).should.have.length(4);
+  }));
+
+  it('should be able to trim the sindex', () => co(function *() {
+    // Triming
+    yield dal.trimIndexes(140);
+    (yield dal.sindexDAL.sqlFind({ identifier: 'SOURCE_1' })).should.have.length(0);
+    (yield dal.sindexDAL.sqlFind({ pos: 4 })).should.have.length(2);
+  }));
+
+  it('should be able to trim the bindex', () => co(function *() {
+    // Triming
+    const server = (yield toolbox.simpleNodeWith2Users({
+      pair: {
+        pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd',
+        sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'
+      },
+      participate: false,
+      sigQty: 1,
+      dtDiffEval: 2,
+      medianTimeBlocks: 3
+    })).s1;
+    // const s1 = server.s1;
+    for (const i of new Array(13)) {
+      yield server.commit();
+    }
+    (yield server.dal.bindexDAL.head(1)).should.have.property('number').equal(12);
+    (yield server.dal.bindexDAL.head(13)).should.have.property('number').equal(0);
+    yield server.commit();
+    should.not.exists(yield server.dal.bindexDAL.head(14)); // Trimed
+  }));
+});