diff --git a/app/lib/computation/blockchainContext.js b/app/lib/computation/blockchainContext.js
index a607ee597911d6977bfeaf165f331e6f1abce2a2..81fc1f8b3311a0b98dc6c27bfa316ce524673176 100644
--- a/app/lib/computation/blockchainContext.js
+++ b/app/lib/computation/blockchainContext.js
@@ -351,7 +351,21 @@ function BlockchainContext() {
// Create/Update nodes in wotb
yield that.updateMembers(block);
- yield dal.trimIndexes(block, conf);
+ const TAIL = yield dal.bindexDAL.tail();
+ const bindexSize = [
+ block.issuersCount,
+ block.issuersFrame,
+ conf.medianTimeBlocks,
+ conf.dtDiffEval
+ ].reduce((max, value) => {
+ return Math.max(max, value);
+ }, 0);
+ const MAX_BINDEX_SIZE = 2*bindexSize;
+ const currentSize = indexes.HEAD.number - TAIL.number + 1;
+ if (currentSize > MAX_BINDEX_SIZE) {
+ yield dal.trimIndexes(indexes.HEAD.number - MAX_BINDEX_SIZE);
+ }
+
yield updateBlocksComputedVars(current, block);
// Saves the block (DAL)
yield dal.saveBlock(block);
diff --git a/app/lib/dal/fileDAL.js b/app/lib/dal/fileDAL.js
index 4a2bfaaa53ca0baac98ee3189a636b3527f1f3e7..3b50fd666529b1e695e6508275b66a399b70aeba 100644
--- a/app/lib/dal/fileDAL.js
+++ b/app/lib/dal/fileDAL.js
@@ -571,9 +571,12 @@ function FileDAL(params) {
}
});
- this.trimIndexes = (block, conf) => co(function*() {
- // TODO: trim should be done on a fork window size
- // yield that.cindexDAL.trimExpiredCerts();
+ this.trimIndexes = (maxNumber) => co(function*() {
+ yield that.bindexDAL.trimBlocks(maxNumber);
+ yield that.iindexDAL.trimRecords(maxNumber);
+ yield that.mindexDAL.trimRecords(maxNumber);
+ yield that.cindexDAL.trimExpiredCerts(maxNumber);
+ yield that.sindexDAL.trimConsumedSource(maxNumber);
return true;
});
diff --git a/app/lib/dal/sqliteDAL/AbstractIndex.js b/app/lib/dal/sqliteDAL/AbstractIndex.js
new file mode 100644
index 0000000000000000000000000000000000000000..a9070a702191d9aecd1eb5b4d53dc17b1f0f7bf6
--- /dev/null
+++ b/app/lib/dal/sqliteDAL/AbstractIndex.js
@@ -0,0 +1,33 @@
+/**
+ * Created by cgeek on 22/08/15.
+ */
+
+const _ = require('underscore');
+const co = require('co');
+const indexer = require('../../dup/indexer');
+
+module.exports = AbstractIndex;
+
+function AbstractIndex() {
+
+ "use strict";
+
+ const that = this;
+
+ this.trimRecords = (belowNumber) => co(function*() {
+ const belowRecords = yield that.query('SELECT * FROM ' + that.table + ' WHERE CAST(written_on as int) < ?', [belowNumber]);
+ const reducedByPub = indexer.DUP_HELPERS.reduceBy(belowRecords, ['pub']);
+ for (const rec of reducedByPub) {
+ const recordsOfPub = yield that.query('SELECT * FROM ' + that.table + ' WHERE pub = ?', [rec.pub]);
+ const toReduce = _.filter(recordsOfPub, (rec) => parseInt(rec.written_on) < belowNumber);
+ if (toReduce.length) {
+ // Clean the records in the DB
+ yield that.exec('DELETE FROM ' + that.table + ' WHERE pub = \'' + rec.pub + '\'');
+ const nonReduced = _.filter(recordsOfPub, (rec) => parseInt(rec.written_on) >= belowNumber);
+ const reduced = indexer.DUP_HELPERS.reduce(toReduce);
+ // Persist
+ yield that.insertBatch([reduced].concat(nonReduced));
+ }
+ }
+ });
+}
\ No newline at end of file
diff --git a/app/lib/dal/sqliteDAL/index/BIndexDAL.js b/app/lib/dal/sqliteDAL/index/BIndexDAL.js
index 147c4e322f40d626af5cd0c141e27f0924255d73..dfc2cb3b03c7bd3a0b6c3caa19e777b723f52226 100644
--- a/app/lib/dal/sqliteDAL/index/BIndexDAL.js
+++ b/app/lib/dal/sqliteDAL/index/BIndexDAL.js
@@ -86,6 +86,14 @@ function BIndexDAL(driver) {
return headRecords[0];
});
+ /**
+ * Get the last record available in bindex
+ */
+ this.tail = () => co(function*() {
+ const tailRecords = yield that.query('SELECT * FROM ' + that.table + ' ORDER BY number ASC LIMIT 1', []);
+ return tailRecords[0];
+ });
+
/**
* Get HEAD~n..m
* @param n
@@ -97,4 +105,6 @@ function BIndexDAL(driver) {
});
this.removeBlock = (number) => that.exec('DELETE FROM ' + that.table + ' WHERE number = ' + number);
+
+ this.trimBlocks = (maxnumber) => that.exec('DELETE FROM ' + that.table + ' WHERE number < ' + maxnumber);
}
diff --git a/app/lib/dal/sqliteDAL/index/CIndexDAL.js b/app/lib/dal/sqliteDAL/index/CIndexDAL.js
index d90d793b6975242c3938aa21bea344aacaa8f330..b4692b61b2a73090853018f4d4d56c992b30cbf5 100644
--- a/app/lib/dal/sqliteDAL/index/CIndexDAL.js
+++ b/app/lib/dal/sqliteDAL/index/CIndexDAL.js
@@ -6,6 +6,7 @@ const co = require('co');
const constants = require('./../../../constants');
const indexer = require('./../../../dup/indexer');
const AbstractSQLite = require('./../AbstractSQLite');
+const AbstractIndex = require('./../AbstractIndex');
module.exports = CIndexDAL;
@@ -14,6 +15,7 @@ function CIndexDAL(driver) {
"use strict";
AbstractSQLite.call(this, driver);
+ AbstractIndex.call(this, driver);
const that = this;
@@ -64,8 +66,8 @@ function CIndexDAL(driver) {
return indexer.DUP_HELPERS.reduceBy(reducables, ['issuer', 'receiver', 'created_on']);
});
- this.trimExpiredCerts = () => co(function*() {
- const toDelete = yield that.sqlFind({ expired_on: { $gt: 0 }});
+ this.trimExpiredCerts = (belowNumber) => co(function*() {
+ const toDelete = yield that.query('SELECT * FROM ' + that.table + ' WHERE expired_on > ? AND CAST(written_on as int) < ?', [0, belowNumber]);
for (const row of toDelete) {
yield that.exec("DELETE FROM " + that.table + " " +
"WHERE issuer like '" + row.issuer + "' " +
diff --git a/app/lib/dal/sqliteDAL/index/IIndexDAL.js b/app/lib/dal/sqliteDAL/index/IIndexDAL.js
index 36620d906a3cb889e726b948b46b551513cc9a0a..69458dc9020f7169c19ea719a64be8040846d220 100644
--- a/app/lib/dal/sqliteDAL/index/IIndexDAL.js
+++ b/app/lib/dal/sqliteDAL/index/IIndexDAL.js
@@ -6,6 +6,7 @@ const co = require('co');
const _ = require('underscore');
const indexer = require('./../../../dup/indexer');
const AbstractSQLite = require('./../AbstractSQLite');
+const AbstractIndex = require('./../AbstractIndex');
module.exports = IIndexDAL;
@@ -14,6 +15,7 @@ function IIndexDAL(driver) {
"use strict";
AbstractSQLite.call(this, driver);
+ AbstractIndex.call(this);
const that = this;
diff --git a/app/lib/dal/sqliteDAL/index/MIndexDAL.js b/app/lib/dal/sqliteDAL/index/MIndexDAL.js
index 67579f8a89edecd9ab3f5d154ae8159535065100..de4a07d4745fbafecd4d9c913fbb3f007b3358f1 100644
--- a/app/lib/dal/sqliteDAL/index/MIndexDAL.js
+++ b/app/lib/dal/sqliteDAL/index/MIndexDAL.js
@@ -5,6 +5,7 @@
const co = require('co');
const indexer = require('./../../../dup/indexer');
const AbstractSQLite = require('./../AbstractSQLite');
+const AbstractIndex = require('./../AbstractIndex');
module.exports = MIndexDAL;
@@ -13,6 +14,7 @@ function MIndexDAL(driver) {
"use strict";
AbstractSQLite.call(this, driver);
+ AbstractIndex.call(this);
const that = this;
diff --git a/app/lib/dal/sqliteDAL/index/SIndexDAL.js b/app/lib/dal/sqliteDAL/index/SIndexDAL.js
index 3f28d6ba019cb7b2390d85ac835e8ac229065559..8e5a19a0e4ab4d6338738354b931b3442b70c20b 100644
--- a/app/lib/dal/sqliteDAL/index/SIndexDAL.js
+++ b/app/lib/dal/sqliteDAL/index/SIndexDAL.js
@@ -96,4 +96,15 @@ function SIndexDAL(driver) {
const filtered = _.filter(sources, (src) => !src.consumed);
return _.sortBy(filtered, (row) => row.type == 'D' ? 0 : 1);
});
+
+ this.trimConsumedSource = (belowNumber) => co(function*() {
+ const toDelete = yield that.query('SELECT * FROM ' + that.table + ' WHERE consumed AND CAST(written_on as int) < ?', [belowNumber]);
+ for (const row of toDelete) {
+ const sql = "DELETE FROM " + that.table + " " +
+ "WHERE identifier like '" + row.identifier + "' " +
+ (toDelete.tx ? 'AND tx like = \'' + toDelete.tx + '\'' : 'AND tx IS NULL ') +
+ "AND pos = " + row.pos;
+ yield that.exec(sql);
+ }
+ });
}
diff --git a/test/dal/triming.js b/test/dal/triming.js
new file mode 100644
index 0000000000000000000000000000000000000000..4fb4fc2583b21020ed2a1344b4010e68a97c9708
--- /dev/null
+++ b/test/dal/triming.js
@@ -0,0 +1,149 @@
+"use strict";
+const co = require('co');
+const _ = require('underscore');
+const should = require('should');
+const FileDAL = require('../../app/lib/dal/fileDAL');
+const dir = require('../../app/lib/system/directory');
+const constants = require('../../app/lib/constants');
+const indexer = require('../../app/lib/dup/indexer');
+const toolbox = require('../integration/tools/toolbox');
+const limiter = require('../../app/lib/system/limiter');
+
+let dal;
+
+describe("Triming", function(){
+
+ before(() => co(function *() {
+ dal = FileDAL(yield dir.getHomeParams(true, 'db0'));
+ yield dal.init();
+ limiter.noLimit();
+ }));
+
+ it('should be able to feed the bindex', () => co(function *() {
+ yield dal.bindexDAL.insertBatch([
+ { number: 121, version: 6, bsize: 0, hash: "HASH", issuer: "ISSUER", time: 0, membersCount: 3, issuersCount: 2, issuersFrame: 1, issuersFrameVar: 2, avgBlockSize: 0, medianTime: 1482500000, dividend: 100, mass: 300, unitBase: 2, powMin: 70, udTime: 0, diffNumber: 5, speed: 1.0 },
+ { number: 122, version: 6, bsize: 0, hash: "HASH", issuer: "ISSUER", time: 0, membersCount: 3, issuersCount: 2, issuersFrame: 1, issuersFrameVar: 2, avgBlockSize: 0, medianTime: 1482500000, dividend: 100, mass: 300, unitBase: 2, powMin: 70, udTime: 0, diffNumber: 5, speed: 1.0 },
+ { number: 123, version: 6, bsize: 0, hash: "HASH", issuer: "ISSUER", time: 0, membersCount: 3, issuersCount: 2, issuersFrame: 1, issuersFrameVar: 2, avgBlockSize: 0, medianTime: 1482500000, dividend: 100, mass: 300, unitBase: 2, powMin: 70, udTime: 0, diffNumber: 5, speed: 1.0 },
+ { number: 124, version: 6, bsize: 0, hash: "HASH", issuer: "ISSUER", time: 0, membersCount: 3, issuersCount: 2, issuersFrame: 1, issuersFrameVar: 2, avgBlockSize: 0, medianTime: 1482500000, dividend: 100, mass: 300, unitBase: 2, powMin: 70, udTime: 0, diffNumber: 5, speed: 1.0 },
+ { number: 125, version: 6, bsize: 0, hash: "HASH", issuer: "ISSUER", time: 0, membersCount: 3, issuersCount: 2, issuersFrame: 1, issuersFrameVar: 2, avgBlockSize: 0, medianTime: 1482500000, dividend: 100, mass: 300, unitBase: 2, powMin: 70, udTime: 0, diffNumber: 5, speed: 1.0 }
+ ]);
+ }));
+
+ it('should have bindex head(1) = 125', () => co(function *() {
+ const head = yield dal.bindexDAL.head(1);
+ head.should.have.property('number').equal(125);
+ }));
+
+ it('should have bindex range(1, 3) = 125, 124, 123', () => co(function *() {
+ const range = yield dal.bindexDAL.range(1,3);
+ range.should.have.length(3);
+ range[0].should.have.property('number').equal(125);
+ range[1].should.have.property('number').equal(124);
+ range[2].should.have.property('number').equal(123);
+ }));
+
+ it('should be able to feed the iindex', () => co(function *() {
+ yield dal.iindexDAL.insertBatch([
+ { op: 'CREATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', uid: 'cat', created_on: '121-H', written_on: '122-H', member: true, wasMember: true, kick: false },
+ { op: 'UPDATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', uid: null, created_on: '121-H', written_on: '123-H', member: null, wasMember: null, kick: true },
+ { op: 'UPDATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', uid: null, created_on: '121-H', written_on: '124-H', member: false, wasMember: null, kick: false },
+ ]);
+ let lignes = yield dal.iindexDAL.reducable('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+ lignes.should.have.length(3);
+ indexer.DUP_HELPERS.reduce(lignes).should.have.property('member').equal(false);
+ }));
+
+ it('should be able to trim the iindex', () => co(function *() {
+ // Triming
+ yield dal.trimIndexes(124);
+ const lignes = yield dal.iindexDAL.reducable('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+ lignes.should.have.length(2);
+ indexer.DUP_HELPERS.reduce(lignes).should.have.property('member').equal(false);
+ }));
+
+ it('triming again the iindex should have no effet', () => co(function *() {
+ // Triming
+ yield dal.trimIndexes(124);
+ const lignes = yield dal.iindexDAL.reducable('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+ lignes.should.have.length(2);
+ indexer.DUP_HELPERS.reduce(lignes).should.have.property('member').equal(false);
+ }));
+
+ it('should be able to feed the mindex', () => co(function *() {
+ yield dal.mindexDAL.insertBatch([
+ { op: 'CREATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', created_on: '121-H', written_on: '122-H', expires_on: 1000, expired_on: null },
+ { op: 'UPDATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', created_on: '121-H', written_on: '123-H', expires_on: 1200, expired_on: null },
+ { op: 'UPDATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', created_on: '121-H', written_on: '124-H', expires_on: null, expired_on: null },
+ { op: 'UPDATE', pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', created_on: '121-H', written_on: '125-H', expires_on: 1400, expired_on: null },
+ ]);
+ const lignes = yield dal.mindexDAL.reducable('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+ lignes.should.have.length(4);
+ indexer.DUP_HELPERS.reduce(lignes).should.have.property('expires_on').equal(1400);
+ }));
+
+ it('should be able to trim the mindex', () => co(function *() {
+ // Triming
+ yield dal.trimIndexes(124);
+ const lignes = yield dal.mindexDAL.reducable('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+ lignes.should.have.length(3);
+ indexer.DUP_HELPERS.reduce(lignes).should.have.property('expires_on').equal(1400);
+ }));
+
+ it('should be able to feed the cindex', () => co(function *() {
+ yield dal.cindexDAL.insertBatch([
+ { op: 'CREATE', issuer: 'HgTT', receiver: 'DNan', created_on: '121-H', written_on: '126-H', expires_on: 1000, expired_on: null },
+ { op: 'UPDATE', issuer: 'HgTT', receiver: 'DNan', created_on: '121-H', written_on: '126-H', expires_on: null, expired_on: 3000 },
+ { op: 'CREATE', issuer: 'DNan', receiver: 'HgTT', created_on: '125-H', written_on: '126-H', expires_on: null, expired_on: null },
+ ]);
+ (yield dal.cindexDAL.sqlFind({ issuer: 'HgTT' })).should.have.length(2);
+ (yield dal.cindexDAL.sqlFind({ issuer: 'DNan' })).should.have.length(1);
+ }));
+
+ it('should be able to trim the cindex', () => co(function *() {
+ // Triming
+ yield dal.trimIndexes(127);
+ (yield dal.cindexDAL.sqlFind({ issuer: 'HgTT' })).should.have.length(0);
+ // { op: 'UPDATE', issuer: 'DNan', receiver: 'HgTT', created_on: '125-H', written_on: '126-H', expires_on: 3600, expired_on: null },/**/
+ (yield dal.cindexDAL.sqlFind({ issuer: 'DNan' })).should.have.length(1);
+ }));
+
+ it('should be able to feed the sindex', () => co(function *() {
+ yield dal.sindexDAL.insertBatch([
+ { op: 'CREATE', identifier: 'SOURCE_1', pos: 4, written_on: '126-H', written_time: 2000, consumed: false },
+ { op: 'UPDATE', identifier: 'SOURCE_1', pos: 4, written_on: '139-H', written_time: 4500, consumed: true },
+ { op: 'CREATE', identifier: 'SOURCE_2', pos: 4, written_on: '126-H', written_time: 2000, consumed: false },
+ { op: 'CREATE', identifier: 'SOURCE_3', pos: 4, written_on: '126-H', written_time: 2000, consumed: false },
+ ]);
+ (yield dal.sindexDAL.sqlFind({ identifier: 'SOURCE_1' })).should.have.length(2);
+ (yield dal.sindexDAL.sqlFind({ pos: 4 })).should.have.length(4);
+ }));
+
+ it('should be able to trim the sindex', () => co(function *() {
+ // Triming
+ yield dal.trimIndexes(140);
+ (yield dal.sindexDAL.sqlFind({ identifier: 'SOURCE_1' })).should.have.length(0);
+ (yield dal.sindexDAL.sqlFind({ pos: 4 })).should.have.length(2);
+ }));
+
+ it('should be able to trim the bindex', () => co(function *() {
+ // Triming
+ const server = (yield toolbox.simpleNodeWith2Users({
+ pair: {
+ pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd',
+ sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'
+ },
+ participate: false,
+ sigQty: 1,
+ dtDiffEval: 2,
+ medianTimeBlocks: 3
+ })).s1;
+ // const s1 = server.s1;
+ for (const i of new Array(13)) {
+ yield server.commit();
+ }
+ (yield server.dal.bindexDAL.head(1)).should.have.property('number').equal(12);
+ (yield server.dal.bindexDAL.head(13)).should.have.property('number').equal(0);
+ yield server.commit();
+ should.not.exists(yield server.dal.bindexDAL.head(14)); // Trimed
+ }));
+});