diff --git a/app/lib/blockchainContext.js b/app/lib/blockchainContext.js index 786831d70b7211c83a302447d7a396a5e618fa88..712ef6d686743a141f89fa01311bcba959562aab 100644 --- a/app/lib/blockchainContext.js +++ b/app/lib/blockchainContext.js @@ -4,6 +4,7 @@ var _ = require('underscore'); var co = require('co'); var Q = require('q'); var sha1 = require('sha1'); +var moment = require('moment'); var rawer = require('./rawer'); var localValidator = require('./localValidator'); var globalValidator = require('./globalValidator'); @@ -176,13 +177,9 @@ function BlockchainContext(conf, dal) { // Create/Update certifications updateCertifications(block, next); }, - function (next) { - // Create/Update certifications - updateMemberships(block, next); - }, function (next){ // Save links - updateLinks(block, next, dal.getBlockOrNull.bind(dal)); + updateLinksForBlocks([block], dal.getBlockOrNull.bind(dal)).then(() => next()).catch(next); }, function (next){ // Compute obsolete links @@ -244,11 +241,13 @@ function BlockchainContext(conf, dal) { this.updateMembers = updateMembers; this.updateCertifications = updateCertifications; - this.updateMemberships = updateMemberships; - this.updateLinks = updateLinks; - this.updateTransactionSources = updateTransactionSources; this.computeObsoleteLinks = computeObsoleteLinks; this.computeObsoleteMemberships = computeObsoleteMemberships; + this.updateTransactionSourcesForBlocks = updateTransactionSourcesForBlocks; + this.updateCertificationsForBlocks = updateCertificationsForBlocks; + this.updateMembershipsForBlocks = updateMembershipsForBlocks; + this.updateLinksForBlocks = updateLinksForBlocks; + this.updateTransactionsForBlocks = updateTransactionsForBlocks; let cleanRejectedIdentities = (idty) => co(function *() { yield dal.removeUnWrittenWithPubkey(idty.pubkey); @@ -396,6 +395,15 @@ function BlockchainContext(conf, dal) { }); } + /** + * Historical method that takes certifications from a block and tries to either: + * * Update the certification found in the DB an set it as written + * * Create it if it does not exist + * + * Has a sibling method named 'updateCertificationsForBlocks'. + * @param block + * @param done + */ function updateCertifications (block, done) { async.forEachSeries(block.certifications, function(inlineCert, callback){ var cert = Certification.statics.fromInline(inlineCert); @@ -430,54 +438,6 @@ function BlockchainContext(conf, dal) { }, done); } - // TODO: no more needed - function updateMemberships (block, done) { - async.forEachSeries(['joiners', 'actives', 'leavers'], function (prop, callback1) { - async.forEach(block[prop], function(inlineJoin, callback){ - var ms = Membership.statics.fromInline(inlineJoin, prop == 'leavers' ? 'OUT' : 'IN'); - async.waterfall([ - function (next){ - dal.getWritten(ms.issuer, next); - }, - function (idty, next){ - if (!idty) { - var err = 'Could not find identity for membership of issuer ' + ms.issuer; - logger.error(err); - next(err); - return; - } - ms.userid = idty.uid; - ms.certts = idty.time; - next(); - } - ], callback); - }, callback1); - }, done); - } - - function updateLinks (block, done, getBlockOrNull) { - async.forEach(block.certifications, function(inlineCert, callback){ - var cert = Certification.statics.fromInline(inlineCert); - return co(function *() { - let tagBlock = block; - if (block.number > 0) { - tagBlock = yield getBlockOrNull(cert.block_number); - } - return dal.saveLink( - new Link({ - source: cert.from, - target: cert.to, - timestamp: tagBlock.medianTime, - block_number: block.number, - block_hash: block.hash, - obsolete: false - })); - }) - .then(_.partial(callback, null)) - .catch(callback); - }, done); - } - that.saveParametersForRootBlock = (block, done) => { if (block.parameters) { var sp = block.parameters.split(':'); @@ -521,16 +481,11 @@ function BlockchainContext(conf, dal) { var pubkey = idty.pubkey; async.waterfall([ function (nextOne){ - async.parallel({ - enoughLinks: function(callback2){ - that.checkHaveEnoughLinks(pubkey, {}, function (err) { - callback2(null, err); - }); - } - }, nextOne); + that.checkHaveEnoughLinks(pubkey, {}, function (err) { + nextOne(null, err); + }); }, - function (res, nextOne){ - var notEnoughLinks = res.enoughLinks; + function (notEnoughLinks, nextOne){ dal.setKicked(pubkey, new Identity(idty).getTargetHash(), notEnoughLinks ? true : false, nextOne); } ], callback); @@ -551,7 +506,7 @@ function BlockchainContext(conf, dal) { next(count < conf.sigQty && 'Key ' + target + ' does not have enough links (' + count + '/' + conf.sigQty + ')'); } ], done); - } + }; function computeObsoleteMemberships (block) { return dal.getMembershipExcludingBlock(block, conf.msValidity) @@ -624,6 +579,193 @@ function BlockchainContext(conf, dal) { }); } + /** + * New method for CREATING memberships found in blocks. + * Made for performance reasons, this method will batch insert all memberships at once. + * @param blocks + * @returns {*} + */ + function updateMembershipsForBlocks(blocks) { + return co(function *() { + let memberships = []; + let types = { + 'join': 'joiners', + 'active': 'actives', + 'leave': 'leavers' + }; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + _.keys(types).forEach(function(type){ + let msType = type == 'leave' ? 'out' : 'in'; + let field = types[type]; + let mss = block[field]; + for (let j = 0, len2 = mss.length; j < len2; j++) { + let msRaw = mss[j]; + var ms = Membership.statics.fromInline(msRaw, type == 'leave' ? 'OUT' : 'IN', block.currency); + ms.membership = msType.toUpperCase(); + ms.written = true; + ms.type = type; + ms.hash = String(sha1(ms.getRawSigned())).toUpperCase(); + ms.idtyHash = (sha1(ms.userid + moment(ms.certts).unix() + ms.issuer) + "").toUpperCase(); + memberships.push(ms); + } + }); + } + return dal.updateMemberships(memberships); + }); + } + + /** + * New method for CREATING links found in blocks. + * Made for performance reasons, this method will batch insert all links at once. + * @param blocks + * @param getBlockOrNull + * @returns {*} + */ + function updateLinksForBlocks(blocks, getBlockOrNull) { + return co(function *() { + let links = []; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + for (let j = 0, len2 = block.certifications.length; j < len2; j++) { + let inlineCert = block.certifications[j]; + let cert = Certification.statics.fromInline(inlineCert); + let tagBlock = block; + if (block.number > 0) { + tagBlock = yield getBlockOrNull(cert.block_number); + } + links.push({ + source: cert.from, + target: cert.to, + timestamp: tagBlock.medianTime, + block_number: block.number, + block_hash: block.hash, + obsolete: false + }); + } + } + return dal.updateLinks(links); + }); + } + + /** + * New method for CREATING transactions found in blocks. + * Made for performance reasons, this method will batch insert all transactions at once. + * @param blocks + * @returns {*} + */ + function updateTransactionsForBlocks(blocks) { + return co(function *() { + let txs = []; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + txs = txs.concat(block.transactions.map((tx) => { + _.extend(tx, { + block_number: block.number, + time: block.medianTime, + currency: block.currency, + written: true, + removed: false + }); + return new Transaction(tx); + })); + } + return dal.updateTransactions(txs); + }); + } + + /** + * New method for CREATING certifications found in blocks. + * Made for performance reasons, this method will batch insert all certifications at once. + * @param blocks + * @returns {*} + */ + function updateCertificationsForBlocks(blocks) { + return co(function *() { + let certs = []; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + for (let j = 0, len2 = block.certifications.length; j < len2; j++) { + let inlineCert = block.certifications[j]; + var cert = Certification.statics.fromInline(inlineCert); + let to = yield dal.getWrittenIdtyByPubkey(cert.to); + let to_uid = to.uid; + cert.target = new Identity(to).getTargetHash(); + let from = yield dal.getWrittenIdtyByPubkey(cert.from); + let from_uid = from.uid; + let existing = yield dal.existsCert(cert); + if (existing) { + cert = existing; + } + cert.written_block = block.number; + cert.written_hash = block.hash; + cert.from_uid = from_uid; + cert.to_uid = to_uid; + cert.linked = true; + certs.push(cert); + } + } + return dal.updateCertifications(certs); + }); + } + + /** + * New method for CREATING sources found in transactions of blocks. + * Made for performance reasons, this method will batch insert all sources at once. + * @param blocks + * @returns {*} + */ + function updateTransactionSourcesForBlocks(blocks) { + return co(function *() { + let sources = []; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + // Dividends + if (block.dividend) { + let idties = yield dal.getMembersP(); + for (let j = 0, len2 = idties.length; j < len2; j++) { + let idty = idties[j]; + sources.push({ + 'pubkey': idty.pubkey, + 'type': 'D', + 'number': block.number, + 'time': block.medianTime, + 'fingerprint': block.hash, + 'block_hash': block.hash, + 'amount': block.dividend, + 'consumed': false, + 'toConsume': false + }); + } + } + // Transactions + for (let j = 0, len2 = block.transactions.length; j < len2; j++) { + let json = block.transactions[j]; + let obj = json; + obj.version = 1; + obj.currency = block.currency; + obj.issuers = json.signatories; + let tx = new Transaction(obj); + let txObj = tx.getTransaction(); + let txHash = tx.getHash(true); + sources = sources.concat(txObj.inputs.map((input) => _.extend({ toConsume: true }, input))); + sources = sources.concat(txObj.outputs.map((output) => _.extend({ + toConsume: false + }, { + 'pubkey': output.pubkey, + 'type': 'T', + 'number': block.number, + 'block_hash': block.hash, + 'fingerprint': txHash, + 'amount': output.amount, + 'consumed': false + }))); + } + } + return dal.updateSources(sources); + }); + } + function deleteTransactions (block, done) { async.forEachSeries(block.transactions, function (json, callback) { var obj = json; diff --git a/app/lib/dal/fileDAL.js b/app/lib/dal/fileDAL.js index 79f6df9278fd7479c470b0bd432ce1e25fd41160..c94ebda8c6daad32e91c4826215275a7bd55a204 100644 --- a/app/lib/dal/fileDAL.js +++ b/app/lib/dal/fileDAL.js @@ -6,6 +6,7 @@ var fs = require('fs'); var qfs = require('q-io/fs'); var sha1 = require('sha1'); var path = require('path'); +var moment = require('moment'); var Configuration = require('../entity/configuration'); var Membership = require('../entity/membership'); var Merkle = require('../entity/merkle'); @@ -13,17 +14,9 @@ var Transaction = require('../entity/transaction'); var constants = require('../constants'); var ConfDAL = require('./fileDALs/confDAL'); var StatDAL = require('./fileDALs/statDAL'); -var CertDAL = require('./fileDALs/CertDAL'); -var TxsDAL = require('./fileDALs/TxsDAL'); -var SourcesDAL = require('./fileDALs/SourcesDAL'); -var LinksDAL = require('./fileDALs/LinksDAL'); -var MembershipDAL = require('./fileDALs/MembershipDAL'); -var IdentityDAL = require('./fileDALs/IdentityDAL'); var IndicatorsDAL = require('./fileDALs/IndicatorsDAL'); -var PeerDAL = require('./fileDALs/PeerDAL'); -var BlockDAL = require('./fileDALs/BlockDAL'); var CFSStorage = require('./fileDALs/AbstractCFS'); -var lokijs = require('lokijs'); +var sqlite3 = require("sqlite3").verbose(); var logger = require('../../lib/logger')('database'); const UCOIN_DB_NAME = 'ucoin'; @@ -32,65 +25,16 @@ module.exports = { memory: function(profile, home) { return getHomeFS(profile, true, home) .then(function(params) { - let loki = new lokijs(UCOIN_DB_NAME, { autosave: false }); - return Q(new FileDAL(profile, params.home, "", params.fs, null, 'fileDal', loki)); + let sqlite = new sqlite3.Database(':memory:'); + return Q(new FileDAL(profile, params.home, "", params.fs, null, 'fileDal', sqlite)); }); }, - file: function(profile, home, forConf) { + file: function(profile, home) { return getHomeFS(profile, false, home) .then(function(params) { - return Q.Promise(function(resolve, reject){ - let loki; - if (forConf) { - // Memory only service dals - loki = new lokijs('temp', { autosave: false }); - resolve(loki); - } else { - let lokiPath = path.join(params.home, UCOIN_DB_NAME + '.json'); - logger.debug('Loading DB at %s...', lokiPath); - co(function *() { - let rawDB; - try { - // Try to read database - rawDB = yield qfs.read(lokiPath); - // Check if content is standard JSON - JSON.parse(rawDB); - } catch (e) { - if (rawDB) { - logger.error('The database could not be loaded, it is probably corrupted due to some system fail.'); - logger.error('Please stop uCoin and re-sync it with another node of the network before restarting, using the following commands:'); - logger.error('> ucoind reset data'); - logger.error('> ucoind sync <some.ucoin.node> <port>'); - logger.error('> ucoind restart'); - // Dirty "won't go any further" - return Q.Promise((resolve) => null); - } - } - - return Q.Promise(function(resolve2){ - let lokiDB; - lokiDB = new lokijs(lokiPath, { - autoload: true, - autosave: true, - autosaveInterval: 30000, - adapter: { - loadDatabase: (dbname, callback) => { - callback(rawDB || null); - resolve2(lokiDB); - }, - saveDatabase: (dbname, dbstring, callback) => fs.writeFile(dbname, dbstring, callback) - } - }); - }); - }) - .then(resolve) - .catch(reject); - } - }) - .then(function(loki){ - loki.autosaveClearFlags(); - return new FileDAL(profile, params.home, "", params.fs, null, 'fileDal', loki); - }); + let sqlitePath = path.join(params.home, UCOIN_DB_NAME + '.db'); + let sqlite = new sqlite3.Database(sqlitePath); + return new FileDAL(profile, params.home, "", params.fs, null, 'fileDal', sqlite); }); }, FileDAL: FileDAL @@ -116,7 +60,7 @@ function getHomeFS(profile, isMemory, homePath) { }); } -function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { +function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, sqlite) { var that = this; @@ -128,26 +72,44 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { var rootPath = home; - let blocksCFS = require('../cfs')(rootPath, myFS); - // DALs this.confDAL = new ConfDAL(rootPath, myFS, parentFileDAL && parentFileDAL.confDAL.coreFS, that, CFSStorage); - this.peerDAL = new PeerDAL(loki); - this.blockDAL = new BlockDAL(loki, blocksCFS, getLowerWindowBlock); - this.sourcesDAL = new SourcesDAL(loki); - this.txsDAL = new TxsDAL(loki); + this.peerDAL = new (require('./sqliteDAL/PeerDAL'))(sqlite); + this.blockDAL = new (require('./sqliteDAL/BlockDAL'))(sqlite); + this.sourcesDAL = new (require('./sqliteDAL/SourcesDAL'))(sqlite); + this.txsDAL = new (require('./sqliteDAL/TxsDAL'))(sqlite); this.indicatorsDAL = new IndicatorsDAL(rootPath, myFS, parentFileDAL && parentFileDAL.indicatorsDAL.coreFS, that, CFSStorage); this.statDAL = new StatDAL(rootPath, myFS, parentFileDAL && parentFileDAL.statDAL.coreFS, that, CFSStorage); - this.linksDAL = new LinksDAL(loki); - this.idtyDAL = new IdentityDAL(loki); - this.certDAL = new CertDAL(loki); - this.msDAL = new MembershipDAL(loki); + this.linksDAL = new (require('./sqliteDAL/LinksDAL'))(sqlite); + this.idtyDAL = new (require('./sqliteDAL/IdentityDAL'))(sqlite); + this.certDAL = new (require('./sqliteDAL/CertDAL'))(sqlite); + this.msDAL = new (require('./sqliteDAL/MembershipDAL'))(sqlite); this.newDals = { + 'blockDAL': that.blockDAL, + 'certDAL': that.certDAL, + 'msDAL': that.msDAL, + 'idtyDAL': that.idtyDAL, + 'sourcesDAL': that.sourcesDAL, + 'linksDAL': that.linksDAL, + 'txsDAL': that.txsDAL, 'peerDAL': that.peerDAL, 'indicatorsDAL': that.indicatorsDAL, 'confDAL': that.confDAL, - 'statDAL': that.statDAL + 'statDAL': that.statDAL, + 'ghostDAL': { + init: () => co(function *() { + + // Create extra views (useful for stats or debug) + return that.blockDAL.exec('BEGIN;' + + 'CREATE VIEW IF NOT EXISTS identities_pending AS SELECT * FROM idty WHERE NOT written;' + + 'CREATE VIEW IF NOT EXISTS certifications_pending AS SELECT * FROM cert WHERE NOT written;' + + 'CREATE VIEW IF NOT EXISTS transactions_pending AS SELECT * FROM txs WHERE NOT written;' + + 'CREATE VIEW IF NOT EXISTS transactions_desc AS SELECT * FROM txs ORDER BY time DESC;' + + 'CREATE VIEW IF NOT EXISTS forks AS SELECT * FROM block WHERE fork ORDER BY number DESC;' + + 'COMMIT;'); + }) + } }; var currency = ''; @@ -156,48 +118,12 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { return co(function *() { yield _.values(that.newDals).map((dal) => dal.init()); return that.loadConf(overrideConf, defaultConf); - }); + }) + .catch((err) => { + throw Error(err); + }); }; - function getLowerWindowBlock() { - return co(function *() { - let rootBlock = yield that.getRootBlock(); - if (!rootBlock) { - return -1; - } - let conf = getParameters(rootBlock); - let needToBeKeptBlocks = getMaxBlocksToStoreAsFile(conf); - let current = yield that.getCurrentBlockOrNull(); - let currentNumber = current ? current.number : -1; - return currentNumber - needToBeKeptBlocks; - }); - } - - function getParameters(block) { - var sp = block.parameters.split(':'); - let theConf = {}; - theConf.c = parseFloat(sp[0]); - theConf.dt = parseInt(sp[1]); - theConf.ud0 = parseInt(sp[2]); - theConf.sigDelay = parseInt(sp[3]); - theConf.sigValidity = parseInt(sp[4]); - theConf.sigQty = parseInt(sp[5]); - theConf.sigWoT = parseInt(sp[6]); - theConf.msValidity = parseInt(sp[7]); - theConf.stepMax = parseInt(sp[8]); - theConf.medianTimeBlocks = parseInt(sp[9]); - theConf.avgGenTime = parseInt(sp[10]); - theConf.dtDiffEval = parseInt(sp[11]); - theConf.blocksRot = parseInt(sp[12]); - theConf.percentRot = parseFloat(sp[13]); - theConf.currency = block.currency; - return theConf; - } - - function getMaxBlocksToStoreAsFile(aConf) { - return Math.floor(Math.max(aConf.dt / aConf.avgGenTime, aConf.medianTimeBlocks, aConf.dtDiffEval, aConf.blocksRot) * constants.SAFE_FACTOR); - } - this.getCurrency = function() { return currency; }; @@ -976,6 +902,7 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { var ms = Membership.statics.fromInline(msRaw, type == 'leave' ? 'OUT' : 'IN', that.getCurrency()); ms.type = type; ms.hash = String(sha1(ms.getRawSigned())).toUpperCase(); + ms.idtyHash = (sha1(ms.userid + moment(ms.certts).unix() + ms.issuer) + "").toUpperCase(); return that.msDAL.saveOfficialMS(msType, ms); }); }, Q()); @@ -1043,6 +970,26 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { return that.sourcesDAL.addSource('available', src.pubkey, src.type, src.number, src.fingerprint, src.amount, src.block_hash, src.time); }; + this.updateSources = function(sources) { + return that.sourcesDAL.updateBatchOfSources(sources); + }; + + this.updateCertifications = function(certs) { + return that.certDAL.updateBatchOfCertifications(certs); + }; + + this.updateMemberships = function(certs) { + return that.msDAL.updateBatchOfMemberships(certs); + }; + + this.updateLinks = function(certs) { + return that.linksDAL.updateBatchOfLinks(certs); + }; + + this.updateTransactions = function(txs) { + return that.txsDAL.updateBatchOfTxs(txs); + }; + this.officializeCertification = function(cert) { return that.certDAL.saveOfficial(cert); }; @@ -1090,17 +1037,17 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { this.unJoinIdentity = (ms) => co(function *() { yield that.idtyDAL.unJoinIdentity(ms); - that.msDAL.unwriteMS(ms); + yield that.msDAL.unwriteMS(ms); }); this.unRenewIdentity = (ms) => co(function *() { yield that.idtyDAL.unRenewIdentity(ms); - that.msDAL.unwriteMS(ms); + yield that.msDAL.unwriteMS(ms); }); this.unLeaveIdentity = (ms) => co(function *() { yield that.idtyDAL.unLeaveIdentity(ms); - that.msDAL.unwriteMS(ms); + yield that.msDAL.unwriteMS(ms); }); this.unExcludeIdentity = that.idtyDAL.unExcludeIdentity; @@ -1183,24 +1130,24 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { this.pushStats = that.statDAL.pushStats; this.needsSave = function() { - return loki.autosaveDirty(); + return true; }; this.close = function() { if (that.needsSave()) { - return Q.nbind(loki.saveDatabase, loki)(); + return Q.nbind(sqlite.close, sqlite)(); } return Q(); }; this.resetAll = function(done) { - var files = ['stats', 'cores', 'current', 'conf', UCOIN_DB_NAME]; + var files = ['stats', 'cores', 'current', 'conf', UCOIN_DB_NAME, UCOIN_DB_NAME + '.db']; var dirs = ['blocks', 'ud_history', 'branches', 'certs', 'txs', 'cores', 'sources', 'links', 'ms', 'identities', 'peers', 'indicators', 'leveldb']; return resetFiles(files, dirs, done); }; this.resetData = function(done) { - var files = ['stats', 'cores', 'current', UCOIN_DB_NAME]; + var files = ['stats', 'cores', 'current', UCOIN_DB_NAME, UCOIN_DB_NAME + '.db']; var dirs = ['blocks', 'ud_history', 'branches', 'certs', 'txs', 'cores', 'sources', 'links', 'ms', 'identities', 'peers', 'indicators', 'leveldb']; return resetFiles(files, dirs, done); }; @@ -1221,7 +1168,7 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { var files = []; var dirs = ['peers']; return co(function *() { - that.peerDAL.lokiRemoveAll(); + that.peerDAL.removeAll(); yield resetFiles(files, dirs); return that.close(); }) @@ -1236,29 +1183,30 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { }; function resetFiles(files, dirs, done) { - return Q.all([ - - // Remove files - Q.all(files.map(function(fName) { - return myFS.exists(rootPath + '/' + fName + '.json') - .then(function(exists){ - return exists ? myFS.remove(rootPath + '/' + fName + '.json') : Q(); - }); - })), - - // Remove directories - Q.all(dirs.map(function(dirName) { - return myFS.exists(rootPath + '/' + dirName) - .then(function(exists){ - return exists ? myFS.removeTree(rootPath + '/' + dirName) : Q(); - }); - })) - ]) - .then(function(){ - done && done(); - }) - .catch(function(err){ - done && done(err); - }); + return co(function *() { + for (let i = 0, len = files.length; i < len; i++) { + let fName = files[i]; + // JSON file? + let existsJSON = yield myFS.exists(rootPath + '/' + fName + '.json'); + if (existsJSON) { + yield myFS.remove(rootPath + '/' + fName + '.json'); + } else { + // Normal file? + let existsFile = yield myFS.exists(rootPath + '/' + fName); + if (existsFile) { + yield myFS.remove(rootPath + '/' + fName); + } + } + } + for (let i = 0, len = dirs.length; i < len; i++) { + let dirName = dirs[i]; + let existsDir = yield myFS.exists(rootPath + '/' + dirName); + if (existsDir) { + yield myFS.removeTree(rootPath + '/' + dirName); + } + } + done && done(); + }) + .catch((err) => done && done(err)); } } diff --git a/app/lib/dal/fileDALs/AbstractLoki.js b/app/lib/dal/fileDALs/AbstractLoki.js deleted file mode 100644 index daab0c6f64a16e89e9d40a95bde08bf08123f2cf..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/AbstractLoki.js +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Created by cgeek on 16/10/15. - */ - -var Q = require('q'); -var _ = require('underscore'); - -module.exports = AbstractLoki; - -function AbstractLoki(collection) { - - "use strict"; - - function find(conditons) { - return collection.find(conditons); - } - - this.IMMUTABLE_FIELDS = true; - - this.collection = collection; - - this.lokiFind = (baseConditions, metaConditions) => - Q(collection.find(getConditions(baseConditions, metaConditions))); - - this.lokiFindOne = (baseConditions, metaConditions) => - Q(collection.find(getConditions(baseConditions, metaConditions))[0] || null); - - this.lokiFindInAll = (metaConditions) => - Q(find(metaConditions)); - - this.lokiExisting = (entity) => { - let uniqueFindConditions = this.idKeys.map((key) => { - let cond = {}; - cond[key] = entity[key]; - return cond; - }); - return find({ - $and: uniqueFindConditions - })[0]; - }; - - this.lokiSave = (fullEntity) => { - let entity = fullEntity; - if (this.propsToSave) { - entity = _.pick(fullEntity, this.propsToSave || []); - } - let existing = this.lokiExisting(entity); - if (existing) { - // Save in main branch: overrides main data - existing = _.extend(existing, entity); - collection.update(existing); - } else { - collection.insert(entity); - } - return Q(entity); - }; - - this.lokiRemove = (fullEntity) => { - let entity = fullEntity; - if (this.propsToSave) { - entity = _.pick(fullEntity, this.propsToSave || []); - } - let existing = this.lokiExisting(entity); - if (existing) { - collection.remove(existing); - return true; - } - return false; - }; - - this.lokiRemoveAll = () => - collection.removeDataOnly(); - - this.lokiRemoveWhere = (conditions) => - collection.removeWhere(conditions); - - function getConditions(baseConditions, metaConditions) { - let conditions = { - $and: [baseConditions, metaConditions] - }; - if (!baseConditions || !metaConditions) { - conditions = baseConditions || metaConditions; - } - return conditions; - } -} \ No newline at end of file diff --git a/app/lib/dal/fileDALs/BlockDAL.js b/app/lib/dal/fileDALs/BlockDAL.js deleted file mode 100644 index d51a57d84f1a0cea19df0102e9c5c2c2200a93e4..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/BlockDAL.js +++ /dev/null @@ -1,244 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var Q = require('q'); -var _ = require('underscore'); -var co = require('co'); -var constants = require('../../constants'); -var logger = require('../../../../app/lib/logger')('blockdal'); - -const BLOCK_FILE_PREFIX = "0000000000"; -const BLOCK_FOLDER_SIZE = 500; - -module.exports = BlockDAL; - -function BlockDAL(loki, rootFS, getLowerWindowBlock) { - - "use strict"; - - let collection = loki.getCollection('blocks') || loki.addCollection('blocks', { indices: ['fork', 'number', 'hash'] }); - let blocksDB = getView(); - let forksDB = getForkView(); - let current = null; - let that = this; - - this.init = () => co(function *() { - yield rootFS.makeTree('blocks/'); - }); - - this.getCurrent = () => { - if (!current) { - current = blocksDB.branchResultset().simplesort('number', true).limit(1).data()[0]; - } - return Q(current); - }; - - this.getBlock = (number) => co(function *() { - let block = blocksDB.branchResultset().find({ number: parseInt(number) }).data()[0]; - if (!block) { - try { - block = yield rootFS.readJSON(pathOfBlock(number) + blockFileName(number) + '.json'); - } catch(e) { - block = null; - } - } - return block; - }); - - this.getAbsoluteBlock = (number, hash) => co(function *() { - let block = collection.find({ - $and: [{ - number: parseInt(number) - }, { - hash: hash - } - ]})[0]; - if (!block) { - try { - block = yield rootFS.readJSON(pathOfBlock(number) + blockFileName(number) + '.json'); - } catch(e) { - block = null; - } - } - return block; - }); - - this.blocksDB = blocksDB; - this.forksDB = forksDB; - this.collection = collection; - - this.getBlocks = (start, end) => { - let lowerInLoki = blocksDB.branchResultset().simplesort('number').limit(1).data()[0]; - let lokiBlocks = blocksDB.branchResultset().find({ - $and: [{ - number: { $gte: start } - }, { - number: { $lte: end } - }] - }).data(); - if (lowerInLoki.number <= start) { - return Q(lokiBlocks); - } - return co(function *() { - let filesBlocks = yield Q.all(_.range(start, Math.min(lowerInLoki.number, end + 1)).map((number) => rootFS.readJSON(pathOfBlock(number) + blockFileName(number) + '.json'))); - return filesBlocks.concat(lokiBlocks); - }); - }; - - this.lastBlockWithDividend = () => { - let blocks = blocksDB.branchResultset().find({ dividend: { $gt: 0 } }).simplesort('number', true).data(); - return blocks[0]; - }; - - this.lastBlockOfIssuer = (issuer) => { - let blocksOfIssuer = blocksDB.branchResultset().find({ issuer: issuer }).simplesort('number', true).limit(1).data(); - return Q(blocksOfIssuer[0]); - }; - - this.getForkBlocks = () => - Q(forksDB.branchResultset().find({ wrong: false }).data()); - - this.saveBunch = (blocks, inFiles) => { - if (!inFiles) { - collection.insert(blocks); - return Q(); - } else { - // Save in files - return co(function *() { - let trees = []; - blocks.forEach(function(block){ - let pathForBlock = pathOfBlock(block.number); - if (!~trees.indexOf(pathForBlock)) { - trees.push(pathForBlock); - } - }); - yield trees.map((tree) => rootFS.makeTree(tree)); - yield blocks.map((block) => rootFS.writeJSON(pathOfBlock(block.number) + blockFileName(block.number) + '.json', block)); - }); - } - }; - - this.saveBlock = (block) => { - if (!current || current.number < block.number) { - current = block; - } - let existing; - existing = collection.find({ - $and: [{ - number: block.number - }, { - hash: block.hash - }] - })[0]; - if (existing) { - // Updates - collection.update(_.extend(existing, block)); - } else { - collection.insert(block); - } - return Q(block); - }; - - this.saveSideBlock = (block) => { - block.fork = true; - let existing; - existing = collection.find({ - $and: [{ - number: block.number - }, { - hash: block.hash - }] - })[0]; - if (existing) { - // Updates - collection.update(_.extend(existing, block)); - } else { - collection.insert(block); - } - return Q(block); - }; - - this.setSideBlock = (block, previousBlock) => co(function *() { - let existing = collection.find({ - $and: [{ - number: block.number - }, { - hash: block.hash - }] - }); - (existing || []).forEach(function(found){ - found.fork = true; - collection.update(found); - }); - current = previousBlock; - let lowerInLoki = blocksDB.branchResultset().simplesort('number').limit(1).data()[0]; - if (lowerInLoki && lowerInLoki.number > 0) { - let newLower = yield that.getBlock(lowerInLoki.number - 1); - yield rootFS.remove(pathOfBlock(newLower.number) + blockFileName(newLower.number) + '.json'); - collection.insert(newLower); - } - }); - - this.migrateOldBlocks = () => co(function *() { - let number = yield getLowerWindowBlock(); - logger.debug("Clean some blocks from memory to disk..."); - logger.debug("Lower block = %s", number); - let lowerInLoki = blocksDB.branchResultset().simplesort('number').limit(1).data()[0]; - if (!lowerInLoki) { - return; - } - let lastUDBlock = that.lastBlockWithDividend(); - if (lastUDBlock) { - logger.debug("LastUD in loki = %s", lastUDBlock.number); - logger.debug("Lower in loki = %s", lowerInLoki.number); - let deadBlocksInLoki = number - lowerInLoki.number; - logger.debug("Dead blocks = %s", deadBlocksInLoki); - if (deadBlocksInLoki >= constants.BLOCKS_COLLECT_THRESHOLD) { - let blocksToPersist = blocksDB.branchResultset().find({ - $and: [{ - number: { $gte: lowerInLoki.number } - }, { - number: { $lte: number } - }] - }).simplesort('number').data(); - logger.debug("To store in files = %s to %s", blocksToPersist[0].number, blocksToPersist[blocksToPersist.length - 1].number); - for (let i = 0; i < blocksToPersist.length; i++) { - let block = blocksToPersist[i]; - yield rootFS.makeTree(pathOfBlock(block.number)); - yield rootFS.writeJSON(pathOfBlock(block.number) + blockFileName(block.number) + '.json', block); - collection.remove(block); - } - lowerInLoki = blocksDB.branchResultset().simplesort('number').limit(1).data()[0]; - logger.debug("Lower in loki now = %s", lowerInLoki.number); - logger.debug("LastUD in loki = %s", that.lastBlockWithDividend().number); - } - } - }); - - function getView() { - let view; - // Main branch - view = collection.addDynamicView('mainBranch'); - view.applyFind({ fork: false }); - return view; - } - - function getForkView() { - let view = collection.addDynamicView('forks'); - view.applyFind({ fork: true }); - return view; - } - - function folderOfBlock(blockNumber) { - return (Math.floor(blockNumber / BLOCK_FOLDER_SIZE) + 1) * BLOCK_FOLDER_SIZE; - } - - function pathOfBlock(blockNumber) { - return 'blocks/' + folderOfBlock(blockNumber) + '/'; - } - - function blockFileName(blockNumber) { - return BLOCK_FILE_PREFIX.substr(0, BLOCK_FILE_PREFIX.length - ("" + blockNumber).length) + blockNumber; - } -} diff --git a/app/lib/dal/fileDALs/CertDAL.js b/app/lib/dal/fileDALs/CertDAL.js deleted file mode 100644 index 9c1cb851e13997a6cb7fd7394ab8a02813a9ec9f..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/CertDAL.js +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var Q = require('q'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = CertDAL; - -function CertDAL(loki) { - - "use strict"; - - let collection = loki.getCollection('certs') || loki.addCollection('certs', { indices: ['from', 'target', 'linked', 'written'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['sig', 'from', 'target']; - this.propsToSave = [ - 'linked', - 'written_block', - 'written_hash', - 'sig', - 'block_number', - 'block_hash', - 'target', - 'to', - 'from', - 'block' - ]; - - this.init = () => null; - - this.getToTarget = (hash) => this.lokiFind({ - target: hash - }); - - this.getFromPubkey = (pubkey) => this.lokiFind({ - from: pubkey - }); - - this.getNotLinked = () => this.lokiFindInAll({ - linked: false - }); - - this.getNotLinkedToTarget = (hash) => this.lokiFind({ - target: hash - },{ - linked: false - }); - - this.listLocalPending = () => Q([]); - - this.saveOfficial = (cert) => { - cert.linked = true; - return this.lokiSave(cert); - }; - - this.saveCert = (cert) => - this.lokiSave(cert); - - this.saveNewCertification = (cert) => - this.lokiSave(cert); - - this.existsGivenCert = (cert) => Q(this.lokiExisting(cert)); -} \ No newline at end of file diff --git a/app/lib/dal/fileDALs/LinksDAL.js b/app/lib/dal/fileDALs/LinksDAL.js deleted file mode 100644 index 154043b3c63978d41a45648e54bd8a24c20801f0..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/LinksDAL.js +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var co = require('co'); -var Q = require('q'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = LinksDAL; - -function LinksDAL(loki) { - - "use strict"; - - let that = this; - let collection = loki.getCollection('links') || loki.addCollection('links', { indices: ['source', 'target', 'block_number', 'block_hash', 'timestamp'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['source', 'target', 'block_number', 'block_hash']; - this.propsToSave = [ - 'source', - 'target', - 'timestamp', - 'block_number', - 'block_hash', - 'obsolete' - ]; - - this.init = () => null; - - this.getValidLinksFrom = (pubkey) => this.lokiFind({ - source: pubkey - }, { - obsolete: false - }); - - this.getSimilarLinksFromDate = (from, to, minDate) => this.lokiFind({ - $and: [ - { source: from }, - { target: to }, - { timestamp: { $gte: minDate }} - ] - }); - - this.getValidLinksTo = (pubkey) => this.lokiFind({ - target: pubkey - }, { - obsolete: false - }); - - this.getLinksWithPath = (from, to) => - this.lokiFind({ - $and: [{ - source: from - },{ - target: to - }] - }); - - this.obsoletesLinks = (minTimestamp) => co(function *() { - let toObsolete = yield that.lokiFind({ - timestamp: { $lte: minTimestamp } - },{ - obsolete: false - }); - for (let i = 0; i < toObsolete.length; i++) { - let link = toObsolete[i]; - link.obsolete = true; - collection.update(link); - } - }); - - this.unObsoletesLinks = (minTimestamp) => co(function *() { - let toObsolete = yield that.lokiFind({ - timestamp: { $gte: minTimestamp } - }); - for (let i = 0; i < toObsolete.length; i++) { - let link = toObsolete[i]; - link.obsolete = false; - collection.update(link); - } - }); - - this.addLink = (link) => { - link.obsolete = false; - return this.lokiSave(link); - }; - - this.removeLink = (link) => - this.lokiRemove(link); -} \ No newline at end of file diff --git a/app/lib/dal/fileDALs/MembershipDAL.js b/app/lib/dal/fileDALs/MembershipDAL.js deleted file mode 100644 index c606b8fb514bfe2feb87da6a1596a75b297a1436..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/MembershipDAL.js +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var Q = require('q'); -var _ = require('underscore'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = MembershipDAL; - -function MembershipDAL(loki) { - - "use strict"; - - let collection = loki.getCollection('memberships') || loki.addCollection('memberships', { indices: ['membership', 'issuer', 'number', 'blockNumber', 'blockHash', 'userid', 'certts', 'block', 'fpr', 'written', 'signature'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['issuer', 'signature']; - this.propsToSave = [ - 'membership', - 'issuer', - 'number', - 'blockNumber', - 'blockHash', - 'userid', - 'certts', - 'block', - 'fpr', - 'idtyHash', - 'written', - 'signature' - ]; - - this.init = () => null; - - this.getMembershipOfIssuer = (ms) => Q(this.lokiExisting(ms)); - - this.getMembershipsOfIssuer = (issuer) => this.lokiFind({ - issuer: issuer - }); - - this.getPendingINOfTarget = (hash) => - this.lokiFind({ - $and: [ - { idtyHash: hash }, - { membership: 'IN' }, - { written: false } - ] - }); - - this.getPendingIN = () => this.lokiFind({ - membership: 'IN' - },{ - written: false - }); - - this.getPendingOUT = () => this.lokiFind({ - membership: 'OUT' - },{ - written: false - }); - - this.unwriteMS = (ms) => { - let existing = this.lokiExisting({ - issuer: ms.issuer, - signature: ms.signature - }); - if (existing) { - existing.written = false; - collection.update(existing); - } - }; - - this.saveOfficialMS = (type, ms) => { - let obj = _.extend({}, ms); - obj.membership = type.toUpperCase(); - obj.written = true; - return this.lokiSave(_.pick(obj, 'membership', 'issuer', 'number', 'blockNumber', 'blockHash', 'userid', 'certts', 'block', 'fpr', 'idtyHash', 'written', 'signature')); - }; - - this.savePendingMembership = (ms) => { - ms.membership = ms.membership.toUpperCase(); - ms.written = false; - return this.lokiSave(_.pick(ms, 'membership', 'issuer', 'number', 'blockNumber', 'blockHash', 'userid', 'certts', 'block', 'fpr', 'idtyHash', 'written', 'signature')); - }; -} diff --git a/app/lib/dal/fileDALs/PeerDAL.js b/app/lib/dal/fileDALs/PeerDAL.js deleted file mode 100644 index 46b667e959037bec9b5299dec3448b483d8e06fb..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/PeerDAL.js +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var Q = require('q'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = PeerDAL; - -function PeerDAL(loki) { - - "use strict"; - - let collection = loki.getCollection('peers') || loki.addCollection('peers', { indices: ['pubkey', 'status'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['pubkey']; - this.propsToSave = [ - 'version', - 'currency', - 'status', - 'statusTS', - 'hash', - 'first_down', - 'last_try', - 'pub', - 'pubkey', - 'block', - 'signature', - 'endpoints', - 'raw' - ]; - - this.init = () => null; - - this.listAll = () => Q(collection.find()); - - this.getPeer = (pubkey) => Q(collection.find({ pubkey: pubkey })[0]); - - this.savePeer = (peer) => this.lokiSave(peer); -} diff --git a/app/lib/dal/fileDALs/SourcesDAL.js b/app/lib/dal/fileDALs/SourcesDAL.js deleted file mode 100644 index c0ed6d6bdbb948703009f42e4a2ec978265fbe4a..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/SourcesDAL.js +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var Q = require('q'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = SourcesDAL; - -function SourcesDAL(loki) { - - "use strict"; - - let collection = loki.getCollection('sources') || loki.addCollection('sources', { indices: ['pubkey', 'type', 'number', 'fingerprint', 'amount', 'block_hash'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['pubkey', 'type', 'number', 'fingerprint', 'amount']; - this.propsToSave = [ - 'pubkey', - 'type', - 'number', - 'time', - 'fingerprint', - 'amount', - 'block_hash', - 'consumed' - ]; - - this.init = () => null; - - this.getAvailableForPubkey = (pubkey) => this.lokiFind({ - pubkey: pubkey - },{ - consumed: false - }); - - this.getUDSources = (pubkey) => this.lokiFind({ - $and: [{ - pubkey: pubkey - },{ - type: 'D' - }] - }); - - this.getSource = (pubkey, type, number) => this.lokiFindOne({ - $and: [ - { pubkey: pubkey }, - { type: type }, - { number: number } - ] - }, null, this.IMMUTABLE_FIELDS); - - this.isAvailableSource = (pubkey, type, number, fingerprint, amount) => { - let src = this.lokiExisting({ - pubkey: pubkey, - type: type, - number: number, - fingerprint: fingerprint, - amount: amount - }); - return Q(src ? !src.consumed : false); - }; - - this.consumeSource = (pubkey, type, number, fingerprint, amount) => { - let src = this.lokiExisting({ - pubkey: pubkey, - type: type, - number: number, - fingerprint: fingerprint, - amount: amount - }); - src.consumed = true; - return this.lokiSave(src); - }; - - this.addSource = (state, pubkey, type, number, fingerprint, amount, block_hash, time) => this.lokiSave({ - pubkey: pubkey, - type: type, - number: number, - fingerprint: fingerprint, - amount: amount, - time: time, - block_hash: block_hash, - consumed: false - }); - - this.unConsumeSource = (type, pubkey, number, fingerprint, amount, time, block_hash) => { - let src = this.lokiExisting({ - pubkey: pubkey, - type: type, - number: number, - fingerprint: fingerprint, - amount: amount - }); - if (src) { - src.consumed = false; - collection.update(src); - } else { - this.lokiSave({ - pubkey: pubkey, - type: type, - number: number, - fingerprint: fingerprint, - amount: amount, - time: time, - block_hash: block_hash, - consumed: false - }); - } - }; - - this.removeAllSourcesOfBlock = (number) => this.lokiRemoveWhere({ - number: number - }); -} \ No newline at end of file diff --git a/app/lib/dal/fileDALs/TxsDAL.js b/app/lib/dal/fileDALs/TxsDAL.js deleted file mode 100644 index 760118f7547ee8ad1f9e86156081d4c96ad539d6..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/TxsDAL.js +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ -var co = require('co'); -var Q = require('q'); -var _ = require('underscore'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = TxsDAL; - -function TxsDAL(loki) { - - "use strict"; - - let that = this; - let collection = loki.getCollection('txs') || loki.addCollection('txs', { indices: ['hash', 'block_number', 'written', 'signature', 'recipients'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['hash', 'block_number']; - this.propsToSave = [ - 'inputs', - 'outputs', - 'issuers', - 'signatories', - 'signatures', - 'comment', - 'hash', - 'version', - 'currency', - 'block_number', - 'time', - 'recipients', - 'written', - 'removed' - ]; - - this.init = () => null; - - this.getAllPending = () => - this.lokiFindInAll({ - $and: [{ - written: false - },{ - removed: false - }] - }); - - this.getTX = (hash) => this.lokiFindOne({ - hash: hash - }, null, this.IMMUTABLE_FIELDS); - - this.removeTX = (hash) => co(function *() { - let tx = yield that.lokiFindOne({ - hash: hash - }); - if (tx) { - tx.removed = true; - return that.lokiSave(tx); - } - return Q(tx); - }); - - this.addLinked = (tx) => { - tx.written = true; - tx.removed = false; - tx.hash = tx.getHash(true); - tx.recipients = tx.outputs.map(function(out) { - return out.match('(.*):')[1]; - }); - return that.lokiSave(tx); - }; - - this.addPending = (tx) => { - tx.written = false; - tx.removed = false; - tx.hash = tx.getHash(true); - tx.recipients = tx.outputs.map(function(out) { - return out.match('(.*):')[1]; - }); - return this.lokiSave(tx); - }; - - this.getLinkedWithIssuer = (pubkey) => this.lokiFind({ - issuers: { $contains: pubkey } - },{ - written: true - }); - - this.getLinkedWithRecipient = (pubkey) => this.lokiFind({ - recipients: { $contains: pubkey } - },{ - written: true - }); - - this.getPendingWithIssuer = (pubkey) => this.lokiFind({ - issuers: { $contains: pubkey } - },{ - written: false - }); - - this.getPendingWithRecipient = (pubkey) => this.lokiFind({ - recipients: { $contains: pubkey } - },{ - written: false - }); -} \ No newline at end of file diff --git a/app/lib/dal/sqliteDAL/AbstractSQLite.js b/app/lib/dal/sqliteDAL/AbstractSQLite.js new file mode 100644 index 0000000000000000000000000000000000000000..0fed0d909f92cc7a34bd9ab15a9977f243537db9 --- /dev/null +++ b/app/lib/dal/sqliteDAL/AbstractSQLite.js @@ -0,0 +1,325 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var _ = require('underscore'); +var co = require('co'); +var colors = require('colors'); + +module.exports = AbstractSQLite; + +function AbstractSQLite(db) { + + "use strict"; + + let that = this; + + this.ASC = false; + this.DESC = true; + this.arrays = []; + this.booleans = []; + this.bigintegers = []; + this.translated = {}; + + this.query = (sql, params) => co(function *() { + try { + if (sql.match(/^INSERT/)) { + //console.log(sql, JSON.stringify(params || [])); + } + let start = new Date(); + let res = yield Q.nbind(db.all, db)(sql, params || []); + let duration = (new Date()) - start; + let entities = res.map(toEntity); + if (true || that.table == "source") { + let msg = sql + ' | %s\t==> %s rows in %s ms'; + if (duration <= 2) { + msg = colors.green(msg); + } else if(duration <= 5) { + msg = colors.yellow(msg); + } else if (duration <= 10) { + msg = colors.magenta(msg); + } else if (duration <= 100) { + msg = colors.red(msg); + } + if (duration > 10) { + //console.log(msg, JSON.stringify([] || []), entities.length, duration); + } + } + return entities; + } catch (e) { + console.error('ERROR >> %s', sql, JSON.stringify(params || []), e.stack || e.message || e); + throw e; + } + }); + + this.sqlListAll = () => this.query("SELECT * FROM " + this.table); + + this.sqlDeleteAll = () => this.exec("DELETE FROM " + this.table); + + this.sqlFind = (obj, sortObj) => co(function *() { + let conditions = toConditionsArray(obj).join(' and '); + let values = toParams(obj); + let sortKeys = _.keys(sortObj); + let sort = sortKeys.length ? ' ORDER BY ' + sortKeys.map((k) => "`" + k + "` " + (sortObj[k] ? 'DESC' : 'ASC')).join(',') : ''; + return that.query('SELECT * FROM ' + that.table + ' WHERE ' + conditions + sort, values); + }); + + this.sqlFindOne = (obj) => co(function *() { + let res = yield that.sqlFind(obj); + return res[0]; + }); + + this.sqlFindLikeAny = (obj, sort) => co(function *() { + let keys = _.keys(obj); + return that.query('SELECT * FROM ' + that.table + ' WHERE ' + keys.map((k) => '`' + k + '` like ?').join(' or '), keys.map((k) => obj[k].toUpperCase()), sort); + }); + + this.sqlUpdateWhere = (obj, where) => co(function *() { + // Valorizations + let setInstructions = toSetArray(obj).join(', '); + let setValues = toParams(obj); + // Conditions + let conditions = toConditionsArray(where).join(' AND '); + let condValues = toParams(where); + return that.query('UPDATE ' + that.table + ' SET ' + setInstructions + ' WHERE ' + conditions, setValues.concat(condValues)); + }); + + this.sqlRemoveWhere = (obj) => co(function *() { + let keys = _.keys(obj); + return that.query('DELETE FROM ' + that.table + ' WHERE ' + keys.map((k) => '`' + k + '` = ?').join(' and '), keys.map((k) => obj[k])); + }); + + this.sqlExisting = (entity) => that.getEntity(entity); + + this.saveEntity = (entity) => co(function *() { + let toSave = entity; + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let existing = yield that.getEntity(toSave); + if (existing) { + toSave = toRow(toSave); + let valorizations = that.fields.map((field) => '`' + field + '` = ?').join(', '); + let conditions = getPKFields().map((field) => '`' + field + '` = ?').join(' and '); + let setValues = that.fields.map((field) => toSave[field]); + let condValues = getPKFields().map((k) => toSave[k]); + return that.query('UPDATE ' + that.table + ' SET ' + valorizations + ' WHERE ' + conditions, setValues.concat(condValues)); + } + yield that.insert(toSave); + }); + + this.updateEntity = (entity, values) => co(function *() { + let toSave = toRow(entity); + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let valuesKeys = _.keys(values); + let valorizations = valuesKeys.map((field) => '`' + field + '` = ?').join(', '); + let conditions = getPKFields().map((field) => '`' + field + '` = ?').join(' and '); + let setValues = valuesKeys.map((field) => values[field]); + let condValues = getPKFields().map((k) => toSave[k]); + return that.query('UPDATE ' + that.table + ' SET ' + valorizations + ' WHERE ' + conditions, setValues.concat(condValues)); + }); + + this.deleteEntity = (entity) => co(function *() { + let toSave = toRow(entity); + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let conditions = getPKFields().map((field) => '`' + field + '` = ?').join(' and '); + let condValues = getPKFields().map((k) => toSave[k]); + return that.query('DELETE FROM ' + that.table + ' WHERE ' + conditions, condValues); + }); + + this.insert = (entity) => co(function *() { + let row = toRow(entity); + let values = that.fields.map((f) => row[f]); + yield that.query(that.getInsertQuery(), values); + }); + + this.getEntity = function(entity) { + return co(function *() { + let conditions = getPKFields().map((field) => '`' + field + '` = ?').join(' and '); + let params = getPKFields().map((field) => entity[field]); + return (yield that.query('SELECT * FROM ' + that.table + ' WHERE ' + conditions, params))[0]; + }); + }; + + this.exec = (sql) => co(function *() { + try { + if (sql.match(/INSERT INTO source/)) { + //console.log('------------'); + //console.log(sql); + //console.log('------------'); + } + return Q.nbind(db.exec, db)(sql); + } catch (e) { + console.error('ERROR >> %s', sql); + throw e; + } + }); + + this.getInsertQuery = () => + "INSERT INTO " + that.table + " (" + getFields(that.fields).map(f => '`' + f + '`').join(',') + ") VALUES (" + "?,".repeat(that.fields.length - 1) + "?);"; + + this.getInsertHead = () => { + let valuesKeys = getFields(that.fields); + return 'INSERT INTO ' + that.table + " (" + valuesKeys.map(f => '`' + f + '`').join(',') + ") VALUES "; + }; + + this.getInsertValue = (toSave) => { + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let row = toRow(toSave); + let valuesKeys = getFields(that.fields); + let values = valuesKeys.map((field) => escapeToSQLite(row[field])); + return "(" + values.join(',') + ")"; + }; + + this.getUpdateRawQuery = (toSave, values) => { + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let valuesKeys = _.keys(values); + let valorizations = valuesKeys.map((field) => '`' + field + '` = ' + escapeToSQLite(values[field])).join(', '); + let conditions = getPKFields().map((field) => '`' + field + '` = ' + escapeToSQLite(toSave[field])).join(' and '); + return 'UPDATE ' + that.table + ' SET ' + valorizations + ' WHERE ' + conditions + ';'; + }; + + this.getDeleteRawQuery = (toSave) => { + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let conditions = getPKFields().map((field) => '`' + field + '` = ' + escapeToSQLite(toSave[field])).join(' and '); + return 'DELETE FROM ' + that.table + ' WHERE ' + conditions + ';'; + }; + + this.getDeleteHead = () => { + return 'DELETE FROM ' + that.table + " WHERE "; + }; + + this.getDeleteValues = (entities) => { + return entities.map((toSave) => { + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let conditions = getPKFields().map((field) => '`' + field + '` = ' + escapeToSQLite(toSave[field])).join(' and '); + return "(" + conditions + ")"; + }).join(' OR\n '); + }; + + this.toInsertValues = (entity) => { + let row = toRow(entity); + let values = that.fields.map((f) => row[f]); + let formatted = values.map(escapeToSQLite); + return "(" + formatted.join(',') + ")"; + }; + + function toConditionsArray(obj) { + return _.keys(obj).map((k) => { + if (obj[k].$lte !== undefined) { + return '`' + k + '` <= ?'; + } else if (obj[k].$gte !== undefined) { + return '`' + k + '` >= ?'; + } else if (obj[k].$contains !== undefined) { + return '`' + k + '` LIKE ?'; + } else { + return '`' + k + '` = ?'; + } + }); + } + + function toSetArray(obj) { + let row = toRow(obj); + return _.keys(row).map((k) => '`' + k + '` = ?'); + } + + function toParams(obj) { + return _.keys(obj).map((k) => { + if (obj[k].$lte !== undefined) { + return obj[k].$lte; + } else if (obj[k].$gte !== undefined) { + return obj[k].$gte; + } else if (obj[k].$contains !== undefined) { + return "%" + obj[k].$contains + "%"; + } else { + return obj[k]; + } + }); + } + + function escapeToSQLite(val) { + if (typeof val == "boolean") { + // SQLite specific: true => 1, false => 0 + return val ? 1 : 0; + } + else if (typeof val == "string") { + return "'" + val.replace(/'/g, "\\'") + "'"; + } + else if (val === undefined) { + return "null"; + } else { + return JSON.stringify(val); + } + } + + function getPKFields() { + return getFields(that.pkFields); + } + + function getFields(fields) { + return fields.map((f) => getField(f)); + } + + function getField(f) { + return that.translated[f] || f; + } + + function toEntity(row) { + for (let i = 0, len = that.arrays.length; i < len; i++) { + let arr = that.arrays[i]; + row[arr] = JSON.parse(row[arr]); + } + // Big integers are stored as strings to avoid data loss + for (let i = 0, len = that.bigintegers.length; i < len; i++) { + let bigint = that.bigintegers[i]; + row[bigint] = parseInt(row[bigint], 10); + } + // Translate some DB fields to obj fields + let toTranslate = that.translated || {}; + let toDBFields = _.keys(toTranslate); + for (let i = 0, len = toDBFields.length; i < len; i++) { + let objField = toDBFields[i]; + row[objField] = row[toTranslate[objField]]; + } + // Booleans + for (let i = 0, len = that.booleans.length; i < len; i++) { + let f = that.booleans[i]; + row[f] = Boolean(row[f]); + } + return row; + } + + function toRow(entity) { + let row = _.clone(entity); + for (let i = 0, len = that.arrays.length; i < len; i++) { + let arr = that.arrays[i]; + row[arr] = JSON.stringify(row[arr] || []); + } + // Big integers are stored as strings to avoid data loss + for (let i = 0, len = that.bigintegers.length; i < len; i++) { + let bigint = that.bigintegers[i]; + row[bigint] = String(entity[bigint]); + } + // Translate some obj fields to DB field name (because of DB keywords) + let toTranslate = that.translated || {}; + let toDBFields = _.keys(toTranslate); + for (let i = 0, len = toDBFields.length; i < len; i++) { + let objField = toDBFields[i]; + row[toTranslate[objField]] = row[objField]; + } + return row; + } +} \ No newline at end of file diff --git a/app/lib/dal/sqliteDAL/BlockDAL.js b/app/lib/dal/sqliteDAL/BlockDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..e2a557ed8846e96c4d557142f11654a61e6e7ee4 --- /dev/null +++ b/app/lib/dal/sqliteDAL/BlockDAL.js @@ -0,0 +1,134 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var co = require('co'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = BlockDAL; + +const IS_FORK = true; +const IS_NOT_FORK = false; + +function BlockDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let current = null; + let that = this; + + this.table = 'block'; + this.fields = ['fork', 'hash', 'signature', 'currency', 'issuer', 'parameters', 'previousHash', 'previousIssuer', 'version', 'membersCount', 'monetaryMass', 'UDTime', 'medianTime', 'dividend', 'time', 'powMin', 'number', 'nonce', 'transactions', 'certifications', 'identities', 'joiners', 'actives', 'leavers', 'excluded']; + this.arrays = ['identities','certifications','actives','excluded','leavers','actives','joiners','transactions']; + this.pkFields = ['number','hash']; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'fork BOOLEAN NOT NULL,' + + 'hash VARCHAR(40) NOT NULL,' + + 'signature VARCHAR(100) NOT NULL,' + + 'currency VARCHAR(50) NOT NULL,' + + 'issuer VARCHAR(50) NOT NULL,' + + 'parameters VARCHAR(255),' + + 'previousHash VARCHAR(50),' + + 'previousIssuer VARCHAR(50),' + + 'version INTEGER NOT NULL,' + + 'membersCount INTEGER NOT NULL,' + + 'monetaryMass INTEGER DEFAULT 0,' + + 'UDTime DATETIME,' + + 'medianTime DATETIME NOT NULL,' + + 'dividend INTEGER,' + + 'time DATETIME NOT NULL,' + + 'powMin INTEGER NOT NULL,' + + 'number INTEGER NOT NULL,' + + 'nonce INTEGER NOT NULL,' + + 'transactions TEXT,' + + 'certifications TEXT,' + + 'identities TEXT,' + + 'joiners TEXT,' + + 'actives TEXT,' + + 'leavers TEXT,' + + 'excluded TEXT,' + + 'created DATETIME DEFAULT NULL,' + + 'updated DATETIME DEFAULT NULL,' + + 'PRIMARY KEY (number,hash)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_block_hash ON block (hash);' + + 'CREATE INDEX IF NOT EXISTS idx_block_fork ON block (fork);' + + 'COMMIT;', []); + }); + + this.getCurrent = () => co(function *() { + if (!current) { + current = (yield that.query('SELECT * FROM block WHERE NOT fork ORDER BY number DESC LIMIT 1'))[0]; + } + return Q(current); + }); + + this.getBlock = (number) => co(function *() { + return (yield that.query('SELECT * FROM block WHERE number = ? and NOT fork', [parseInt(number)]))[0]; + }); + + this.getAbsoluteBlock = (number, hash) => co(function *() { + return (yield that.query('SELECT * FROM block WHERE number = ? and hash = ?', [parseInt(number), hash]))[0]; + }); + + this.getBlocks = (start, end) => { + return that.query('SELECT * FROM block WHERE number BETWEEN ? and ? and NOT fork ORDER BY number ASC', [start, end]); + }; + + this.lastBlockWithDividend = () => co(function *() { + return (yield that.query('SELECT * FROM block WHERE dividend > 0 and NOT fork ORDER BY number DESC LIMIT 1'))[0]; + }); + + this.lastBlockOfIssuer = (issuer) => co(function *() { + return (yield that.query('SELECT * FROM block WHERE issuer = ? and NOT fork ORDER BY number DESC LIMIT 1', [issuer]))[0]; + }); + + this.getForkBlocks = () => { + return that.query('SELECT * FROM block WHERE fork ORDER BY number'); + }; + + this.saveBunch = (blocks) => co(function *() { + let queries = "INSERT INTO block (" + that.fields.join(',') + ") VALUES "; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + queries += that.toInsertValues(block); + if (i + 1 < len) { + queries += ",\n"; + } + } + yield that.exec(queries); + }); + + this.saveBlock = (block) => co(function *() { + if (!current || current.number < block.number) { + current = block; + } + return saveBlockAs(block, IS_NOT_FORK); + }); + + this.saveSideBlock = (block) => + saveBlockAs(block, IS_FORK); + + function saveBlockAs(block, fork) { + return co(function *() { + let existing = yield that.getEntity(block); + if (existing) { + return that.query('UPDATE block SET fork = ? WHERE number = ? and hash = ?', [fork, block.number, block.hash]); + } + yield that.insert(block); + }); + } + + this.setSideBlock = (block, previousBlock) => co(function *() { + yield that.query('UPDATE block SET fork = ? WHERE number = ? and hash = ?', [true, block.number, block.hash]); + current = previousBlock; + }); + + this.migrateOldBlocks = () => Q(); +} diff --git a/app/lib/dal/sqliteDAL/CertDAL.js b/app/lib/dal/sqliteDAL/CertDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..27ce340dfd48a791a0a52ee7d4c7792d629a3673 --- /dev/null +++ b/app/lib/dal/sqliteDAL/CertDAL.js @@ -0,0 +1,105 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var co = require('co'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = CertDAL; + +function CertDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'cert'; + this.fields = [ + 'linked', + 'written', + 'written_block', + 'written_hash', + 'sig', + 'block_number', + 'block_hash', + 'target', + 'to', + 'from', + 'block' + ]; + this.arrays = []; + this.booleans = ['linked', 'written']; + this.pkFields = ['from','target','sig']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + '`from` VARCHAR(50) NOT NULL,' + + '`to` VARCHAR(50) NOT NULL,' + + 'target CHAR(40) NOT NULL,' + + 'sig VARCHAR(100) NOT NULL,' + + 'block_number INTEGER NOT NULL,' + + 'block_hash VARCHAR(50),' + + 'block INTEGER NOT NULL,' + + 'linked BOOLEAN NOT NULL,' + + 'written BOOLEAN NOT NULL,' + + 'written_block INTEGER,' + + 'written_hash VARCHAR(50),' + + 'PRIMARY KEY (`from`, target, sig, written_block)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_cert_from ON cert (`from`);' + + 'CREATE INDEX IF NOT EXISTS idx_cert_target ON cert (target);' + + 'CREATE INDEX IF NOT EXISTS idx_cert_linked ON cert (linked);' + + 'COMMIT;', []); + }); + + this.beforeSaveHook = function(entity) { + entity.written = entity.written || !!(entity.written_hash); + }; + + this.getToTarget = (hash) => this.sqlFind({ + target: hash + }); + + this.getFromPubkey = (pubkey) => this.sqlFind({ + from: pubkey + }); + + this.getNotLinked = () => this.sqlFind({ + linked: false + }); + + this.getNotLinkedToTarget = (hash) => this.sqlFind({ + target: hash, + linked: false + }); + + this.listLocalPending = () => Q([]); + + this.saveOfficial = (cert) => { + cert.linked = true; + return this.saveEntity(cert); + }; + + this.saveCert = (cert) => this.saveEntity(cert); + + this.saveNewCertification = (cert) => this.saveEntity(cert); + + this.existsGivenCert = (cert) => Q(this.sqlExisting(cert)); + + this.updateBatchOfCertifications = (certs) => co(function *() { + let queries = []; + let insert = that.getInsertHead(); + let values = certs.map((cert) => that.getInsertValue(cert)); + if (certs.length) { + queries.push(insert + '\n' + values.join(',\n') + ';'); + } + if (queries.length) { + return that.exec(queries.join('\n')); + } + }); +} \ No newline at end of file diff --git a/app/lib/dal/fileDALs/IdentityDAL.js b/app/lib/dal/sqliteDAL/IdentityDAL.js similarity index 65% rename from app/lib/dal/fileDALs/IdentityDAL.js rename to app/lib/dal/sqliteDAL/IdentityDAL.js index 2d932ddb04a00c975d709125209ec8863e93e043..e3213d98b2e11030d1b79ae67651997317d5c6ec 100644 --- a/app/lib/dal/fileDALs/IdentityDAL.js +++ b/app/lib/dal/sqliteDAL/IdentityDAL.js @@ -3,22 +3,21 @@ */ var Q = require('q'); -var _ = require('underscore'); var co = require('co'); -var AbstractLoki = require('./AbstractLoki'); +var AbstractSQLite = require('./AbstractSQLite'); module.exports = IdentityDAL; -function IdentityDAL(loki) { +function IdentityDAL(db) { "use strict"; - let collection = loki.getCollection('identities') || loki.addCollection('identities', { indices: ['uid', 'pubkey', 'timestamp', 'member', 'written'] }); + AbstractSQLite.call(this, db); + let that = this; - AbstractLoki.call(this, collection); - this.idKeys = ['pubkey', 'uid', 'hash']; - this.propsToSave = [ + this.table = 'idty'; + this.fields = [ 'revoked', 'currentMSN', 'memberships', @@ -33,8 +32,39 @@ function IdentityDAL(loki) { 'hash', 'written' ]; - - this.init = () => null; + this.arrays = ['memberships']; + this.booleans = ['revoked', 'member', 'kick', 'leaving', 'wasMember', 'written']; + this.pkFields = ['pubkey', 'uid', 'hash']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'revoked BOOLEAN NOT NULL,' + + 'currentMSN INTEGER NOT NULL,' + + 'memberships TEXT,' + + 'time DATETIME NOT NULL,' + + 'member BOOLEAN NOT NULL,' + + 'kick BOOLEAN NOT NULL,' + + 'leaving BOOLEAN NOT NULL,' + + 'wasMember BOOLEAN NOT NULL,' + + 'pubkey VARCHAR(50) NOT NULL,' + + 'uid VARCHAR(255) NOT NULL,' + + 'sig VARCHAR(100) NOT NULL,' + + 'hash VARCHAR(40) NOT NULL,' + + 'written BOOLEAN NOT NULL,' + + 'PRIMARY KEY (pubkey,uid,hash)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_idty_pubkey ON idty (pubkey);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_uid ON idty (uid);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_kick ON idty (kick);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_member ON idty (member);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_wasMember ON idty (wasMember);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_hash ON idty (hash);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_written ON idty (written);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_currentMSN ON idty (currentMSN);' + + 'COMMIT;', []); + }); this.excludeIdentity = (pubkey) => { return co(function *() { @@ -150,52 +180,47 @@ function IdentityDAL(loki) { }); }; - this.removeUnWrittenWithPubkey = (pubkey) => this.lokiRemoveWhere({ - $and: [ - { pubkey: pubkey }, - { written: false } - ] + this.removeUnWrittenWithPubkey = (pubkey) => this.sqlRemoveWhere({ + pubkey: pubkey, + written: false }); - this.removeUnWrittenWithUID = (uid) => this.lokiRemoveWhere({ - $and: [ - { uid: uid }, - { written: false } - ] + this.removeUnWrittenWithUID = (uid) => this.sqlRemoveWhere({ + uid: uid, + written: false }); this.getFromPubkey = function(pubkey) { - return that.lokiFindOne({ - pubkey: pubkey - }, { + return that.sqlFindOne({ + pubkey: pubkey, wasMember: true - }, that.IMMUTABLE_FIELDS); + }); }; this.getFromUID = function(uid) { - return that.lokiFindOne({ - uid: uid - }, { + return that.sqlFindOne({ + uid: uid, wasMember: true - }, that.IMMUTABLE_FIELDS); + }); }; this.getByHash = function(hash) { - return that.lokiFindOne({ + return that.sqlFindOne({ hash: hash }); }; - this.saveIdentity = (idty) => this.lokiSave(idty); + this.saveIdentity = (idty) => + this.saveEntity(idty); this.getWhoIsOrWasMember = function() { - return that.lokiFindInAll({ + return that.sqlFind({ wasMember: true }); }; this.getPendingIdentities = function() { - return that.lokiFindInAll({ + return that.sqlFind({ wasMember: false }); }; @@ -203,26 +228,22 @@ function IdentityDAL(loki) { this.listLocalPending = () => Q([]); this.searchThoseMatching = function(search) { - return that.lokiFind({ - $or: [{ - pubkey: { $regex: new RegExp(search, 'i') } - },{ - uid: { $regex: new RegExp(search, 'i') } - }] + return that.sqlFindLikeAny({ + pubkey: "%" + search + "%", + uid: "%" + search + "%" }); }; this.kickMembersForMembershipBelow = (maxNumber) => co(function *() { - let toKick = yield that.lokiFind({ - currentMSN: { $lte: maxNumber } - },{ + let toKick = yield that.sqlFind({ + currentMSN: { $lte: maxNumber }, kick: false, member: true }); for (let i = 0; i < toKick.length; i++) { let idty = toKick[i]; idty.kick = true; - collection.update(idty); + yield that.saveEntity(idty); } }); } \ No newline at end of file diff --git a/app/lib/dal/sqliteDAL/LinksDAL.js b/app/lib/dal/sqliteDAL/LinksDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..90da01b68e5a01e71eaaac3d5cf0088c8b654cb7 --- /dev/null +++ b/app/lib/dal/sqliteDAL/LinksDAL.js @@ -0,0 +1,105 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var co = require('co'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = LinksDAL; + +function LinksDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'link'; + this.fields = [ + 'source', + 'target', + 'timestamp', + 'block_number', + 'block_hash', + 'obsolete' + ]; + this.arrays = []; + this.booleans = ['obsolete']; + this.pkFields = ['source', 'target', 'block_number', 'block_hash']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'source VARCHAR(50) NOT NULL,' + + 'target CHAR(40) NOT NULL,' + + 'timestamp INTEGER NOT NULL,' + + 'block_number INTEGER NOT NULL,' + + 'block_hash VARCHAR(50),' + + 'obsolete BOOLEAN NOT NULL,' + + 'PRIMARY KEY (source,target,block_number,block_hash)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_link_source ON link (source);' + + 'CREATE INDEX IF NOT EXISTS idx_link_obsolete ON link (obsolete);' + + 'CREATE INDEX IF NOT EXISTS idx_link_target ON link (target);' + + 'CREATE INDEX IF NOT EXISTS idx_link_timestamp ON link (timestamp);' + + 'COMMIT;', []); + }); + + this.getValidLinksFrom = (pubkey) => this.sqlFind({ + source: pubkey, + obsolete: false + }); + + this.getSimilarLinksFromDate = (from, to, minDate) => this.sqlFind({ + source: from, + target: to, + timestamp: { $gte: minDate } + }); + + this.getValidLinksTo = (pubkey) => this.sqlFind({ + target: pubkey, + obsolete: false + }); + + this.getLinksWithPath = (from, to) => + this.sqlFind({ + source: from, + target: to + }); + + this.obsoletesLinks = (minTimestamp) => co(function *() { + return that.sqlUpdateWhere({ obsolete: true }, { + timestamp: { $lte: minTimestamp }, + obsolete: false + }); + }); + + this.unObsoletesLinks = (minTimestamp) => co(function *() { + return that.sqlUpdateWhere({ obsolete: false }, { + timestamp: { $gte: minTimestamp } + }); + }); + + this.addLink = (link) => { + link.obsolete = false; + return that.saveEntity(link); + }; + + this.removeLink = (link) => + this.deleteEntity(link); + + this.updateBatchOfLinks = (links) => co(function *() { + let queries = []; + let insert = that.getInsertHead(); + let values = links.map((cert) => that.getInsertValue(cert)); + if (links.length) { + queries.push(insert + '\n' + values.join(',\n') + ';'); + } + if (queries.length) { + return that.exec(queries.join('\n')); + } + }); +} \ No newline at end of file diff --git a/app/lib/dal/sqliteDAL/MembershipDAL.js b/app/lib/dal/sqliteDAL/MembershipDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..a41d8e96c3e91d4d0658054145184ed74932ac3a --- /dev/null +++ b/app/lib/dal/sqliteDAL/MembershipDAL.js @@ -0,0 +1,121 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var co = require('co'); +var _ = require('underscore'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = MembershipDAL; + +function MembershipDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'membership'; + this.fields = [ + 'membership', + 'issuer', + 'number', + 'blockNumber', + 'blockHash', + 'userid', + 'certts', + 'block', + 'fpr', + 'idtyHash', + 'written', + 'signature' + ]; + this.arrays = []; + this.booleans = ['written']; + this.pkFields = ['issuer','signature']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS membership (' + + 'membership CHAR(2) NOT NULL,' + + 'issuer VARCHAR(50) NOT NULL,' + + 'number INTEGER NOT NULL,' + + 'blockNumber INTEGER,' + + 'blockHash VARCHAR(40) NOT NULL,' + + 'userid VARCHAR(255) NOT NULL,' + + 'certts DATETIME NOT NULL,' + + 'block INTEGER,' + + 'fpr VARCHAR(50),' + + 'idtyHash VARCHAR(40),' + + 'written BOOLEAN NOT NULL,' + + 'signature VARCHAR(50),' + + 'PRIMARY KEY (issuer,signature)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_mmembership_idtyHash ON membership (idtyHash);' + + 'CREATE INDEX IF NOT EXISTS idx_mmembership_membership ON membership (membership);' + + 'CREATE INDEX IF NOT EXISTS idx_mmembership_written ON membership (written);' + + 'COMMIT;', []); + }); + + this.getMembershipOfIssuer = (ms) => this.sqlExisting(ms); + + this.getMembershipsOfIssuer = (issuer) => this.sqlFind({ + issuer: issuer + }); + + this.getPendingINOfTarget = (hash) => + this.sqlFind({ + idtyHash: hash, + membership: 'IN', + written: false + }); + + this.getPendingIN = () => this.sqlFind({ + membership: 'IN', + written: false + }); + + this.getPendingOUT = () => this.sqlFind({ + membership: 'OUT', + written: false + }); + + this.unwriteMS = (ms) => co(function *() { + let existing = yield that.sqlExisting({ + issuer: ms.issuer, + signature: ms.signature + }); + if (existing) { + existing.written = false; + that.saveEntity(existing); + } + }); + + this.saveOfficialMS = (type, ms) => { + let obj = _.extend({}, ms); + obj.membership = type.toUpperCase(); + obj.written = true; + return this.saveEntity(_.pick(obj, 'membership', 'issuer', 'number', 'blockNumber', 'blockHash', 'userid', 'certts', 'block', 'fpr', 'idtyHash', 'written', 'signature')); + }; + + this.savePendingMembership = (ms) => { + ms.membership = ms.membership.toUpperCase(); + ms.written = false; + return this.saveEntity(_.pick(ms, 'membership', 'issuer', 'number', 'blockNumber', 'blockHash', 'userid', 'certts', 'block', 'fpr', 'idtyHash', 'written', 'signature')); + }; + + this.updateBatchOfMemberships = (mss) => co(function *() { + let queries = []; + let insert = that.getInsertHead(); + let values = mss.map((cert) => that.getInsertValue(cert)); + if (mss.length) { + queries.push(insert + '\n' + values.join(',\n') + ';'); + } + if (queries.length) { + return that.exec(queries.join('\n')); + } + }); +} diff --git a/app/lib/dal/sqliteDAL/PeerDAL.js b/app/lib/dal/sqliteDAL/PeerDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..b721b0c1670992195b2a364f2635342c7785466c --- /dev/null +++ b/app/lib/dal/sqliteDAL/PeerDAL.js @@ -0,0 +1,66 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var co = require('co'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = PeerDAL; + +function PeerDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'peer'; + this.fields = [ + 'version', + 'currency', + 'status', + 'statusTS', + 'hash', + 'first_down', + 'last_try', + 'pubkey', + 'block', + 'signature', + 'endpoints', + 'raw' + ]; + this.arrays = ['endpoints']; + this.booleans = []; + this.pkFields = ['pubkey']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'version INTEGER NOT NULL,' + + 'currency VARCHAR(50) NOT NULL,' + + 'status VARCHAR(10),' + + 'statusTS INTEGER NOT NULL,' + + 'hash CHAR(40),' + + 'first_down INTEGER,' + + 'last_try INTEGER,' + + 'pubkey VARCHAR(50) NOT NULL,' + + 'block VARCHAR(60) NOT NULL,' + + 'signature VARCHAR(100),' + + 'endpoints TEXT NOT NULL,' + + 'raw TEXT,' + + 'PRIMARY KEY (pubkey)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_link_source ON peer (pubkey);' + + 'COMMIT;', []); + }); + + this.listAll = () => this.sqlListAll(); + + this.getPeer = (pubkey) => this.sqlFindOne({ pubkey: pubkey }); + + this.savePeer = (peer) => this.saveEntity(peer); + + this.removeAll = () => this.sqlDeleteAll(); +} diff --git a/app/lib/dal/sqliteDAL/SourcesDAL.js b/app/lib/dal/sqliteDAL/SourcesDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..b2127f5490ac05880ff8ccf206e8eed5f799b3c9 --- /dev/null +++ b/app/lib/dal/sqliteDAL/SourcesDAL.js @@ -0,0 +1,153 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var co = require('co'); +var _ = require('underscore'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = SourcesDAL; + +function SourcesDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'source'; + this.fields = [ + 'pubkey', + 'type', + 'number', + 'time', + 'fingerprint', + 'amount', + 'block_hash', + 'consumed' + ]; + this.arrays = []; + this.bigintegers = ['amount']; + this.booleans = ['consumed']; + this.pkFields = ['pubkey', 'type', 'number', 'fingerprint', 'amount']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'pubkey VARCHAR(50) NOT NULL,' + + 'type VARCHAR(1) NOT NULL,' + + 'number INTEGER NOT NULL,' + + 'time DATETIME,' + + 'fingerprint VARCHAR(40) NOT NULL,' + + 'amount VARCHAR(50) NOT NULL,' + + 'block_hash VARCHAR(40) NOT NULL,' + + 'consumed BOOLEAN NOT NULL,' + + 'PRIMARY KEY (pubkey,type,number,fingerprint,amount)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_source_pubkey ON source (pubkey);' + + 'CREATE INDEX IF NOT EXISTS idx_source_type ON source (type);' + + 'CREATE INDEX IF NOT EXISTS idx_source_number ON source (number);' + + 'CREATE INDEX IF NOT EXISTS idx_source_fingerprint ON source (fingerprint);' + + 'COMMIT;', []); + }); + + this.getAvailableForPubkey = (pubkey) => this.sqlFind({ + pubkey: pubkey, + consumed: false + }); + + this.getUDSources = (pubkey) => this.sqlFind({ + pubkey: pubkey, + type: 'D' + }); + + this.getSource = (pubkey, type, number) => this.sqlFindOne({ + pubkey: pubkey, + type: type, + number: number + }); + + this.isAvailableSource = (pubkey, type, number, fingerprint, amount) => co(function *() { + let src = yield that.sqlExisting({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount + }); + return src ? !src.consumed : false; + }); + + this.consumeSource = (pubkey, type, number, fingerprint, amount) => co(function *() { + return that.updateEntity({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount + },{ + consumed: true + }); + }); + + this.addSource = (state, pubkey, type, number, fingerprint, amount, block_hash, time) => this.saveEntity({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount, + time: time, + block_hash: block_hash, + consumed: false + }); + + this.unConsumeSource = (type, pubkey, number, fingerprint, amount, time, block_hash) => co(function *() { + let src = yield that.sqlExisting({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount + }); + if (src) { + src.consumed = false; + that.saveEntity(src); + } else { + return that.updateEntity({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount, + block_hash: block_hash + },{ + consumed: false + }); + } + }); + + this.updateBatchOfSources = (sources) => co(function *() { + let inserts = _.filter(sources, { toConsume: false }); + let updates = _.filter(sources, { toConsume: true }); + let queries = []; + if (inserts.length) { + let insert = that.getInsertHead(); + let values = inserts.map((src) => that.getInsertValue(_.extend(src, { consumed: false }))); + queries.push(insert + '\n' + values.join(',\n') + ';'); + } + if (updates.length) { + let del = that.getDeleteHead(); + let values = that.getDeleteValues(updates); + queries.push(del + '\n' + values + ';'); + } + if (queries.length) { + return that.exec(queries.join('\n')); + } + }); + + this.removeAllSourcesOfBlock = (number) => this.sqlRemoveWhere({ + number: number + }); +} \ No newline at end of file diff --git a/app/lib/dal/sqliteDAL/TxsDAL.js b/app/lib/dal/sqliteDAL/TxsDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..bbfdfcc55262c4b47a2cd28100f4b2b499dc0a95 --- /dev/null +++ b/app/lib/dal/sqliteDAL/TxsDAL.js @@ -0,0 +1,138 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var co = require('co'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = TxsDAL; + +function TxsDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'txs'; + this.fields = [ + 'hash', + 'block_number', + 'version', + 'currency', + 'comment', + 'time', + 'written', + 'removed', + 'inputs', + 'outputs', + 'issuers', + 'signatories', + 'signatures', + 'recipients' + ]; + this.arrays = ['inputs','outputs','issuers','signatories','signatures','recipients']; + this.booleans = ['written','removed']; + this.pkFields = ['hash']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'hash CHAR(40) NOT NULL,' + + 'block_number INTEGER,' + + 'version INTEGER NOT NULL,' + + 'currency VARCHAR(50) NOT NULL,' + + 'comment VARCHAR(255) NOT NULL,' + + 'time DATETIME,' + + 'inputs TEXT NOT NULL,' + + 'outputs TEXT NOT NULL,' + + 'issuers TEXT NOT NULL,' + + 'signatories TEXT NOT NULL,' + + 'signatures TEXT NOT NULL,' + + 'recipients TEXT NOT NULL,' + + 'written BOOLEAN NOT NULL,' + + 'removed BOOLEAN NOT NULL,' + + 'PRIMARY KEY (hash)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_txs_issuers ON txs (issuers);' + + 'CREATE INDEX IF NOT EXISTS idx_txs_written ON txs (written);' + + 'CREATE INDEX IF NOT EXISTS idx_txs_removed ON txs (removed);' + + 'CREATE INDEX IF NOT EXISTS idx_txs_hash ON txs (hash);' + + 'COMMIT;', []); + }); + + this.getAllPending = () => this.sqlFind({ + written: false, + removed: false + }); + + this.getTX = (hash) => this.sqlFindOne({ + hash: hash + }); + + this.removeTX = (hash) => co(function *() { + let tx = yield that.sqlFindOne({ + hash: hash + }); + if (tx) { + tx.removed = true; + return that.saveEntity(tx); + } + return Q(tx); + }); + + this.addLinked = (tx) => { + tx.written = true; + tx.removed = false; + tx.hash = tx.getHash(true); + tx.recipients = tx.outputs.map(function(out) { + return out.match('(.*):')[1]; + }); + return that.saveEntity(tx); + }; + + this.addPending = (tx) => { + tx.written = false; + tx.removed = false; + tx.hash = tx.getHash(true); + tx.recipients = tx.outputs.map(function(out) { + return out.match('(.*):')[1]; + }); + return this.saveEntity(tx); + }; + + this.getLinkedWithIssuer = (pubkey) => this.sqlFind({ + issuers: { $contains: pubkey }, + written: true + }); + + this.getLinkedWithRecipient = (pubkey) => this.sqlFind({ + recipients: { $contains: pubkey }, + written: true + }); + + this.getPendingWithIssuer = (pubkey) => this.sqlFind({ + issuers: { $contains: pubkey }, + written: false + }); + + this.getPendingWithRecipient = (pubkey) => this.sqlFind({ + recipients: { $contains: pubkey }, + written: false + }); + + this.updateBatchOfTxs = (txs) => co(function *() { + let queries = []; + let insert = that.getInsertHead(); + let values = txs.map((cert) => that.getInsertValue(cert)); + if (txs.length) { + queries.push(insert + '\n' + values.join(',\n') + ';'); + } + if (queries.length) { + return that.exec(queries.join('\n')); + } + }); +} \ No newline at end of file diff --git a/app/service/BlockchainService.js b/app/service/BlockchainService.js index 9c7e5ceb71903414dbcdbfb1cdd5c16ac36d928e..52f1e3925897a3a59afc0855e0a3fd3c0063c5c4 100644 --- a/app/service/BlockchainService.js +++ b/app/service/BlockchainService.js @@ -1270,6 +1270,15 @@ function BlockchainService (conf, mainDAL, pair) { if (blocks[0].number == 0) { yield that.saveParametersForRootBlock(blocks[0]); } + // Helper to retrieve a block with local cache + let getBlockOrNull = (number) => { + let firstLocalNumber = blocks[0].number; + if (number >= firstLocalNumber) { + let offset = number - firstLocalNumber; + return Q(blocks[offset]); + } + return mainDAL.getBlockOrNull(number); + }; // Insert a bunch of blocks let lastPrevious = blocks[0].number == 0 ? null : yield mainDAL.getBlock(blocks[0].number - 1); let rootBlock = (blocks[0].number == 0 ? blocks[0] : null) || (yield mainDAL.getBlockOrNull(0)); @@ -1294,50 +1303,26 @@ function BlockchainService (conf, mainDAL, pair) { } else { block.UDTime = previousBlock.UDTime; } - // Transactions & Memberships recording - yield mainDAL.saveTxsInFiles(block.transactions, { block_number: block.number, time: block.medianTime }); - yield mainDAL.saveMemberships('join', block.joiners); - yield mainDAL.saveMemberships('active', block.actives); - yield mainDAL.saveMemberships('leave', block.leavers); yield Q.Promise(function(resolve, reject){ - // Compute resulting entities - async.waterfall([ - function (next) { - // Create/Update members (create new identities if do not exist) - mainContext.updateMembers(block, next); - }, - function (next) { - // Create/Update certifications - mainContext.updateCertifications(block, next); - }, - function (next) { - // Create/Update memberships - mainContext.updateMemberships(block, next); - }, - function (next){ - // Save links - mainContext.updateLinks(block, next, (number) => { - let firstLocalNumber = blocks[0].number; - if (number >= firstLocalNumber) { - let offset = number - firstLocalNumber; - return Q(blocks[offset]); - } - return mainDAL.getBlockOrNull(number); - }); - }, - function (next){ - // Update consumed sources & create new ones - mainContext.updateTransactionSources(block, next); - } - ], function (err) { + // Create/Update members (create new identities if do not exist) + mainContext.updateMembers(block, function (err) { if (err) return reject(err); resolve(); }); }); } + // Transactions recording + yield mainContext.updateTransactionsForBlocks(blocks); + // Create certifications + yield mainContext.updateMembershipsForBlocks(blocks); + // Create certifications + yield mainContext.updateLinksForBlocks(blocks, getBlockOrNull); + // Create certifications + yield mainContext.updateCertificationsForBlocks(blocks); + // Create / Update sources + yield mainContext.updateTransactionSourcesForBlocks(blocks); yield mainDAL.blockDAL.saveBunch(blocks, (targetLastNumber - lastBlockToSave.number) > maxBlock); yield pushStatsForBlocks(blocks); - //console.log('Saved'); }); function pushStatsForBlocks(blocks) { diff --git a/bin/ucoind b/bin/ucoind index 1312178575b4ec2f0f56c0bc771e3a2959b5830a..a1cc1b9805b364266381e3911d354342840cecab 100755 --- a/bin/ucoind +++ b/bin/ucoind @@ -179,7 +179,7 @@ program program .command('init [host] [port]') .description('Setup a node configuration and sync data with given node') - .action(connect(bootstrapServer, true, true)); + .action(connect(bootstrapServer, true)); program .command('forward [host] [port] [to]') @@ -415,7 +415,7 @@ program server.disconnect(); process.exit(); }); - }, true)); + })); program .command('reset [config|data|peers|tx|stats|all]') @@ -693,7 +693,7 @@ function commandLineConf(conf) { return _(conf).extend({ routing: true }); } -function connect(callback, forConf, useDefaultConf) { +function connect(callback, useDefaultConf) { return function () { var cbArgs = arguments; var dbName = program.mdb || "ucoin_default"; @@ -702,7 +702,7 @@ function connect(callback, forConf, useDefaultConf) { var server = ucoin({ home: dbHome, name: dbName }, commandLineConf()); // Initialize server (db connection, ...) - server.connectDB(forConf, useDefaultConf) + server.connectDB(useDefaultConf) .then(function(){ cbArgs.length--; cbArgs[cbArgs.length++] = server; diff --git a/package.json b/package.json index e212caa832446936e1b28c119f05f654bb2fc174..be316a7393ba49520ff6466b48b38aceefcfe14c 100644 --- a/package.json +++ b/package.json @@ -36,13 +36,13 @@ "async": "0.2.9", "bindings": "1.2.1", "co": "4.6.0", + "colors": "1.1.2", "commander": "2.1.0", "event-stream": "3.1.5", "express": "3.4.7", "express-cors": "0.0.3", "inquirer": "0.8.5", "log4js": "0.6.9", - "lokijs": "1.3.10", "merkle": "0.1.0", "moment": "2.6.0", "multimeter": "0.1.1", @@ -55,6 +55,7 @@ "scrypt": "5.4.1", "sha1": "1.1.0", "socket.io": "1.3.7", + "sqlite3": "3.1.1", "superagent": "1.4.0", "tweetnacl": "0.11.2", "underscore": "1.8.3", diff --git a/server.js b/server.js index de73c823688b76a432074c9a41d4e49b06e9365f..18051e6d48abe21ab224e1934a412b95c82437e4 100644 --- a/server.js +++ b/server.js @@ -48,9 +48,9 @@ function Server (dbConf, overrideConf) { }); }; - this.connectDB = function (forConf, useDefaultConf) { + this.connectDB = function (useDefaultConf) { // Connect only once - return connectionPromise || (connectionPromise = that.connect(forConf, useDefaultConf)); + return connectionPromise || (connectionPromise = that.connect(useDefaultConf)); }; this.initWithServices = function (done) { @@ -93,13 +93,13 @@ function Server (dbConf, overrideConf) { this.submitP = (obj, isInnerWrite) => Q.nbind(this.submit, this)(obj, isInnerWrite); - this.connect = function (forConf, useDefaultConf) { + this.connect = function (useDefaultConf) { // Init connection if (that.dal) { return Q(); } var dbType = dbConf && dbConf.memory ? fileDAL.memory : fileDAL.file; - return dbType(dbConf.name || "default", dbConf.home, forConf) + return dbType(dbConf.name || "default", dbConf.home) .then(function(dal){ that.dal = dal; return that.dal.init(overrideConf, useDefaultConf); diff --git a/test/dal/dal.js b/test/dal/dal.js index b28430f0d8bb37ab4af78bcfefee65e49ed5427e..17adf43516c16078a8ed37cac32e98d404b544b4 100644 --- a/test/dal/dal.js +++ b/test/dal/dal.js @@ -116,22 +116,22 @@ var mocks = { "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw" ], "leavers" : [ - "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:ccJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", - "HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk:1lFIiaR0QX0jibr5zQpXVGzBvMGqcsTRlmHiwGz5HOAZT8PTdVUb5q6YGZ6qAUZjdMjPmhLaiMIpYc47wUnzBA==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421786393:cgeek", - "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:ctyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", - "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:uoiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" + "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:3cJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", + "HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk:3lFIiaR0QX0jibr5zQpXVGzBvMGqcsTRlmHiwGz5HOAZT8PTdVUb5q6YGZ6qAUZjdMjPmhLaiMIpYc47wUnzBA==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421786393:cgeek", + "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:3tyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", + "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:3oiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" ], "actives" : [ - "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:ccJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", - "HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk:1lFIiaR0QX0jibr5zQpXVGzBvMGqcsTRlmHiwGz5HOAZT8PTdVUb5q6YGZ6qAUZjdMjPmhLaiMIpYc47wUnzBA==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421786393:cgeek", - "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:ctyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", - "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:uoiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" + "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:2cJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", + "HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk:2lFIiaR0QX0jibr5zQpXVGzBvMGqcsTRlmHiwGz5HOAZT8PTdVUb5q6YGZ6qAUZjdMjPmhLaiMIpYc47wUnzBA==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421786393:cgeek", + "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:2tyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", + "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:2oiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" ], "joiners" : [ - "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:ccJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", + "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:1cJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", "HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk:1lFIiaR0QX0jibr5zQpXVGzBvMGqcsTRlmHiwGz5HOAZT8PTdVUb5q6YGZ6qAUZjdMjPmhLaiMIpYc47wUnzBA==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421786393:cgeek", - "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:ctyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", - "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:uoiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" + "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:1tyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", + "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:1oiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" ], "identities" : [ "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:Ot3zIp/nsHT3zgJy+2YcXPL6vaM5WFsD+F8w3qnJoBRuBG6lv761zoaExp2iyUnm8fDAyKPpMxRK2kf437QSCw==:1421787800:inso", @@ -169,7 +169,7 @@ describe("DAL", function(){ it('should have no peer in a first time', function(){ return fileDAL.listAllPeers().then(function(peers){ peers.should.have.length(0); - }) + }); }); it('should have 1 peer if 1 is created', function(){ @@ -182,19 +182,19 @@ describe("DAL", function(){ peers[0].should.have.property('currency').equal('bb'); peers[0].should.have.property('endpoints').length(1); peers[0].endpoints[0].should.equal('BASIC_MERKLED_API localhost 7777'); - }) + }); }); it('should have no current block', function(){ return fileDAL.getCurrentBlockOrNull().then(function(current){ should.not.exist(current); - }) + }); }); it('should have no blocks in first time', function(){ return fileDAL.getCurrentBlockOrNull().then(function(block){ should.not.exist(block); - }) + }); }); it('should be able to save a Block', function(){ diff --git a/test/integration/identity-clean-test.js b/test/integration/identity-clean-test.js index eb1e70b50f4f4db7a3d939f239ff1170d716d151..24b7e1a0b227f4828726dda138871d164665f519 100644 --- a/test/integration/identity-clean-test.js +++ b/test/integration/identity-clean-test.js @@ -53,7 +53,6 @@ describe("Identities cleaned", function() { yield cat.selfCertPromise(now); yield tic.selfCertPromise(now); yield toc.selfCertPromise(now); - yield tic.selfCertPromise(now + 1); yield expectAnswer(rp('http://127.0.0.1:7733/wot/lookup/cat', { json: true }), function(res) { res.should.have.property('results').length(2); @@ -66,6 +65,7 @@ describe("Identities cleaned", function() { yield tic.certPromise(cat); yield cat.joinPromise(); yield tic.joinPromise(); + yield tic.selfCertPromise(now + 1); yield commitS1(); // We have the following WoT (diameter 1):