diff --git a/.travis.yml b/.travis.yml index 81e3f29f9d24c35cce2a6092afb96ed7355c9d0a..13eb4542221b3d3c7b560773d32991d365414bc5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,15 @@ language: node_js node_js: - - 0.12 + - 4.2.0 +env: + - CXX=g++-4.8 +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-4.8 sudo: false diff --git a/app/controllers/abstract.js b/app/controllers/abstract.js new file mode 100644 index 0000000000000000000000000000000000000000..9c57444660af390e3936318a48e633eb2ee47b31 --- /dev/null +++ b/app/controllers/abstract.js @@ -0,0 +1,15 @@ + +"use strict"; +var co = require('co'); +var dos2unix = require('../lib/dos2unix'); + +module.exports = function AbstractController (server) { + + this.pushEntity = (req, rawer, parser) => co(function *() { + let rawDocument = rawer(req); + rawDocument = dos2unix(rawDocument); + let obj = parser.syncWrite(rawDocument); + let written = yield server.singleWritePromise(obj); + return written.json(); + }); +}; diff --git a/app/controllers/blockchain.js b/app/controllers/blockchain.js index 3c04c582217c7094f6d046b3d9ec1c355af46761..a073f5451016a409d01db0f4ddd6ae7c4c094459 100644 --- a/app/controllers/blockchain.js +++ b/app/controllers/blockchain.js @@ -1,20 +1,15 @@ "use strict"; -var _ = require('underscore'); -var Q = require('q'); var co = require('co'); var async = require('async'); -var es = require('event-stream'); var moment = require('moment'); -var dos2unix = require('../lib/dos2unix'); +var constants = require('../lib/constants'); var http2raw = require('../lib/streams/parsers/http2raw'); -var jsoner = require('../lib/streams/jsoner'); -var http400 = require('../lib/http/http400'); var parsers = require('../lib/streams/parsers/doc'); var blockchainDao = require('../lib/blockchainDao'); -var localValidator = require('../lib/localValidator'); var globalValidator = require('../lib/globalValidator'); var Membership = require('../lib/entity/membership'); +var AbstractController = require('./abstract'); module.exports = function (server) { return new BlockchainBinding(server); @@ -22,9 +17,9 @@ module.exports = function (server) { function BlockchainBinding (server) { + AbstractController.call(this, server); + var conf = server.conf; - var local = localValidator(conf); - var global = globalValidator(conf); // Services var ParametersService = server.ParametersService; @@ -35,44 +30,11 @@ function BlockchainBinding (server) { var Block = require('../lib/entity/block'); var Stat = require('../lib/entity/stat'); - this.parseMembership = function (req, res) { - res.type('application/json'); - var onError = http400(res); - http2raw.membership(req, onError) - .pipe(dos2unix()) - .pipe(parsers.parseMembership(onError)) - .pipe(local.versionFilter(onError)) - .pipe(global.currencyFilter(onError)) - .pipe(server.singleWriteStream(onError)) - .pipe(jsoner()) - .pipe(es.stringify()) - .pipe(res); - }; + this.parseMembership = (req) => this.pushEntity(req, http2raw.membership, parsers.parseMembership); - this.parseBlock = function (req, res) { - res.type('application/json'); - var onError = http400(res); - http2raw.block(req, onError) - .pipe(dos2unix()) - .pipe(parsers.parseBlock(onError)) - .pipe(local.versionFilter(onError)) - .pipe(global.currencyFilter(onError)) - .pipe(server.singleWriteStream(onError)) - .pipe(jsoner()) - .pipe(es.stringify()) - .pipe(res); - }; + this.parseBlock = (req) => this.pushEntity(req, http2raw.block, parsers.parseBlock); - this.parameters = function (req, res) { - res.type('application/json'); - server.dal.getParameters() - .then(function(parameters){ - res.send(200, JSON.stringify(parameters, null, " ")); - }) - .catch(function(){ - res.send(200, JSON.stringify({}, null, " ")); - }) - }; + this.parameters = () => server.dal.getParameters(); this.with = { @@ -87,145 +49,81 @@ function BlockchainBinding (server) { }; function getStat (statName) { - return function (req, res) { - async.waterfall([ - function (next) { - server.dal.getStat(statName).then(_.partial(next, null)).catch(next); - } - ], function (err, stat) { - if(err){ - res.send(400, err); - return; - } - res.type('application/json'); - res.send(200, JSON.stringify({ result: new Stat(stat).json() }, null, " ")); - }); - } - } - - this.promoted = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next){ - ParametersService.getNumber(req, next); - }, - function (number, next){ - BlockchainService.promoted(number, next); - } - ], function (err, promoted) { - if(err){ - res.send(404, err && (err.message || err)); - return; - } - res.send(200, JSON.stringify(new Block(promoted).json(), null, " ")); - }); - }; - - this.blocks = function (req, res) { - res.type('application/json'); - co(function *() { - try { - let params = ParametersService.getCountAndFrom(req); - var count = parseInt(params.count); - var from = parseInt(params.from); - let blocks = yield BlockchainService.blocksBetween(from, count); - blocks = blocks.map((b) => (new Block(b).json())); - res.send(200, JSON.stringify(blocks, null, " ")); - } catch(e) { - res.send(400, e); - } - }); - }; - - this.current = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next){ - BlockchainService.current(next); - } - ], function (err, current) { - if(err || !current){ - res.send(404, err); - return; - } - res.send(200, JSON.stringify(new Block(current).json(), null, " ")); - }); - }; - - this.hardship = function (req, res) { - res.type('application/json'); - return co(function *() { - let nextBlockNumber = 0; - try { - let search = yield ParametersService.getSearchP(req); - let idty = yield IdentityService.findMemberWithoutMemberships(search); - if (!idty) { - throw 'Identity not found'; - } - if (!idty.member) { - throw 'Not a member'; - } - let current = yield BlockchainService.current(); - if (current) { - nextBlockNumber = current ? current.number + 1 : 0; - } - let nbZeros = yield globalValidator(conf, blockchainDao(null, server.dal)).getTrialLevel(idty.pubkey); - res.send(200, JSON.stringify({ - "block": nextBlockNumber, - "level": nbZeros - }, null, " ")); - } catch(e) { - res.send(400, e); - } + return () => co(function *() { + let stat = yield server.dal.getStat(statName); + return { result: new Stat(stat).json() }; }); - }; + } - this.memberships = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next){ - ParametersService.getSearch(req, next); - }, - function (search, next){ - IdentityService.findMember(search, next); - } - ], function (err, idty) { - if(err){ - res.send(400, err); - return; - } - var json = { - pubkey: idty.pubkey, - uid: idty.uid, - sigDate: moment(idty.time).unix(), - memberships: [] - }; - idty.memberships.forEach(function(ms){ - ms = new Membership(ms); - json.memberships.push({ - version: ms.version, - currency: conf.currency, - membership: ms.membership, - blockNumber: parseInt(ms.blockNumber), - blockHash: ms.blockHash - }); + this.promoted = (req) => co(function *() { + let number = yield ParametersService.getNumberP(req); + let promoted = yield BlockchainService.promoted(number); + return new Block(promoted).json(); + }); + + this.blocks = (req) => co(function *() { + let params = ParametersService.getCountAndFrom(req); + var count = parseInt(params.count); + var from = parseInt(params.from); + let blocks = yield BlockchainService.blocksBetween(from, count); + blocks = blocks.map((b) => (new Block(b).json())); + return blocks; + }); + + this.current = () => co(function *() { + let current = yield server.dal.getCurrentBlockOrNull(); + if (!current) throw constants.ERRORS.NO_CURRENT_BLOCK; + return new Block(current).json(); + }); + + this.hardship = (req) => co(function *() { + let nextBlockNumber = 0; + let search = yield ParametersService.getSearchP(req); + let idty = yield IdentityService.findMemberWithoutMemberships(search); + if (!idty) { + throw constants.ERRORS.NO_MATCHING_IDENTITY; + } + if (!idty.member) { + throw constants.ERRORS.NOT_A_MEMBER; + } + let current = yield BlockchainService.current(); + if (current) { + nextBlockNumber = current ? current.number + 1 : 0; + } + let nbZeros = yield globalValidator(conf, blockchainDao(null, server.dal)).getTrialLevel(idty.pubkey); + return { + "block": nextBlockNumber, + "level": nbZeros + }; + }); + + this.memberships = (req) => co(function *() { + let search = yield ParametersService.getSearchP(req); + let idty = yield IdentityService.findMember(search); + var json = { + pubkey: idty.pubkey, + uid: idty.uid, + sigDate: moment(idty.time).unix(), + memberships: [] + }; + idty.memberships.forEach(function(ms){ + ms = new Membership(ms); + json.memberships.push({ + version: ms.version, + currency: conf.currency, + membership: ms.membership, + blockNumber: parseInt(ms.blockNumber), + blockHash: ms.blockHash }); - res.send(200, JSON.stringify(json, null, " ")); }); - }; - - this.branches = function (req, res) { - res.type('application/json'); - co(function *() { - let branches = yield BlockchainService.branches(); - let blocks = branches.map((b) => new Block(b).json()); - res.send(200, JSON.stringify({ - blocks: blocks - }, null, " ")); - }) - .catch(function(err){ - console.error(err.stack || err.message || err); - res.send(404, err && (err.message || err)); - }); - }; + return json; + }); + + this.branches = () => co(function *() { + let branches = yield BlockchainService.branches(); + let blocks = branches.map((b) => new Block(b).json()); + return { + blocks: blocks + }; + }); } diff --git a/app/controllers/network.js b/app/controllers/network.js index 990f2ffd54a96f032266e4797f4a3eea53b796bf..326782f244b7c5e5244714f894cdcc08da6d2d5b 100644 --- a/app/controllers/network.js +++ b/app/controllers/network.js @@ -1,26 +1,21 @@ "use strict"; var _ = require('underscore'); var co = require('co'); +var Q = require('q'); var async = require('async'); -var es = require('event-stream'); -var dos2unix = require('../lib/dos2unix'); -var localValidator = require('../lib/localValidator'); -var globalValidator = require('../lib/globalValidator'); var http2raw = require('../lib/streams/parsers/http2raw'); -var jsoner = require('../lib/streams/jsoner'); -var http400 = require('../lib/http/http400'); var parsers = require('../lib/streams/parsers/doc'); var constants = require('../lib/constants'); var Peer = require('../lib/entity/peer'); +var AbstractController = require('./abstract'); -module.exports = function (server, conf) { - return new NetworkBinding(server, conf); +module.exports = function (server) { + return new NetworkBinding(server); }; -function NetworkBinding (server, conf) { +function NetworkBinding (server) { - var local = localValidator(conf); - var global = globalValidator(conf); + AbstractController.call(this, server); // Services var MerkleService = server.MerkleService; @@ -28,76 +23,45 @@ function NetworkBinding (server, conf) { this.cert = PeeringService.cert; - this.peer = function (req, res) { - res.type('application/json'); + this.peer = () => co(function *() { var p = PeeringService.peer(); - p ? res.send(200, JSON.stringify(p.json(), null, " ")) : res.send(500, 'Self peering was not found.'); - }; + if (!p) { + throw constants.ERRORS.SELF_PEER_NOT_FOUND; + } + return p.json(); + }); - this.peersGet = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next){ - server.dal.merkleForPeers(next); - }, - function (merkle, next){ - MerkleService.processForURL(req, merkle, function (hashes, done) { - server.dal.findPeersWhoseHashIsIn(hashes) - .then(function(peers) { - var map = {}; - peers.forEach(function (peer){ - map[peer.hash] = Peer.statics.peerize(peer).json(); - }); - done(null, map); - }); - }, next); - } - ], function (err, json) { - if(err){ - res.send(500, err); - return; - } - MerkleService.merkleDone(req, res, json); + this.peersGet = (req) => co(function *() { + let merkle = yield server.dal.merkleForPeers(); + return Q.nfcall(MerkleService.processForURL, req, merkle, function (hashes, done) { + server.dal.findPeersWhoseHashIsIn(hashes) + .then(function(peers) { + var map = {}; + peers.forEach(function (peer){ + map[peer.hash] = Peer.statics.peerize(peer).json(); + }); + done(null, map); + }); }); - }; + }); - this.peersPost = function (req, res) { - res.type('application/json'); - var onError = http400(res); - http2raw.peer(req, onError) - .pipe(dos2unix()) - .pipe(parsers.parsePeer(onError)) - .pipe(local.versionFilter(onError)) - .pipe(global.currencyFilter(onError)) - .pipe(server.singleWriteStream(onError)) - .pipe(jsoner()) - .pipe(es.stringify()) - .pipe(res); - }; + this.peersPost = (req) => this.pushEntity(req, http2raw.peer, parsers.parsePeer); - this.peers = function (req, res) { - res.type('application/json'); - co(function *() { - try { - let peers = yield server.dal.listAllPeers(); - var json = { - peers: peers.map((p) => { - return _.pick(p, - 'version', - 'currency', - 'status', - 'first_down', - 'last_try', - 'pubkey', - 'block', - 'signature', - 'endpoints'); - }) - }; - res.send(200, JSON.stringify(json, null, " ")); - } catch (err) { - res.send(400, err); - } - }); - }; + this.peers = () => co(function *() { + let peers = yield server.dal.listAllPeers(); + return { + peers: peers.map((p) => { + return _.pick(p, + 'version', + 'currency', + 'status', + 'first_down', + 'last_try', + 'pubkey', + 'block', + 'signature', + 'endpoints'); + }) + }; + }); } diff --git a/app/controllers/node.js b/app/controllers/node.js index 49a4ce8a13d183dfdda42cbc47b71a0c87b244ff..0d8ac7dbb9b299db47582b0361626b4bb4608d4d 100644 --- a/app/controllers/node.js +++ b/app/controllers/node.js @@ -1,21 +1,18 @@ "use strict"; -var co = require('co'); - module.exports = function (server) { return new NodeBinding(server); }; function NodeBinding (server) { - this.summary = function (req, res) { - res.type('application/json'); - res.send(200, JSON.stringify({ + this.summary = () => { + return { "ucoin": { "software": "ucoind", "version": server.version, "forkWindowSize": server.conf.forksize } - }, null, " ")); + }; }; } diff --git a/app/controllers/transactions.js b/app/controllers/transactions.js index 6b7b6c1bcc2507ac07dfadd7c0fb50f1c4cc9cc5..cec9883530c466332aea7fcc8c52c9562db66214 100644 --- a/app/controllers/transactions.js +++ b/app/controllers/transactions.js @@ -1,16 +1,12 @@ "use strict"; +var co = require('co'); +var Q = require('q'); var async = require('async'); var _ = require('underscore'); -var es = require('event-stream'); -var jsoner = require('../lib/streams/jsoner'); -var dos2unix = require('../lib/dos2unix'); -var localValidator = require('../lib/localValidator'); -var globalValidator = require('../lib/globalValidator'); var http2raw = require('../lib/streams/parsers/http2raw'); -var http400 = require('../lib/http/http400'); var parsers = require('../lib/streams/parsers/doc'); -var logger = require('../lib/logger')('transaction'); var Transaction = require('../lib/entity/transaction'); +var AbstractController = require('./abstract'); module.exports = function (server) { return new TransactionBinding(server); @@ -18,9 +14,9 @@ module.exports = function (server) { function TransactionBinding(server) { + AbstractController.call(this, server); + var conf = server.conf; - var local = localValidator(conf); - var global = globalValidator(conf); // Services var ParametersService = server.ParametersService; @@ -28,168 +24,75 @@ function TransactionBinding(server) { // Models var Source = require('../lib/entity/source'); - this.parseTransaction = function (req, res) { - res.type('application/json'); - var onError = http400(res); - http2raw.transaction(req, onError) - .pipe(dos2unix()) - .pipe(parsers.parseTransaction(onError)) - .pipe(local.versionFilter(onError)) - .pipe(global.currencyFilter(onError)) - .pipe(server.singleWriteStream(onError)) - .pipe(jsoner()) - .pipe(es.stringify()) - .pipe(res); - }; - - this.getSources = function (req, res) { - res.type('application/json'); - var pubkey = ""; - async.waterfall([ - function (next) { - ParametersService.getPubkey(req, next); - }, - function (pPubkey, next) { - pubkey = pPubkey; - server.dal.getAvailableSourcesByPubkey(pubkey).then(_.partial(next, null)).catch(next); - }, - function (sources, next) { - var result = { - "currency": conf.currency, - "pubkey": pubkey, - "sources": [] - }; - sources.forEach(function (src) { - result.sources.push(new Source(src).json()); - }); - next(null, result); - } - ], function (err, result) { - if (err) { - res.send(500, err); - } else { - res.send(200, JSON.stringify(result, null, " ")); - } - }); - }; + let getHistoryP = (pubkey, filter) => Q.nbind(getHistory, this)(pubkey, filter); - this.getHistory = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next) { - ParametersService.getPubkey(req, next); - }, - function (pubkey, next) { - getHistory(pubkey, function(results) { - return results; - }, next); - } - ], function (err, result) { - if (err) { - res.send(500, err); - } else { - res.send(200, JSON.stringify(result, null, " ")); - } + this.parseTransaction = (req) => this.pushEntity(req, http2raw.transaction, parsers.parseTransaction); + + this.getSources = (req) => co(function *() { + let pubkey = yield ParametersService.getPubkeyP(req); + let sources = yield server.dal.getAvailableSourcesByPubkey(pubkey); + var result = { + "currency": conf.currency, + "pubkey": pubkey, + "sources": [] + }; + sources.forEach(function (src) { + result.sources.push(new Source(src).json()); }); - }; + return result; + }); - this.getHistoryBetweenBlocks = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next) { - async.parallel({ - pubkey: ParametersService.getPubkey.bind(ParametersService, req), - from: ParametersService.getFrom.bind(ParametersService, req), - to: ParametersService.getTo.bind(ParametersService, req) - }, next); - }, - function (res, next) { - var pubkey = res.pubkey, from = res.from, to = res.to; - getHistory(pubkey, function(res) { - var histo = res.history; - histo.sent = _.filter(histo.sent, function(tx){ return tx && tx.block_number >= from && tx.block_number <= to; }); - histo.received = _.filter(histo.received, function(tx){ return tx && tx.block_number >= from && tx.block_number <= to; }); - _.extend(histo, { sending: [], receiving: [] }); - return res; - }, next); - } - ], function (err, result) { - if (err) { - res.send(500, err); - } else { - res.send(200, JSON.stringify(result, null, " ")); - } + this.getHistory = (req) => co(function *() { + let pubkey = yield ParametersService.getPubkeyP(req); + return getHistoryP(pubkey, (results) => results); + }); + + this.getHistoryBetweenBlocks = (req) => co(function *() { + let pubkey = yield ParametersService.getPubkeyP(req); + let from = yield ParametersService.getFromP(req); + let to = yield ParametersService.getToP(req); + return getHistoryP(pubkey, (res) => { + var histo = res.history; + histo.sent = _.filter(histo.sent, function(tx){ return tx && tx.block_number >= from && tx.block_number <= to; }); + histo.received = _.filter(histo.received, function(tx){ return tx && tx.block_number >= from && tx.block_number <= to; }); + _.extend(histo, { sending: [], receiving: [] }); + return res; }); - }; + }); - this.getHistoryBetweenTimes = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next) { - async.parallel({ - pubkey: ParametersService.getPubkey.bind(ParametersService, req), - from: ParametersService.getFrom.bind(ParametersService, req), - to: ParametersService.getTo.bind(ParametersService, req) - }, next); - }, - function (res, next) { - var pubkey = res.pubkey, from = res.from, to = res.to; - getHistory(pubkey, function(res) { - var histo = res.history; - histo.sent = _.filter(histo.sent, function(tx){ return tx && tx.time >= from && tx.time <= to; }); - histo.received = _.filter(histo.received, function(tx){ return tx && tx.time >= from && tx.time <= to; }); - _.extend(histo, { sending: [], receiving: [] }); - return res; - }, next); - } - ], function (err, result) { - if (err) { - res.send(500, err); - } else { - res.send(200, JSON.stringify(result, null, " ")); - } + this.getHistoryBetweenTimes = (req) => co(function *() { + let pubkey = yield ParametersService.getPubkeyP(req); + let from = yield ParametersService.getFromP(req); + let to = yield ParametersService.getToP(req); + return getHistoryP(pubkey, (res) => { + var histo = res.history; + histo.sent = _.filter(histo.sent, function(tx){ return tx && tx.time >= from && tx.time <= to; }); + histo.received = _.filter(histo.received, function(tx){ return tx && tx.time >= from && tx.time <= to; }); + _.extend(histo, { sending: [], receiving: [] }); + return res; }); - }; + }); - this.getPendingForPubkey = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next) { - async.parallel({ - pubkey: ParametersService.getPubkey.bind(ParametersService, req) - }, next); - }, - function (res, next) { - var pubkey = res.pubkey; - getHistory(pubkey, function(res) { - var histo = res.history; - _.extend(histo, { sent: [], received: [] }); - return res; - }, next); - } - ], function (err, result) { - if (err) { - res.send(500, err); - } else { - res.send(200, JSON.stringify(result, null, " ")); - } + this.getPendingForPubkey = (req) => co(function *() { + let pubkey = yield ParametersService.getPubkeyP(req); + return getHistoryP(pubkey, function(res) { + var histo = res.history; + _.extend(histo, { sent: [], received: [] }); + return res; }); - }; + }); - this.getPending = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next) { - getPending(next); - } - ], function (err, result) { - if (err) { - res.send(500, err); - } else { - res.send(200, JSON.stringify(result, null, " ")); - } + this.getPending = (req) => co(function *() { + let pending = yield server.dal.getTransactionsPending(); + let res = { + "currency": conf.currency, + "pending": pending + }; + pending.map(function(tx, index) { + pending[index] = _.omit(new Transaction(tx).json(), 'currency', 'raw'); }); - }; + return res; + }); function getHistory(pubkey, filter, done) { async.waterfall([ @@ -213,24 +116,5 @@ function TransactionBinding(server) { ], done); } - function getPending(done) { - server.dal.getTransactionsPending() - .then(function(pending){ - var res = { - "currency": conf.currency, - "pending": pending - }; - pending.map(function(tx, index) { - pending[index] = _.omit(new Transaction(tx).json(), 'currency', 'raw'); - }); - done && done(null, res); - return res; - }) - .catch(function(err){ - done && done(err); - throw err; - }); - } - return this; } diff --git a/app/controllers/uds.js b/app/controllers/uds.js index 0330dfcf80f88709ac6234b2a88841c52518faee..90629afb7a65f9b7b4bce32928a39dc54cfa9908 100644 --- a/app/controllers/uds.js +++ b/app/controllers/uds.js @@ -1,6 +1,7 @@ "use strict"; -var async = require('async'); -var _ = require('underscore'); +var co = require('co'); +var Q = require('q'); +var _ = require('underscore'); module.exports = function (server) { return new UDBinding(server); @@ -16,98 +17,47 @@ function UDBinding(server) { // Models var Source = require('../lib/entity/source'); - this.getHistory = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next) { - ParametersService.getPubkey(req, next); - }, - function (pubkey, next) { - getUDSources(pubkey, function(results) { - return results; - }, next); - } - ], function (err, result) { - if (err) { - res.send(500, err); - } else { - res.send(200, JSON.stringify(result, null, " ")); - } - }); - }; + this.getHistory = (req) => co(function *() { + let pubkey = yield ParametersService.getPubkeyP(req); + return getUDSources(pubkey, (results) => results); + }); - this.getHistoryBetweenBlocks = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next) { - async.parallel({ - pubkey: ParametersService.getPubkey.bind(ParametersService, req), - from: ParametersService.getFrom.bind(ParametersService, req), - to: ParametersService.getTo.bind(ParametersService, req) - }, next); - }, - function (params, next) { - var pubkey = params.pubkey, from = params.from, to = params.to; - getUDSources(pubkey, function(results) { - results.history.history = _.filter(results.history.history, function(ud){ return ud.block_number >= from && ud.block_number <= to; }); - return results; - }, next); - } - ], function (err, result) { - if (err) { - res.send(500, err); - } else { - res.send(200, JSON.stringify(result, null, " ")); - } + this.getHistoryBetweenBlocks = (req) => co(function *() { + let pubkey = yield ParametersService.getPubkeyP(req); + let from = yield ParametersService.getFromP(req); + let to = yield ParametersService.getToP(req); + return getUDSources(pubkey, (results) => { + results.history.history = _.filter(results.history.history, function(ud){ return ud.block_number >= from && ud.block_number <= to; }); + return results; }); - }; + }); - this.getHistoryBetweenTimes = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next) { - async.parallel({ - pubkey: ParametersService.getPubkey.bind(ParametersService, req), - from: ParametersService.getFrom.bind(ParametersService, req), - to: ParametersService.getTo.bind(ParametersService, req) - }, next); - }, - function (params, next) { - var pubkey = params.pubkey, from = params.from, to = params.to; - getUDSources(pubkey, function(results) { - results.history.history = _.filter(results.history.history, function(ud){ return ud.time >= from && ud.time <= to; }); - return results; - }, next); - } - ], function (err, result) { - if (err) { - res.send(500, err); - } else { - res.send(200, JSON.stringify(result, null, " ")); - } + this.getHistoryBetweenTimes = (req) => co(function *() { + let pubkey = yield ParametersService.getPubkeyP(req); + let from = yield ParametersService.getFromP(req); + let to = yield ParametersService.getToP(req); + return getUDSources(pubkey, (results) => { + results.history.history = _.filter(results.history.history, function(ud){ return ud.time >= from && ud.time <= to; }); + return results; }); - }; + }); - function getUDSources(pubkey, filter, done) { - async.waterfall([ - function (next) { - server.dal.getUDHistory(pubkey, next); - }, - function (history, next) { - var result = { - "currency": conf.currency, - "pubkey": pubkey, - "history": history - }; - _.keys(history).map(function(key) { - history[key].map(function (src, index) { - history[key][index] = _.omit(new Source(src).UDjson(), 'currency', 'raw'); - _.extend(history[key][index], { block_number: src && src.block_number, time: src && src.time }); - }); + function getUDSources(pubkey, filter) { + return co(function *() { + let history = yield server.dal.getUDHistory(pubkey); + var result = { + "currency": conf.currency, + "pubkey": pubkey, + "history": history + }; + _.keys(history).map(function(key) { + history[key].map(function (src, index) { + history[key][index] = _.omit(new Source(src).UDjson(), 'currency', 'raw'); + _.extend(history[key][index], { block_number: src && src.block_number, time: src && src.time }); }); - next(null, filter(result)); - } - ], done); + }); + return filter(result); + }); } return this; diff --git a/app/controllers/wot.js b/app/controllers/wot.js index 3990193ac7f40765664e940918093caa4405332b..faa16b53d915ba2331e706a7cb03fe101e6b0743 100644 --- a/app/controllers/wot.js +++ b/app/controllers/wot.js @@ -10,6 +10,8 @@ var jsoner = require('../lib/streams/jsoner'); var parsers = require('../lib/streams/parsers/doc'); var es = require('event-stream'); var http400 = require('../lib/http/http400'); +var constants = require('../lib/constants'); +var AbstractController = require('./abstract'); var logger = require('../lib/logger')(); module.exports = function (server) { @@ -18,312 +20,211 @@ module.exports = function (server) { function WOTBinding (server) { + AbstractController.call(this, server); + var ParametersService = server.ParametersService; var IdentityService = server.IdentityService; var BlockchainService = server.BlockchainService; var Identity = require('../lib/entity/identity'); - this.lookup = function (req, res) { - res.type('application/json'); - return co(function *() { - var search = yield ParametersService.getSearchP(req); - var identities = yield IdentityService.searchIdentities(search); - identities.forEach(function(idty, index){ - identities[index] = new Identity(idty); - }); - var excluding = yield BlockchainService.getCertificationsExludingBlock(); - for (let i = 0; i < identities.length; i++) { - let idty = identities[i]; - var certs = yield server.dal.certsToTarget(idty.getTargetHash()); - var validCerts = []; - for (let j = 0; j < certs.length; j++) { - let cert = certs[j]; - if (!(excluding && cert.block <= excluding.number)) { - let member = yield IdentityService.getWrittenByPubkey(cert.from); - if (member) { - cert.uids = [member.uid]; - cert.isMember = member.member; - cert.wasMember = member.wasMember; - } else { - let potentials = yield IdentityService.getPendingFromPubkey(cert.from); - cert.uids = _(potentials).pluck('uid'); - cert.isMember = false; - cert.wasMember = false; - } - validCerts.push(cert); + this.lookup = (req) => co(function *() { + var search = yield ParametersService.getSearchP(req); + var identities = yield IdentityService.searchIdentities(search); + identities.forEach(function(idty, index){ + identities[index] = new Identity(idty); + }); + var excluding = yield BlockchainService.getCertificationsExludingBlock(); + for (let i = 0; i < identities.length; i++) { + let idty = identities[i]; + var certs = yield server.dal.certsToTarget(idty.getTargetHash()); + var validCerts = []; + for (let j = 0; j < certs.length; j++) { + let cert = certs[j]; + if (!(excluding && cert.block <= excluding.number)) { + let member = yield IdentityService.getWrittenByPubkey(cert.from); + if (member) { + cert.uids = [member.uid]; + cert.isMember = member.member; + cert.wasMember = member.wasMember; + } else { + let potentials = yield IdentityService.getPendingFromPubkey(cert.from); + cert.uids = _(potentials).pluck('uid'); + cert.isMember = false; + cert.wasMember = false; } + validCerts.push(cert); } - idty.certs = validCerts; - var signed = yield server.dal.certsFrom(idty.pubkey); - var validSigned = []; - for (let j = 0; j < signed.length; j++) { - let cert = _.clone(signed[j]); - if (!(excluding && cert.block <= excluding.number)) { - cert.idty = yield server.dal.getIdentityByHashOrNull(cert.target); - validSigned.push(cert); - } + } + idty.certs = validCerts; + var signed = yield server.dal.certsFrom(idty.pubkey); + var validSigned = []; + for (let j = 0; j < signed.length; j++) { + let cert = _.clone(signed[j]); + if (!(excluding && cert.block <= excluding.number)) { + cert.idty = yield server.dal.getIdentityByHashOrNull(cert.target); + validSigned.push(cert); } - idty.signed = validSigned; } - return identities; - }) - .then(function(identities){ - var json = { - partial: false, - results: [] - }; - identities.forEach(function(identity){ - json.results.push(identity.json()); - }); - res.send(200, JSON.stringify(json, null, " ")); - }) - .catch(function(err){ - res.send(400, ((err && err.message) || err)); - }); - }; + idty.signed = validSigned; + } + var json = { + partial: false, + results: [] + }; + if (identities.length == 0) { + throw constants.ERRORS.NO_MATCHING_IDENTITY; + } + identities.forEach(function(identity){ + json.results.push(identity.json()); + }); + return json; + }); - this.members = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next){ - server.dal.getMembers(next); - } - ], function (err, identities) { - if(err){ - res.send(400, err); - return; - } - var json = { - results: [] - }; - identities.forEach(function(identity){ - json.results.push({ pubkey: identity.pubkey, uid: identity.uid }); - }); - res.send(200, JSON.stringify(json, null, " ")); + this.members = () => co(function *() { + let identities = yield server.dal.getMembersP(); + let json = { + results: [] + }; + identities.forEach(function(identity){ + json.results.push({ pubkey: identity.pubkey, uid: identity.uid }); }); - }; + return json; + }); - this.certifiersOf = function (req, res) { - res.type('application/json'); - co(function *() { - try { - let search = yield ParametersService.getSearchP(req); - let idty = yield IdentityService.findMemberWithoutMemberships(search); - let excluding = yield BlockchainService.getCertificationsExludingBlock(); - let certs = yield server.dal.certsToTarget(idty.getTargetHash()); - idty.certs = []; - for (let i = 0; i < certs.length; i++) { - let cert = certs[i]; - if (!(excluding && cert.block <= excluding.number)) { - let certifier = yield server.dal.getWrittenIdtyByPubkey(cert.from); - if (certifier) { - cert.uid = certifier.uid; - cert.isMember = certifier.member; - cert.sigDate = moment(certifier.time).unix(); - cert.wasMember = true; // As we checked if(certified) - if (!cert.cert_time) { - // TODO: would be more efficient to save medianTime on certification reception - let certBlock = yield server.dal.getBlock(cert.block_number); - cert.cert_time = { - block: certBlock.number, - medianTime: certBlock.medianTime - }; - } - idty.certs.push(cert); - } + this.certifiersOf = (req) => co(function *() { + let search = yield ParametersService.getSearchP(req); + let idty = yield IdentityService.findMemberWithoutMemberships(search); + let excluding = yield BlockchainService.getCertificationsExludingBlock(); + let certs = yield server.dal.certsToTarget(idty.getTargetHash()); + idty.certs = []; + for (let i = 0; i < certs.length; i++) { + let cert = certs[i]; + if (!(excluding && cert.block <= excluding.number)) { + let certifier = yield server.dal.getWrittenIdtyByPubkey(cert.from); + if (certifier) { + cert.uid = certifier.uid; + cert.isMember = certifier.member; + cert.sigDate = moment(certifier.time).unix(); + cert.wasMember = true; // As we checked if(certified) + if (!cert.cert_time) { + // TODO: would be more efficient to save medianTime on certification reception + let certBlock = yield server.dal.getBlock(cert.block_number); + cert.cert_time = { + block: certBlock.number, + medianTime: certBlock.medianTime + }; } + idty.certs.push(cert); } - var json = { - pubkey: idty.pubkey, - uid: idty.uid, - sigDate: moment(idty.time).unix(), - isMember: idty.member, - certifications: [] - }; - idty.certs.forEach(function(cert){ - json.certifications.push({ - pubkey: cert.from, - uid: cert.uid, - isMember: cert.isMember, - wasMember: cert.wasMember, - cert_time: cert.cert_time, - sigDate: cert.sigDate, - written: cert.linked ? { - number: cert.written_block, - hash: cert.written_hash - } : null, - signature: cert.sig - }); - }); - res.send(200, JSON.stringify(json, null, " ")); - } catch (err) { - if (err == 'No member matching this pubkey or uid') { - res.send(404, err); - return; - } - res.send(400, err); } + } + var json = { + pubkey: idty.pubkey, + uid: idty.uid, + sigDate: moment(idty.time).unix(), + isMember: idty.member, + certifications: [] + }; + idty.certs.forEach(function(cert){ + json.certifications.push({ + pubkey: cert.from, + uid: cert.uid, + isMember: cert.isMember, + wasMember: cert.wasMember, + cert_time: cert.cert_time, + sigDate: cert.sigDate, + written: cert.linked ? { + number: cert.written_block, + hash: cert.written_hash + } : null, + signature: cert.sig + }); }); - }; + return json; + }); - this.requirements = function (req, res) { - res.type('application/json'); - async.waterfall([ - function (next){ - ParametersService.getPubkey(req, next); - }, - function (search, next){ - IdentityService.searchIdentities(search).then(_.partial(next, null)).catch(next); - }, - function (identities, next){ - return identities.reduce(function(p, identity) { - return p - .then(function(all){ - return BlockchainService.requirementsOfIdentity(new Identity(identity)) - .then(function(requirements){ - return all.concat([requirements]); - }) - .catch(function(err){ - logger.warn(err); - return all; - }); - }); - }, Q([])) - .then(function(all){ - if (!all || !all.length) { - return next('No member matching this pubkey or uid'); - } - next(null, { - pubkey: all[0].pubkey, - identities: all.map(function(idty) { - return _.omit(idty, 'pubkey'); - }) - }); - }) - .catch(next); - } - ], function (err, json) { - if(err){ - if (err == 'No member matching this pubkey or uid') { - res.send(404, err); - return; - } - res.send(400, err); - return; - } - res.send(200, JSON.stringify(json, null, " ")); - }); - }; + this.requirements = (req) => co(function *() { + let search = yield ParametersService.getSearchP(req); + let identities = yield IdentityService.searchIdentities(search); + let all = yield BlockchainService.requirementsOfIdentities(identities); + if (!all || !all.length) { + throw constants.ERRORS.NO_MEMBER_MATCHING_PUB_OR_UID; + } + return { + identities: all + }; + }); - this.certifiedBy = function (req, res) { - res.type('application/json'); - co(function *() { - try { - let search = yield ParametersService.getSearchP(req); - let idty = yield IdentityService.findMemberWithoutMemberships(search); - let excluding = yield BlockchainService.getCertificationsExludingBlock(); - let certs = yield server.dal.certsFrom(idty.pubkey); - idty.certs = []; - for (let i = 0; i < certs.length; i++) { - let cert = certs[i]; - if (!(excluding && cert.block <= excluding.number)) { - let certified = yield server.dal.getWrittenIdtyByPubkey(cert.to); - if (certified) { - cert.uid = certified.uid; - cert.isMember = certified.member; - cert.sigDate = moment(certified.time).unix(); - cert.wasMember = true; // As we checked if(certified) - if (!cert.cert_time) { - // TODO: would be more efficient to save medianTime on certification reception - let certBlock = yield server.dal.getBlock(cert.block_number); - cert.cert_time = { - block: certBlock.number, - medianTime: certBlock.medianTime - }; - } - idty.certs.push(cert); - } + this.certifiedBy = (req) => co(function *() { + let search = yield ParametersService.getSearchP(req); + let idty = yield IdentityService.findMemberWithoutMemberships(search); + let excluding = yield BlockchainService.getCertificationsExludingBlock(); + let certs = yield server.dal.certsFrom(idty.pubkey); + idty.certs = []; + for (let i = 0; i < certs.length; i++) { + let cert = certs[i]; + if (!(excluding && cert.block <= excluding.number)) { + let certified = yield server.dal.getWrittenIdtyByPubkey(cert.to); + if (certified) { + cert.uid = certified.uid; + cert.isMember = certified.member; + cert.sigDate = moment(certified.time).unix(); + cert.wasMember = true; // As we checked if(certified) + if (!cert.cert_time) { + // TODO: would be more efficient to save medianTime on certification reception + let certBlock = yield server.dal.getBlock(cert.block_number); + cert.cert_time = { + block: certBlock.number, + medianTime: certBlock.medianTime + }; } + idty.certs.push(cert); } - var json = { - pubkey: idty.pubkey, - uid: idty.uid, - sigDate: moment(idty.time).unix(), - isMember: idty.member, - certifications: [] - }; - idty.certs.forEach(function(cert){ - json.certifications.push({ - pubkey: cert.to, - uid: cert.uid, - isMember: cert.isMember, - wasMember: cert.wasMember, - cert_time: cert.cert_time, - sigDate: cert.sigDate, - written: cert.linked ? { - number: cert.written_block, - hash: cert.written_hash - } : null, - signature: cert.sig - }); - }); - res.send(200, JSON.stringify(json, null, " ")); - } catch (err) { - if (err == 'No member matching this pubkey or uid') { - res.send(404, err); - return; - } - res.send(400, err); } + } + var json = { + pubkey: idty.pubkey, + uid: idty.uid, + sigDate: moment(idty.time).unix(), + isMember: idty.member, + certifications: [] + }; + idty.certs.forEach(function(cert){ + json.certifications.push({ + pubkey: cert.to, + uid: cert.uid, + isMember: cert.isMember, + wasMember: cert.wasMember, + cert_time: cert.cert_time, + sigDate: cert.sigDate, + written: cert.linked ? { + number: cert.written_block, + hash: cert.written_hash + } : null, + signature: cert.sig + }); }); - }; + return json; + }); - this.identityOf = function (req, res) { - res.type('application/json'); - return co(function *() { - try { - let search = yield ParametersService.getSearchP(req); - let idty = yield IdentityService.findMemberWithoutMemberships(search); - if (!idty) { - throw 'Identity not found'; - } - if (!idty.member) { - throw 'Not a member'; - } - var json = { - pubkey: idty.pubkey, - uid: idty.uid, - sigDate: moment(idty.time).unix() - }; - res.send(200, JSON.stringify(json, null, " ")); - } catch(e) { - res.send(400, e); - } - }); - }; + this.identityOf = (req) => co(function *() { + let search = yield ParametersService.getSearchP(req); + let idty = yield IdentityService.findMemberWithoutMemberships(search); + if (!idty) { + throw 'Identity not found'; + } + if (!idty.member) { + throw 'Not a member'; + } + return { + pubkey: idty.pubkey, + uid: idty.uid, + sigDate: moment(idty.time).unix() + }; + }); - this.add = function (req, res) { - res.type('application/json'); - var onError = http400(res); - http2raw.identity(req, onError) - .pipe(dos2unix()) - .pipe(parsers.parseIdentity(onError)) - .pipe(server.singleWriteStream(onError)) - .pipe(jsoner()) - .pipe(es.stringify()) - .pipe(res); - }; + this.add = (req) => this.pushEntity(req, http2raw.identity, parsers.parseIdentity); - this.revoke = function (req, res) { - res.type('application/json'); - var onError = http400(res); - http2raw.revocation(req, onError) - .pipe(dos2unix()) - .pipe(parsers.parseRevocation(onError)) - .pipe(server.singleWriteStream(onError)) - .pipe(jsoner()) - .pipe(es.stringify()) - .pipe(res); - }; + this.revoke = (req) => this.pushEntity(req, http2raw.revocation, parsers.parseRevocation); } diff --git a/app/lib/blockchainContext.js b/app/lib/blockchainContext.js index a507c78360f678a232658018630cabe5dee19ba0..712ef6d686743a141f89fa01311bcba959562aab 100644 --- a/app/lib/blockchainContext.js +++ b/app/lib/blockchainContext.js @@ -4,6 +4,7 @@ var _ = require('underscore'); var co = require('co'); var Q = require('q'); var sha1 = require('sha1'); +var moment = require('moment'); var rawer = require('./rawer'); var localValidator = require('./localValidator'); var globalValidator = require('./globalValidator'); @@ -176,13 +177,9 @@ function BlockchainContext(conf, dal) { // Create/Update certifications updateCertifications(block, next); }, - function (next) { - // Create/Update certifications - updateMemberships(block, next); - }, function (next){ // Save links - updateLinks(block, next, dal.getBlockOrNull.bind(dal)); + updateLinksForBlocks([block], dal.getBlockOrNull.bind(dal)).then(() => next()).catch(next); }, function (next){ // Compute obsolete links @@ -244,52 +241,57 @@ function BlockchainContext(conf, dal) { this.updateMembers = updateMembers; this.updateCertifications = updateCertifications; - this.updateMemberships = updateMemberships; - this.updateLinks = updateLinks; - this.updateTransactionSources = updateTransactionSources; this.computeObsoleteLinks = computeObsoleteLinks; this.computeObsoleteMemberships = computeObsoleteMemberships; + this.updateTransactionSourcesForBlocks = updateTransactionSourcesForBlocks; + this.updateCertificationsForBlocks = updateCertificationsForBlocks; + this.updateMembershipsForBlocks = updateMembershipsForBlocks; + this.updateLinksForBlocks = updateLinksForBlocks; + this.updateTransactionsForBlocks = updateTransactionsForBlocks; + + let cleanRejectedIdentities = (idty) => co(function *() { + yield dal.removeUnWrittenWithPubkey(idty.pubkey); + yield dal.removeUnWrittenWithUID(idty.uid); + }); function updateMembers (block, done) { - async.waterfall([ - function (next) { - // Newcomers - async.forEachSeries(block.identities, function(identity, callback){ - var idty = Identity.statics.fromInline(identity); - // Computes the hash if not done yet - if (!idty.hash) - idty.hash = (sha1(rawer.getIdentity(idty)) + "").toUpperCase(); - dal.newIdentity(idty, block.number).then(_.partial(callback, null)).catch(callback); - }, next); - }, - function (next) { - // Joiners (come back) - async.forEachSeries(block.joiners, function(inlineMS, callback){ - var ms = Identity.statics.fromInline(inlineMS); - dal.joinIdentity(ms.pubkey, block.number).then(_.partial(callback, null)).catch(callback); - }, next); - }, - function (next) { - // Actives - async.forEachSeries(block.actives, function(inlineMS, callback){ - var ms = Identity.statics.fromInline(inlineMS); - dal.activeIdentity(ms.pubkey, block.number).then(_.partial(callback, null)).catch(callback); - }, next); - }, - function (next) { - // Leavers - async.forEachSeries(block.leavers, function(inlineMS, callback){ - var ms = Identity.statics.fromInline(inlineMS); - dal.leaveIdentity(ms.pubkey, block.number).then(_.partial(callback, null)).catch(callback); - }, next); - }, - function (next) { - // Excluded - async.forEachSeries(block.excluded, function (pubkey, callback) { - dal.excludeIdentity(pubkey).then(_.partial(callback, null)).catch(callback); - }, next); + return co(function *() { + // Newcomers + for (let i = 0, len = block.identities.length; i < len; i++) { + let identity = block.identities[i]; + let idty = Identity.statics.fromInline(identity); + // Computes the hash if not done yet + if (!idty.hash) + idty.hash = (sha1(rawer.getIdentity(idty)) + "").toUpperCase(); + yield dal.newIdentity(idty, block.number); + yield cleanRejectedIdentities(idty); } - ], done); + // Joiners (come back) + for (let i = 0, len = block.joiners.length; i < len; i++) { + let inlineMS = block.joiners[i]; + let ms = Identity.statics.fromInline(inlineMS); + yield dal.joinIdentity(ms.pubkey, block.number); + } + // Actives + for (let i = 0, len = block.actives.length; i < len; i++) { + let inlineMS = block.actives[i]; + let ms = Identity.statics.fromInline(inlineMS); + yield dal.activeIdentity(ms.pubkey, block.number); + } + // Actives + for (let i = 0, len = block.leavers.length; i < len; i++) { + let inlineMS = block.leavers[i]; + let ms = Identity.statics.fromInline(inlineMS); + yield dal.leaveIdentity(ms.pubkey, block.number); + } + // Excluded + for (let i = 0, len = block.excluded.length; i < len; i++) { + let excluded = block.excluded[i]; + dal.excludeIdentity(excluded); + } + done(); + }) + .catch(done); } function undoMembersUpdate (block) { @@ -393,6 +395,15 @@ function BlockchainContext(conf, dal) { }); } + /** + * Historical method that takes certifications from a block and tries to either: + * * Update the certification found in the DB an set it as written + * * Create it if it does not exist + * + * Has a sibling method named 'updateCertificationsForBlocks'. + * @param block + * @param done + */ function updateCertifications (block, done) { async.forEachSeries(block.certifications, function(inlineCert, callback){ var cert = Certification.statics.fromInline(inlineCert); @@ -427,54 +438,6 @@ function BlockchainContext(conf, dal) { }, done); } - // TODO: no more needed - function updateMemberships (block, done) { - async.forEachSeries(['joiners', 'actives', 'leavers'], function (prop, callback1) { - async.forEach(block[prop], function(inlineJoin, callback){ - var ms = Membership.statics.fromInline(inlineJoin, prop == 'leavers' ? 'OUT' : 'IN'); - async.waterfall([ - function (next){ - dal.getWritten(ms.issuer, next); - }, - function (idty, next){ - if (!idty) { - var err = 'Could not find identity for membership of issuer ' + ms.issuer; - logger.error(err); - next(err); - return; - } - ms.userid = idty.uid; - ms.certts = idty.time; - next(); - } - ], callback); - }, callback1); - }, done); - } - - function updateLinks (block, done, getBlockOrNull) { - async.forEach(block.certifications, function(inlineCert, callback){ - var cert = Certification.statics.fromInline(inlineCert); - return co(function *() { - let tagBlock = block; - if (block.number > 0) { - tagBlock = yield getBlockOrNull(cert.block_number); - } - return dal.saveLink( - new Link({ - source: cert.from, - target: cert.to, - timestamp: tagBlock.medianTime, - block_number: block.number, - block_hash: block.hash, - obsolete: false - })); - }) - .then(_.partial(callback, null)) - .catch(callback); - }, done); - } - that.saveParametersForRootBlock = (block, done) => { if (block.parameters) { var sp = block.parameters.split(':'); @@ -518,16 +481,11 @@ function BlockchainContext(conf, dal) { var pubkey = idty.pubkey; async.waterfall([ function (nextOne){ - async.parallel({ - enoughLinks: function(callback2){ - that.checkHaveEnoughLinks(pubkey, {}, function (err) { - callback2(null, err); - }); - } - }, nextOne); + that.checkHaveEnoughLinks(pubkey, {}, function (err) { + nextOne(null, err); + }); }, - function (res, nextOne){ - var notEnoughLinks = res.enoughLinks; + function (notEnoughLinks, nextOne){ dal.setKicked(pubkey, new Identity(idty).getTargetHash(), notEnoughLinks ? true : false, nextOne); } ], callback); @@ -548,7 +506,7 @@ function BlockchainContext(conf, dal) { next(count < conf.sigQty && 'Key ' + target + ' does not have enough links (' + count + '/' + conf.sigQty + ')'); } ], done); - } + }; function computeObsoleteMemberships (block) { return dal.getMembershipExcludingBlock(block, conf.msValidity) @@ -621,6 +579,193 @@ function BlockchainContext(conf, dal) { }); } + /** + * New method for CREATING memberships found in blocks. + * Made for performance reasons, this method will batch insert all memberships at once. + * @param blocks + * @returns {*} + */ + function updateMembershipsForBlocks(blocks) { + return co(function *() { + let memberships = []; + let types = { + 'join': 'joiners', + 'active': 'actives', + 'leave': 'leavers' + }; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + _.keys(types).forEach(function(type){ + let msType = type == 'leave' ? 'out' : 'in'; + let field = types[type]; + let mss = block[field]; + for (let j = 0, len2 = mss.length; j < len2; j++) { + let msRaw = mss[j]; + var ms = Membership.statics.fromInline(msRaw, type == 'leave' ? 'OUT' : 'IN', block.currency); + ms.membership = msType.toUpperCase(); + ms.written = true; + ms.type = type; + ms.hash = String(sha1(ms.getRawSigned())).toUpperCase(); + ms.idtyHash = (sha1(ms.userid + moment(ms.certts).unix() + ms.issuer) + "").toUpperCase(); + memberships.push(ms); + } + }); + } + return dal.updateMemberships(memberships); + }); + } + + /** + * New method for CREATING links found in blocks. + * Made for performance reasons, this method will batch insert all links at once. + * @param blocks + * @param getBlockOrNull + * @returns {*} + */ + function updateLinksForBlocks(blocks, getBlockOrNull) { + return co(function *() { + let links = []; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + for (let j = 0, len2 = block.certifications.length; j < len2; j++) { + let inlineCert = block.certifications[j]; + let cert = Certification.statics.fromInline(inlineCert); + let tagBlock = block; + if (block.number > 0) { + tagBlock = yield getBlockOrNull(cert.block_number); + } + links.push({ + source: cert.from, + target: cert.to, + timestamp: tagBlock.medianTime, + block_number: block.number, + block_hash: block.hash, + obsolete: false + }); + } + } + return dal.updateLinks(links); + }); + } + + /** + * New method for CREATING transactions found in blocks. + * Made for performance reasons, this method will batch insert all transactions at once. + * @param blocks + * @returns {*} + */ + function updateTransactionsForBlocks(blocks) { + return co(function *() { + let txs = []; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + txs = txs.concat(block.transactions.map((tx) => { + _.extend(tx, { + block_number: block.number, + time: block.medianTime, + currency: block.currency, + written: true, + removed: false + }); + return new Transaction(tx); + })); + } + return dal.updateTransactions(txs); + }); + } + + /** + * New method for CREATING certifications found in blocks. + * Made for performance reasons, this method will batch insert all certifications at once. + * @param blocks + * @returns {*} + */ + function updateCertificationsForBlocks(blocks) { + return co(function *() { + let certs = []; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + for (let j = 0, len2 = block.certifications.length; j < len2; j++) { + let inlineCert = block.certifications[j]; + var cert = Certification.statics.fromInline(inlineCert); + let to = yield dal.getWrittenIdtyByPubkey(cert.to); + let to_uid = to.uid; + cert.target = new Identity(to).getTargetHash(); + let from = yield dal.getWrittenIdtyByPubkey(cert.from); + let from_uid = from.uid; + let existing = yield dal.existsCert(cert); + if (existing) { + cert = existing; + } + cert.written_block = block.number; + cert.written_hash = block.hash; + cert.from_uid = from_uid; + cert.to_uid = to_uid; + cert.linked = true; + certs.push(cert); + } + } + return dal.updateCertifications(certs); + }); + } + + /** + * New method for CREATING sources found in transactions of blocks. + * Made for performance reasons, this method will batch insert all sources at once. + * @param blocks + * @returns {*} + */ + function updateTransactionSourcesForBlocks(blocks) { + return co(function *() { + let sources = []; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + // Dividends + if (block.dividend) { + let idties = yield dal.getMembersP(); + for (let j = 0, len2 = idties.length; j < len2; j++) { + let idty = idties[j]; + sources.push({ + 'pubkey': idty.pubkey, + 'type': 'D', + 'number': block.number, + 'time': block.medianTime, + 'fingerprint': block.hash, + 'block_hash': block.hash, + 'amount': block.dividend, + 'consumed': false, + 'toConsume': false + }); + } + } + // Transactions + for (let j = 0, len2 = block.transactions.length; j < len2; j++) { + let json = block.transactions[j]; + let obj = json; + obj.version = 1; + obj.currency = block.currency; + obj.issuers = json.signatories; + let tx = new Transaction(obj); + let txObj = tx.getTransaction(); + let txHash = tx.getHash(true); + sources = sources.concat(txObj.inputs.map((input) => _.extend({ toConsume: true }, input))); + sources = sources.concat(txObj.outputs.map((output) => _.extend({ + toConsume: false + }, { + 'pubkey': output.pubkey, + 'type': 'T', + 'number': block.number, + 'block_hash': block.hash, + 'fingerprint': txHash, + 'amount': output.amount, + 'consumed': false + }))); + } + } + return dal.updateSources(sources); + }); + } + function deleteTransactions (block, done) { async.forEachSeries(block.transactions, function (json, callback) { var obj = json; diff --git a/app/lib/constants.js b/app/lib/constants.js index e4090714a2f2da4f981e16b593867f2c1219477a..07408574b7c0134dc4bc10251db67ca442b6bb94 100644 --- a/app/lib/constants.js +++ b/app/lib/constants.js @@ -30,6 +30,37 @@ module.exports = { } }, + ERRORS: { + + // Technical errors + UNKNOWN: { httpCode: 500, uerr: { ucode: 1001, message: "An unknown error occured" }}, + UNHANDLED: { httpCode: 500, uerr: { ucode: 1002, message: "An unhandled error occured" }}, + SIGNATURE_DOES_NOT_MATCH: { httpCode: 400, uerr: { ucode: 1003, message: "Signature does not match" }}, + ALREADY_UP_TO_DATE: { httpCode: 400, uerr: { ucode: 1004, message: "Already up-to-date" }}, + WRONG_DOCUMENT: { httpCode: 400, uerr: { ucode: 1005, message: "Document has unkown fields or wrong line ending format" }}, + + HTTP_PARAM_PUBKEY_REQUIRED: { httpCode: 400, uerr: { ucode: 1101, message: "Parameter `pubkey` is required" }}, + HTTP_PARAM_SELF_REQUIRED: { httpCode: 400, uerr: { ucode: 1102, message: "Parameter `self` is required" }}, + HTTP_PARAM_PEER_REQUIRED: { httpCode: 400, uerr: { ucode: 1103, message: "Requires a peer" }}, + HTTP_PARAM_BLOCK_REQUIRED: { httpCode: 400, uerr: { ucode: 1104, message: "Requires a block" }}, + HTTP_PARAM_MEMBERSHIP_REQUIRED: { httpCode: 400, uerr: { ucode: 1105, message: "Requires a membership" }}, + HTTP_PARAM_TX_REQUIRED: { httpCode: 400, uerr: { ucode: 1106, message: "Requires a transaction" }}, + HTTP_PARAM_SIG_REQUIRED: { httpCode: 400, uerr: { ucode: 1107, message: "Parameter `sig` is required" }}, + + // Business errors + NO_MATCHING_IDENTITY: { httpCode: 404, uerr: { ucode: 2001, message: "No matching identity" }}, + UID_ALREADY_USED: { httpCode: 400, uerr: { ucode: 2002, message: "UID already used in the blockchain" }}, + PUBKEY_ALREADY_USED: { httpCode: 400, uerr: { ucode: 2003, message: "Pubkey already used in the blockchain" }}, + NO_MEMBER_MATCHING_PUB_OR_UID: { httpCode: 404, uerr: { ucode: 2004, message: "No member matching this pubkey or uid" }}, + SELF_PEER_NOT_FOUND: { httpCode: 404, uerr: { ucode: 2005, message: "Self peering was not found" }}, + WRONG_SIGNATURE_MEMBERSHIP: { httpCode: 400, uerr: { ucode: 2006, message: "wrong signature for membership" }}, + ALREADY_RECEIVED_MEMBERSHIP: { httpCode: 400, uerr: { ucode: 2007, message: "Already received membership" }}, + MEMBERSHIP_A_NON_MEMBER_CANNOT_LEAVE: { httpCode: 400, uerr: { ucode: 2008, message: "A non-member cannot leave" }}, + NOT_A_MEMBER: { httpCode: 400, uerr: { ucode: 2009, message: "Not a member" }}, + NO_CURRENT_BLOCK: { httpCode: 404, uerr: { ucode: 2010, message: "No current block" }}, + BLOCK_NOT_FOUND: { httpCode: 404, uerr: { ucode: 2011, message: "Block not found" }} + }, + DEBUG: { LONG_DAL_PROCESS: 50 }, @@ -100,6 +131,9 @@ module.exports = { SPECIAL_BLOCK: '0-DA39A3EE5E6B4B0D3255BFEF95601890AFD80709' }, NETWORK: { + MAX_NON_MEMBERS_TO_FORWARD_TO: 4, + MAX_MEMBERS_TO_FORWARD_TO: 2, + COUNT_FOR_ENOUGH_PEERS: 4, MAX_CONCURRENT_POST: 3, DEFAULT_TIMEOUT: 5000, SYNC_LONG_TIMEOUT: 30 * 1000, // 30 seconds @@ -118,6 +152,7 @@ module.exports = { UPDATE: 6, // Every X blocks MAX: 20 // MAX Y blocks }, + SYNC_PEERS_INTERVAL: 3, // Every 3 block average generation time SYNC_BLOCK_INTERVAL: 1, // Every 1 block average generation time TEST_PEERS_INTERVAL: 10 // In seconds }, @@ -166,13 +201,7 @@ module.exports = { MEMORY_CLEAN_INTERVAL: 60 * 60, // hourly SAFE_FACTOR: 3, - BLOCKS_COLLECT_THRESHOLD: 30, // Blocks to collect from memory and persist - - setUDID2Format: function () { - module.exports.USER_ID = module.exports.UDID2_FORMAT; - module.exports.CERT.SELF.UID = exact("UID:" + UDID2); - module.exports.IDENTITY.INLINE = exact(PUBKEY + ":" + SIGNATURE + ":" + TIMESTAMP + ":" + UDID2); - } + BLOCKS_COLLECT_THRESHOLD: 30 // Blocks to collect from memory and persist }; function exact (regexpContent) { diff --git a/app/lib/crypto.js b/app/lib/crypto.js index 4b57b2c2da9baaf4beb7af208d20714d0819e4b8..6bbfd06168c69afaf2a66683e89c264506fe3f43 100644 --- a/app/lib/crypto.js +++ b/app/lib/crypto.js @@ -117,10 +117,7 @@ module.exports = { function getScryptKey(key, salt, callback) { // console.log('Derivating the key...'); - scrypt.kdf.config.saltEncoding = "ascii"; - scrypt.kdf.config.keyEncoding = "ascii"; - scrypt.kdf.config.outputEncoding = "base64"; - scrypt.kdf(key, TEST_PARAMS, SEED_LENGTH, salt, function (err, res) { - callback(dec(res.hash)); + scrypt.hash(key, TEST_PARAMS, SEED_LENGTH, salt, function (err, res) { + callback(dec(res.toString("base64"))); }); } diff --git a/app/lib/dal/fileDAL.js b/app/lib/dal/fileDAL.js index 711ad8821cfe53c1f62b8714071a568de3cbb29e..9df323b52a256d272009e97715282094821c2a92 100644 --- a/app/lib/dal/fileDAL.js +++ b/app/lib/dal/fileDAL.js @@ -6,6 +6,7 @@ var fs = require('fs'); var qfs = require('q-io/fs'); var sha1 = require('sha1'); var path = require('path'); +var moment = require('moment'); var Configuration = require('../entity/configuration'); var Membership = require('../entity/membership'); var Merkle = require('../entity/merkle'); @@ -13,84 +14,27 @@ var Transaction = require('../entity/transaction'); var constants = require('../constants'); var ConfDAL = require('./fileDALs/confDAL'); var StatDAL = require('./fileDALs/statDAL'); -var CertDAL = require('./fileDALs/CertDAL'); -var TxsDAL = require('./fileDALs/TxsDAL'); -var SourcesDAL = require('./fileDALs/SourcesDAL'); -var LinksDAL = require('./fileDALs/LinksDAL'); -var MembershipDAL = require('./fileDALs/MembershipDAL'); -var IdentityDAL = require('./fileDALs/IdentityDAL'); var IndicatorsDAL = require('./fileDALs/IndicatorsDAL'); -var PeerDAL = require('./fileDALs/PeerDAL'); -var BlockDAL = require('./fileDALs/BlockDAL'); var CFSStorage = require('./fileDALs/AbstractCFS'); -var lokijs = require('lokijs'); +var sqlite3 = require("sqlite3").verbose(); var logger = require('../../lib/logger')('database'); const UCOIN_DB_NAME = 'ucoin'; module.exports = { - memory: function(profile) { - return getHomeFS(profile, true) + memory: function(home) { + return getHomeFS(true, home) .then(function(params) { - let loki = new lokijs(UCOIN_DB_NAME, { autosave: false }); - return Q(new FileDAL(profile, params.home, "", params.fs, null, 'fileDal', loki)); + let sqlite = new sqlite3.Database(':memory:'); + return Q(new FileDAL(params.home, "", params.fs, 'fileDal', sqlite)); }); }, - file: function(profile, forConf) { - return getHomeFS(profile, false) + file: function(home) { + return getHomeFS(false, home) .then(function(params) { - return Q.Promise(function(resolve, reject){ - let loki; - if (forConf) { - // Memory only service dals - loki = new lokijs('temp', { autosave: false }); - resolve(loki); - } else { - let lokiPath = path.join(params.home, UCOIN_DB_NAME + '.json'); - logger.debug('Loading DB at %s...', lokiPath); - co(function *() { - let rawDB; - try { - // Try to read database - rawDB = yield qfs.read(lokiPath); - // Check if content is standard JSON - JSON.parse(rawDB); - } catch (e) { - if (rawDB) { - logger.error('The database could not be loaded, it is probably corrupted due to some system fail.'); - logger.error('Please stop uCoin and re-sync it with another node of the network before restarting, using the following commands:'); - logger.error('> ucoind reset data'); - logger.error('> ucoind sync <some.ucoin.node> <port>'); - logger.error('> ucoind restart'); - // Dirty "won't go any further" - return Q.Promise((resolve) => null); - } - } - - return Q.Promise(function(resolve2){ - let lokiDB; - lokiDB = new lokijs(lokiPath, { - autoload: true, - autosave: true, - autosaveInterval: 30000, - adapter: { - loadDatabase: (dbname, callback) => { - callback(rawDB || null); - resolve2(lokiDB); - }, - saveDatabase: (dbname, dbstring, callback) => fs.writeFile(dbname, dbstring, callback) - } - }); - }); - }) - .then(resolve) - .catch(reject); - } - }) - .then(function(loki){ - loki.autosaveClearFlags(); - return new FileDAL(profile, params.home, "", params.fs, null, 'fileDal', loki); - }); + let sqlitePath = path.join(params.home, UCOIN_DB_NAME + '.db'); + let sqlite = new sqlite3.Database(sqlitePath); + return new FileDAL(params.home, "", params.fs, 'fileDal', sqlite); }); }, FileDAL: FileDAL @@ -102,52 +46,68 @@ function someDelayFix() { }); } -function getHomeFS(profile, isMemory) { - let userHome = process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE; - let home = userHome + '/.config/ucoin/' + profile; - let fs; +function getHomeFS(isMemory, home) { + let myfs; return someDelayFix() .then(function() { - fs = (isMemory ? require('q-io/fs-mock')({}) : qfs); - return fs.makeTree(home); + myfs = (isMemory ? require('q-io/fs-mock')({}) : qfs); + return myfs.makeTree(home); }) .then(function(){ - return { fs: fs, home: home }; + return { fs: myfs, home: home }; }); } -function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { +function FileDAL(home, localDir, myFS, dalName, sqlite) { var that = this; let localHome = path.join(home, localDir); this.name = dalName; - this.profile = profile; - this.parentDAL = parentFileDAL; - + this.profile = 'DAL'; var rootPath = home; - let blocksCFS = require('../cfs')(rootPath, myFS); - // DALs - this.confDAL = new ConfDAL(rootPath, myFS, parentFileDAL && parentFileDAL.confDAL.coreFS, that, CFSStorage); - this.peerDAL = new PeerDAL(loki); - this.blockDAL = new BlockDAL(loki, blocksCFS, getLowerWindowBlock); - this.sourcesDAL = new SourcesDAL(loki); - this.txsDAL = new TxsDAL(loki); - this.indicatorsDAL = new IndicatorsDAL(rootPath, myFS, parentFileDAL && parentFileDAL.indicatorsDAL.coreFS, that, CFSStorage); - this.statDAL = new StatDAL(rootPath, myFS, parentFileDAL && parentFileDAL.statDAL.coreFS, that, CFSStorage); - this.linksDAL = new LinksDAL(loki); - this.idtyDAL = new IdentityDAL(loki); - this.certDAL = new CertDAL(loki); - this.msDAL = new MembershipDAL(loki); + this.confDAL = new ConfDAL(rootPath, myFS, null, that, CFSStorage); + this.peerDAL = new (require('./sqliteDAL/PeerDAL'))(sqlite); + this.blockDAL = new (require('./sqliteDAL/BlockDAL'))(sqlite); + this.sourcesDAL = new (require('./sqliteDAL/SourcesDAL'))(sqlite); + this.txsDAL = new (require('./sqliteDAL/TxsDAL'))(sqlite); + this.indicatorsDAL = new IndicatorsDAL(rootPath, myFS, null, that, CFSStorage); + this.statDAL = new StatDAL(rootPath, myFS, null, that, CFSStorage); + this.linksDAL = new (require('./sqliteDAL/LinksDAL'))(sqlite); + this.idtyDAL = new (require('./sqliteDAL/IdentityDAL'))(sqlite); + this.certDAL = new (require('./sqliteDAL/CertDAL'))(sqlite); + this.msDAL = new (require('./sqliteDAL/MembershipDAL'))(sqlite); this.newDals = { + 'blockDAL': that.blockDAL, + 'certDAL': that.certDAL, + 'msDAL': that.msDAL, + 'idtyDAL': that.idtyDAL, + 'sourcesDAL': that.sourcesDAL, + 'linksDAL': that.linksDAL, + 'txsDAL': that.txsDAL, 'peerDAL': that.peerDAL, 'indicatorsDAL': that.indicatorsDAL, 'confDAL': that.confDAL, - 'statDAL': that.statDAL + 'statDAL': that.statDAL, + 'ghostDAL': { + init: () => co(function *() { + + // Create extra views (useful for stats or debug) + return that.blockDAL.exec('BEGIN;' + + 'CREATE VIEW IF NOT EXISTS identities_pending AS SELECT * FROM idty WHERE NOT written;' + + 'CREATE VIEW IF NOT EXISTS certifications_pending AS SELECT * FROM cert WHERE NOT written;' + + 'CREATE VIEW IF NOT EXISTS transactions_pending AS SELECT * FROM txs WHERE NOT written;' + + 'CREATE VIEW IF NOT EXISTS transactions_desc AS SELECT * FROM txs ORDER BY time DESC;' + + 'CREATE VIEW IF NOT EXISTS forks AS SELECT number, hash, issuer, monetaryMass, dividend, UDTime, membersCount, medianTime, time, * FROM block WHERE fork ORDER BY number DESC;' + + 'CREATE VIEW IF NOT EXISTS blockchain AS SELECT number, hash, issuer, monetaryMass, dividend, UDTime, membersCount, medianTime, time, * FROM block WHERE NOT fork ORDER BY number DESC;' + + 'CREATE VIEW IF NOT EXISTS network AS select i.uid, (last_try - first_down) / 1000 as down_delay_in_sec, p.* from peer p LEFT JOIN idty i on i.pubkey = p.pubkey ORDER by down_delay_in_sec;' + + 'COMMIT;'); + }) + } }; var currency = ''; @@ -156,48 +116,12 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { return co(function *() { yield _.values(that.newDals).map((dal) => dal.init()); return that.loadConf(overrideConf, defaultConf); - }); + }) + .catch((err) => { + throw Error(err); + }); }; - function getLowerWindowBlock() { - return co(function *() { - let rootBlock = yield that.getRootBlock(); - if (!rootBlock) { - return -1; - } - let conf = getParameters(rootBlock); - let needToBeKeptBlocks = getMaxBlocksToStoreAsFile(conf); - let current = yield that.getCurrentBlockOrNull(); - let currentNumber = current ? current.number : -1; - return currentNumber - needToBeKeptBlocks; - }); - } - - function getParameters(block) { - var sp = block.parameters.split(':'); - let theConf = {}; - theConf.c = parseFloat(sp[0]); - theConf.dt = parseInt(sp[1]); - theConf.ud0 = parseInt(sp[2]); - theConf.sigDelay = parseInt(sp[3]); - theConf.sigValidity = parseInt(sp[4]); - theConf.sigQty = parseInt(sp[5]); - theConf.sigWoT = parseInt(sp[6]); - theConf.msValidity = parseInt(sp[7]); - theConf.stepMax = parseInt(sp[8]); - theConf.medianTimeBlocks = parseInt(sp[9]); - theConf.avgGenTime = parseInt(sp[10]); - theConf.dtDiffEval = parseInt(sp[11]); - theConf.blocksRot = parseInt(sp[12]); - theConf.percentRot = parseFloat(sp[13]); - theConf.currency = block.currency; - return theConf; - } - - function getMaxBlocksToStoreAsFile(aConf) { - return Math.floor(Math.max(aConf.dt / aConf.avgGenTime, aConf.medianTimeBlocks, aConf.dtDiffEval, aConf.blocksRot) * constants.SAFE_FACTOR); - } - this.getCurrency = function() { return currency; }; @@ -441,6 +365,13 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { }); }; + this.getMembersP = () => co(function *() { + let idties = yield that.idtyDAL.getWhoIsOrWasMember(); + return _.chain(idties). + where({ member: true }). + value(); + }); + // TODO: this should definitely be reduced by removing fillInMembershipsOfIdentity this.getWritten = function(pubkey, done) { return that.fillInMembershipsOfIdentity( @@ -621,9 +552,10 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { }); }; - this.listPendingLocalMemberships = function() { - return that.msDAL.getPendingLocal(); - }; + this.lastJoinOfIdentity = (target) => co(function *() { + let pending = yield that.msDAL.getPendingINOfTarget(target); + return _(pending).sortBy((ms) => -ms.number)[0]; + }); this.findNewcomers = function() { return that.msDAL.getPendingIN() @@ -892,6 +824,15 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { }); }; + this.listAllPeersWithStatusNewUPWithtout = (pubkey) => co(function *() { + let peers = yield that.peerDAL.listAll(); + let matching = _.chain(peers). + filter((p) => p.status == 'UP'). + filter((p) => p.pubkey != pubkey). + value(); + return Q(matching); + }); + this.findPeers = function(pubkey) { return that.getPeer(pubkey) .catch(function(){ @@ -919,6 +860,15 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { }); }; + this.setPeerUP = (pubkey) => co(function *() { + let p = yield that.getPeer(pubkey); + p.status = 'UP'; + p.first_down = null; + p.last_try = null; + return that.peerDAL.savePeer(p); + }) + .catch(() => null); + this.setPeerDown = (pubkey) => co(function *() { let p = yield that.getPeer(pubkey); let now = (new Date()).getTime(); @@ -959,6 +909,7 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { var ms = Membership.statics.fromInline(msRaw, type == 'leave' ? 'OUT' : 'IN', that.getCurrency()); ms.type = type; ms.hash = String(sha1(ms.getRawSigned())).toUpperCase(); + ms.idtyHash = (sha1(ms.userid + moment(ms.certts).unix() + ms.issuer) + "").toUpperCase(); return that.msDAL.saveOfficialMS(msType, ms); }); }, Q()); @@ -1001,20 +952,13 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { this.donable = donable; - this.merkleForPeers = function(done) { - return that.listAllPeersWithStatusNewUP() - .then(function(peers){ - var leaves = peers.map(function(peer) { return peer.hash; }); - var merkle = new Merkle(); - merkle.initialize(leaves); - done && done(null, merkle); - return merkle; - }) - .catch(function(err){ - done && done(err); - throw err; - }); - }; + this.merkleForPeers = () => co(function *() { + let peers = yield that.listAllPeersWithStatusNewUP(); + var leaves = peers.map(function(peer) { return peer.hash; }); + var merkle = new Merkle(); + merkle.initialize(leaves); + return merkle; + }); this.saveLink = function(link) { return that.linksDAL.addLink(link); @@ -1033,6 +977,26 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { return that.sourcesDAL.addSource('available', src.pubkey, src.type, src.number, src.fingerprint, src.amount, src.block_hash, src.time); }; + this.updateSources = function(sources) { + return that.sourcesDAL.updateBatchOfSources(sources); + }; + + this.updateCertifications = function(certs) { + return that.certDAL.updateBatchOfCertifications(certs); + }; + + this.updateMemberships = function(certs) { + return that.msDAL.updateBatchOfMemberships(certs); + }; + + this.updateLinks = function(certs) { + return that.linksDAL.updateBatchOfLinks(certs); + }; + + this.updateTransactions = function(txs) { + return that.txsDAL.updateBatchOfTxs(txs); + }; + this.officializeCertification = function(cert) { return that.certDAL.saveOfficial(cert); }; @@ -1068,21 +1032,29 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { return that.idtyDAL.leaveIdentity(pubkey, onBlock); }; + this.removeUnWrittenWithPubkey = function(pubkey) { + return Q(that.idtyDAL.removeUnWrittenWithPubkey(pubkey)); + }; + + this.removeUnWrittenWithUID = function(pubkey) { + return Q(that.idtyDAL.removeUnWrittenWithUID(pubkey)); + }; + this.unacceptIdentity = that.idtyDAL.unacceptIdentity; this.unJoinIdentity = (ms) => co(function *() { yield that.idtyDAL.unJoinIdentity(ms); - that.msDAL.unwriteMS(ms); + yield that.msDAL.unwriteMS(ms); }); this.unRenewIdentity = (ms) => co(function *() { yield that.idtyDAL.unRenewIdentity(ms); - that.msDAL.unwriteMS(ms); + yield that.msDAL.unwriteMS(ms); }); this.unLeaveIdentity = (ms) => co(function *() { yield that.idtyDAL.unLeaveIdentity(ms); - that.msDAL.unwriteMS(ms); + yield that.msDAL.unwriteMS(ms); }); this.unExcludeIdentity = that.idtyDAL.unExcludeIdentity; @@ -1120,24 +1092,14 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { }); }; - this.getUDHistory = function(pubkey, done) { - return that.sourcesDAL.getUDSources(pubkey) - .then(function(sources){ - return { - history: sources.map((src) => _.extend({ - block_number: src.number - }, src)) - }; - }) - .then(function(obj){ - done && done(null, obj); - return obj; - }) - .catch(function(err){ - done && done(err); - throw err; - }); - }; + this.getUDHistory = (pubkey) => co(function *() { + let sources = yield that.sourcesDAL.getUDSources(pubkey); + return { + history: sources.map((src) => _.extend({ + block_number: src.number + }, src)) + }; + }); this.savePeer = function(peer) { return that.peerDAL.savePeer(peer); @@ -1175,24 +1137,24 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { this.pushStats = that.statDAL.pushStats; this.needsSave = function() { - return loki.autosaveDirty(); + return true; }; this.close = function() { if (that.needsSave()) { - return Q.nbind(loki.saveDatabase, loki)(); + return Q.nbind(sqlite.close, sqlite)(); } return Q(); }; this.resetAll = function(done) { - var files = ['stats', 'cores', 'current', 'conf', UCOIN_DB_NAME]; + var files = ['stats', 'cores', 'current', 'conf', UCOIN_DB_NAME, UCOIN_DB_NAME + '.db']; var dirs = ['blocks', 'ud_history', 'branches', 'certs', 'txs', 'cores', 'sources', 'links', 'ms', 'identities', 'peers', 'indicators', 'leveldb']; return resetFiles(files, dirs, done); }; this.resetData = function(done) { - var files = ['stats', 'cores', 'current', UCOIN_DB_NAME]; + var files = ['stats', 'cores', 'current', UCOIN_DB_NAME, UCOIN_DB_NAME + '.db']; var dirs = ['blocks', 'ud_history', 'branches', 'certs', 'txs', 'cores', 'sources', 'links', 'ms', 'identities', 'peers', 'indicators', 'leveldb']; return resetFiles(files, dirs, done); }; @@ -1213,7 +1175,7 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { var files = []; var dirs = ['peers']; return co(function *() { - that.peerDAL.lokiRemoveAll(); + that.peerDAL.removeAll(); yield resetFiles(files, dirs); return that.close(); }) @@ -1228,29 +1190,30 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, loki) { }; function resetFiles(files, dirs, done) { - return Q.all([ - - // Remove files - Q.all(files.map(function(fName) { - return myFS.exists(rootPath + '/' + fName + '.json') - .then(function(exists){ - return exists ? myFS.remove(rootPath + '/' + fName + '.json') : Q(); - }); - })), - - // Remove directories - Q.all(dirs.map(function(dirName) { - return myFS.exists(rootPath + '/' + dirName) - .then(function(exists){ - return exists ? myFS.removeTree(rootPath + '/' + dirName) : Q(); - }); - })) - ]) - .then(function(){ - done && done(); - }) - .catch(function(err){ - done && done(err); - }); + return co(function *() { + for (let i = 0, len = files.length; i < len; i++) { + let fName = files[i]; + // JSON file? + let existsJSON = yield myFS.exists(rootPath + '/' + fName + '.json'); + if (existsJSON) { + yield myFS.remove(rootPath + '/' + fName + '.json'); + } else { + // Normal file? + let existsFile = yield myFS.exists(rootPath + '/' + fName); + if (existsFile) { + yield myFS.remove(rootPath + '/' + fName); + } + } + } + for (let i = 0, len = dirs.length; i < len; i++) { + let dirName = dirs[i]; + let existsDir = yield myFS.exists(rootPath + '/' + dirName); + if (existsDir) { + yield myFS.removeTree(rootPath + '/' + dirName); + } + } + done && done(); + }) + .catch((err) => done && done(err)); } } diff --git a/app/lib/dal/fileDALs/AbstractLoki.js b/app/lib/dal/fileDALs/AbstractLoki.js deleted file mode 100644 index daab0c6f64a16e89e9d40a95bde08bf08123f2cf..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/AbstractLoki.js +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Created by cgeek on 16/10/15. - */ - -var Q = require('q'); -var _ = require('underscore'); - -module.exports = AbstractLoki; - -function AbstractLoki(collection) { - - "use strict"; - - function find(conditons) { - return collection.find(conditons); - } - - this.IMMUTABLE_FIELDS = true; - - this.collection = collection; - - this.lokiFind = (baseConditions, metaConditions) => - Q(collection.find(getConditions(baseConditions, metaConditions))); - - this.lokiFindOne = (baseConditions, metaConditions) => - Q(collection.find(getConditions(baseConditions, metaConditions))[0] || null); - - this.lokiFindInAll = (metaConditions) => - Q(find(metaConditions)); - - this.lokiExisting = (entity) => { - let uniqueFindConditions = this.idKeys.map((key) => { - let cond = {}; - cond[key] = entity[key]; - return cond; - }); - return find({ - $and: uniqueFindConditions - })[0]; - }; - - this.lokiSave = (fullEntity) => { - let entity = fullEntity; - if (this.propsToSave) { - entity = _.pick(fullEntity, this.propsToSave || []); - } - let existing = this.lokiExisting(entity); - if (existing) { - // Save in main branch: overrides main data - existing = _.extend(existing, entity); - collection.update(existing); - } else { - collection.insert(entity); - } - return Q(entity); - }; - - this.lokiRemove = (fullEntity) => { - let entity = fullEntity; - if (this.propsToSave) { - entity = _.pick(fullEntity, this.propsToSave || []); - } - let existing = this.lokiExisting(entity); - if (existing) { - collection.remove(existing); - return true; - } - return false; - }; - - this.lokiRemoveAll = () => - collection.removeDataOnly(); - - this.lokiRemoveWhere = (conditions) => - collection.removeWhere(conditions); - - function getConditions(baseConditions, metaConditions) { - let conditions = { - $and: [baseConditions, metaConditions] - }; - if (!baseConditions || !metaConditions) { - conditions = baseConditions || metaConditions; - } - return conditions; - } -} \ No newline at end of file diff --git a/app/lib/dal/fileDALs/BlockDAL.js b/app/lib/dal/fileDALs/BlockDAL.js deleted file mode 100644 index d51a57d84f1a0cea19df0102e9c5c2c2200a93e4..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/BlockDAL.js +++ /dev/null @@ -1,244 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var Q = require('q'); -var _ = require('underscore'); -var co = require('co'); -var constants = require('../../constants'); -var logger = require('../../../../app/lib/logger')('blockdal'); - -const BLOCK_FILE_PREFIX = "0000000000"; -const BLOCK_FOLDER_SIZE = 500; - -module.exports = BlockDAL; - -function BlockDAL(loki, rootFS, getLowerWindowBlock) { - - "use strict"; - - let collection = loki.getCollection('blocks') || loki.addCollection('blocks', { indices: ['fork', 'number', 'hash'] }); - let blocksDB = getView(); - let forksDB = getForkView(); - let current = null; - let that = this; - - this.init = () => co(function *() { - yield rootFS.makeTree('blocks/'); - }); - - this.getCurrent = () => { - if (!current) { - current = blocksDB.branchResultset().simplesort('number', true).limit(1).data()[0]; - } - return Q(current); - }; - - this.getBlock = (number) => co(function *() { - let block = blocksDB.branchResultset().find({ number: parseInt(number) }).data()[0]; - if (!block) { - try { - block = yield rootFS.readJSON(pathOfBlock(number) + blockFileName(number) + '.json'); - } catch(e) { - block = null; - } - } - return block; - }); - - this.getAbsoluteBlock = (number, hash) => co(function *() { - let block = collection.find({ - $and: [{ - number: parseInt(number) - }, { - hash: hash - } - ]})[0]; - if (!block) { - try { - block = yield rootFS.readJSON(pathOfBlock(number) + blockFileName(number) + '.json'); - } catch(e) { - block = null; - } - } - return block; - }); - - this.blocksDB = blocksDB; - this.forksDB = forksDB; - this.collection = collection; - - this.getBlocks = (start, end) => { - let lowerInLoki = blocksDB.branchResultset().simplesort('number').limit(1).data()[0]; - let lokiBlocks = blocksDB.branchResultset().find({ - $and: [{ - number: { $gte: start } - }, { - number: { $lte: end } - }] - }).data(); - if (lowerInLoki.number <= start) { - return Q(lokiBlocks); - } - return co(function *() { - let filesBlocks = yield Q.all(_.range(start, Math.min(lowerInLoki.number, end + 1)).map((number) => rootFS.readJSON(pathOfBlock(number) + blockFileName(number) + '.json'))); - return filesBlocks.concat(lokiBlocks); - }); - }; - - this.lastBlockWithDividend = () => { - let blocks = blocksDB.branchResultset().find({ dividend: { $gt: 0 } }).simplesort('number', true).data(); - return blocks[0]; - }; - - this.lastBlockOfIssuer = (issuer) => { - let blocksOfIssuer = blocksDB.branchResultset().find({ issuer: issuer }).simplesort('number', true).limit(1).data(); - return Q(blocksOfIssuer[0]); - }; - - this.getForkBlocks = () => - Q(forksDB.branchResultset().find({ wrong: false }).data()); - - this.saveBunch = (blocks, inFiles) => { - if (!inFiles) { - collection.insert(blocks); - return Q(); - } else { - // Save in files - return co(function *() { - let trees = []; - blocks.forEach(function(block){ - let pathForBlock = pathOfBlock(block.number); - if (!~trees.indexOf(pathForBlock)) { - trees.push(pathForBlock); - } - }); - yield trees.map((tree) => rootFS.makeTree(tree)); - yield blocks.map((block) => rootFS.writeJSON(pathOfBlock(block.number) + blockFileName(block.number) + '.json', block)); - }); - } - }; - - this.saveBlock = (block) => { - if (!current || current.number < block.number) { - current = block; - } - let existing; - existing = collection.find({ - $and: [{ - number: block.number - }, { - hash: block.hash - }] - })[0]; - if (existing) { - // Updates - collection.update(_.extend(existing, block)); - } else { - collection.insert(block); - } - return Q(block); - }; - - this.saveSideBlock = (block) => { - block.fork = true; - let existing; - existing = collection.find({ - $and: [{ - number: block.number - }, { - hash: block.hash - }] - })[0]; - if (existing) { - // Updates - collection.update(_.extend(existing, block)); - } else { - collection.insert(block); - } - return Q(block); - }; - - this.setSideBlock = (block, previousBlock) => co(function *() { - let existing = collection.find({ - $and: [{ - number: block.number - }, { - hash: block.hash - }] - }); - (existing || []).forEach(function(found){ - found.fork = true; - collection.update(found); - }); - current = previousBlock; - let lowerInLoki = blocksDB.branchResultset().simplesort('number').limit(1).data()[0]; - if (lowerInLoki && lowerInLoki.number > 0) { - let newLower = yield that.getBlock(lowerInLoki.number - 1); - yield rootFS.remove(pathOfBlock(newLower.number) + blockFileName(newLower.number) + '.json'); - collection.insert(newLower); - } - }); - - this.migrateOldBlocks = () => co(function *() { - let number = yield getLowerWindowBlock(); - logger.debug("Clean some blocks from memory to disk..."); - logger.debug("Lower block = %s", number); - let lowerInLoki = blocksDB.branchResultset().simplesort('number').limit(1).data()[0]; - if (!lowerInLoki) { - return; - } - let lastUDBlock = that.lastBlockWithDividend(); - if (lastUDBlock) { - logger.debug("LastUD in loki = %s", lastUDBlock.number); - logger.debug("Lower in loki = %s", lowerInLoki.number); - let deadBlocksInLoki = number - lowerInLoki.number; - logger.debug("Dead blocks = %s", deadBlocksInLoki); - if (deadBlocksInLoki >= constants.BLOCKS_COLLECT_THRESHOLD) { - let blocksToPersist = blocksDB.branchResultset().find({ - $and: [{ - number: { $gte: lowerInLoki.number } - }, { - number: { $lte: number } - }] - }).simplesort('number').data(); - logger.debug("To store in files = %s to %s", blocksToPersist[0].number, blocksToPersist[blocksToPersist.length - 1].number); - for (let i = 0; i < blocksToPersist.length; i++) { - let block = blocksToPersist[i]; - yield rootFS.makeTree(pathOfBlock(block.number)); - yield rootFS.writeJSON(pathOfBlock(block.number) + blockFileName(block.number) + '.json', block); - collection.remove(block); - } - lowerInLoki = blocksDB.branchResultset().simplesort('number').limit(1).data()[0]; - logger.debug("Lower in loki now = %s", lowerInLoki.number); - logger.debug("LastUD in loki = %s", that.lastBlockWithDividend().number); - } - } - }); - - function getView() { - let view; - // Main branch - view = collection.addDynamicView('mainBranch'); - view.applyFind({ fork: false }); - return view; - } - - function getForkView() { - let view = collection.addDynamicView('forks'); - view.applyFind({ fork: true }); - return view; - } - - function folderOfBlock(blockNumber) { - return (Math.floor(blockNumber / BLOCK_FOLDER_SIZE) + 1) * BLOCK_FOLDER_SIZE; - } - - function pathOfBlock(blockNumber) { - return 'blocks/' + folderOfBlock(blockNumber) + '/'; - } - - function blockFileName(blockNumber) { - return BLOCK_FILE_PREFIX.substr(0, BLOCK_FILE_PREFIX.length - ("" + blockNumber).length) + blockNumber; - } -} diff --git a/app/lib/dal/fileDALs/CertDAL.js b/app/lib/dal/fileDALs/CertDAL.js deleted file mode 100644 index 9c1cb851e13997a6cb7fd7394ab8a02813a9ec9f..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/CertDAL.js +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var Q = require('q'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = CertDAL; - -function CertDAL(loki) { - - "use strict"; - - let collection = loki.getCollection('certs') || loki.addCollection('certs', { indices: ['from', 'target', 'linked', 'written'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['sig', 'from', 'target']; - this.propsToSave = [ - 'linked', - 'written_block', - 'written_hash', - 'sig', - 'block_number', - 'block_hash', - 'target', - 'to', - 'from', - 'block' - ]; - - this.init = () => null; - - this.getToTarget = (hash) => this.lokiFind({ - target: hash - }); - - this.getFromPubkey = (pubkey) => this.lokiFind({ - from: pubkey - }); - - this.getNotLinked = () => this.lokiFindInAll({ - linked: false - }); - - this.getNotLinkedToTarget = (hash) => this.lokiFind({ - target: hash - },{ - linked: false - }); - - this.listLocalPending = () => Q([]); - - this.saveOfficial = (cert) => { - cert.linked = true; - return this.lokiSave(cert); - }; - - this.saveCert = (cert) => - this.lokiSave(cert); - - this.saveNewCertification = (cert) => - this.lokiSave(cert); - - this.existsGivenCert = (cert) => Q(this.lokiExisting(cert)); -} \ No newline at end of file diff --git a/app/lib/dal/fileDALs/LinksDAL.js b/app/lib/dal/fileDALs/LinksDAL.js deleted file mode 100644 index 154043b3c63978d41a45648e54bd8a24c20801f0..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/LinksDAL.js +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var co = require('co'); -var Q = require('q'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = LinksDAL; - -function LinksDAL(loki) { - - "use strict"; - - let that = this; - let collection = loki.getCollection('links') || loki.addCollection('links', { indices: ['source', 'target', 'block_number', 'block_hash', 'timestamp'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['source', 'target', 'block_number', 'block_hash']; - this.propsToSave = [ - 'source', - 'target', - 'timestamp', - 'block_number', - 'block_hash', - 'obsolete' - ]; - - this.init = () => null; - - this.getValidLinksFrom = (pubkey) => this.lokiFind({ - source: pubkey - }, { - obsolete: false - }); - - this.getSimilarLinksFromDate = (from, to, minDate) => this.lokiFind({ - $and: [ - { source: from }, - { target: to }, - { timestamp: { $gte: minDate }} - ] - }); - - this.getValidLinksTo = (pubkey) => this.lokiFind({ - target: pubkey - }, { - obsolete: false - }); - - this.getLinksWithPath = (from, to) => - this.lokiFind({ - $and: [{ - source: from - },{ - target: to - }] - }); - - this.obsoletesLinks = (minTimestamp) => co(function *() { - let toObsolete = yield that.lokiFind({ - timestamp: { $lte: minTimestamp } - },{ - obsolete: false - }); - for (let i = 0; i < toObsolete.length; i++) { - let link = toObsolete[i]; - link.obsolete = true; - collection.update(link); - } - }); - - this.unObsoletesLinks = (minTimestamp) => co(function *() { - let toObsolete = yield that.lokiFind({ - timestamp: { $gte: minTimestamp } - }); - for (let i = 0; i < toObsolete.length; i++) { - let link = toObsolete[i]; - link.obsolete = false; - collection.update(link); - } - }); - - this.addLink = (link) => { - link.obsolete = false; - return this.lokiSave(link); - }; - - this.removeLink = (link) => - this.lokiRemove(link); -} \ No newline at end of file diff --git a/app/lib/dal/fileDALs/MembershipDAL.js b/app/lib/dal/fileDALs/MembershipDAL.js deleted file mode 100644 index df58330c8eead2a7fcec68fcf1f412e95779e519..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/MembershipDAL.js +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var Q = require('q'); -var _ = require('underscore'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = MembershipDAL; - -function MembershipDAL(loki) { - - "use strict"; - - let collection = loki.getCollection('memberships') || loki.addCollection('memberships', { indices: ['membership', 'issuer', 'number', 'blockNumber', 'blockHash', 'userid', 'certts', 'block', 'fpr', 'written', 'signature'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['issuer', 'signature']; - this.propsToSave = [ - 'membership', - 'issuer', - 'number', - 'blockNumber', - 'blockHash', - 'userid', - 'certts', - 'block', - 'fpr', - 'written', - 'signature' - ]; - - this.init = () => null; - - this.getMembershipOfIssuer = (ms) => Q(this.lokiExisting(ms)); - - this.getMembershipsOfIssuer = (issuer) => this.lokiFind({ - issuer: issuer - }); - - this.getPendingLocal = () => Q([]); - - this.getPendingIN = () => this.lokiFind({ - membership: 'IN' - },{ - written: false - }); - - this.getPendingOUT = () => this.lokiFind({ - membership: 'OUT' - },{ - written: false - }); - - this.unwriteMS = (ms) => { - let existing = this.lokiExisting({ - issuer: ms.issuer, - signature: ms.signature - }); - if (existing) { - existing.written = false; - collection.update(existing); - } - }; - - this.saveOfficialMS = (type, ms) => { - let obj = _.extend({}, ms); - obj.membership = type.toUpperCase(); - obj.written = true; - return this.lokiSave(_.pick(obj, 'membership', 'issuer', 'number', 'blockNumber', 'blockHash', 'userid', 'certts', 'block', 'fpr', 'written', 'signature')); - }; - - this.savePendingMembership = (ms) => { - ms.membership = ms.membership.toUpperCase(); - ms.written = false; - return this.lokiSave(_.pick(ms, 'membership', 'issuer', 'number', 'blockNumber', 'blockHash', 'userid', 'certts', 'block', 'fpr', 'written', 'signature')); - }; -} diff --git a/app/lib/dal/fileDALs/PeerDAL.js b/app/lib/dal/fileDALs/PeerDAL.js deleted file mode 100644 index 46b667e959037bec9b5299dec3448b483d8e06fb..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/PeerDAL.js +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var Q = require('q'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = PeerDAL; - -function PeerDAL(loki) { - - "use strict"; - - let collection = loki.getCollection('peers') || loki.addCollection('peers', { indices: ['pubkey', 'status'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['pubkey']; - this.propsToSave = [ - 'version', - 'currency', - 'status', - 'statusTS', - 'hash', - 'first_down', - 'last_try', - 'pub', - 'pubkey', - 'block', - 'signature', - 'endpoints', - 'raw' - ]; - - this.init = () => null; - - this.listAll = () => Q(collection.find()); - - this.getPeer = (pubkey) => Q(collection.find({ pubkey: pubkey })[0]); - - this.savePeer = (peer) => this.lokiSave(peer); -} diff --git a/app/lib/dal/fileDALs/SourcesDAL.js b/app/lib/dal/fileDALs/SourcesDAL.js deleted file mode 100644 index c0ed6d6bdbb948703009f42e4a2ec978265fbe4a..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/SourcesDAL.js +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ - -var Q = require('q'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = SourcesDAL; - -function SourcesDAL(loki) { - - "use strict"; - - let collection = loki.getCollection('sources') || loki.addCollection('sources', { indices: ['pubkey', 'type', 'number', 'fingerprint', 'amount', 'block_hash'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['pubkey', 'type', 'number', 'fingerprint', 'amount']; - this.propsToSave = [ - 'pubkey', - 'type', - 'number', - 'time', - 'fingerprint', - 'amount', - 'block_hash', - 'consumed' - ]; - - this.init = () => null; - - this.getAvailableForPubkey = (pubkey) => this.lokiFind({ - pubkey: pubkey - },{ - consumed: false - }); - - this.getUDSources = (pubkey) => this.lokiFind({ - $and: [{ - pubkey: pubkey - },{ - type: 'D' - }] - }); - - this.getSource = (pubkey, type, number) => this.lokiFindOne({ - $and: [ - { pubkey: pubkey }, - { type: type }, - { number: number } - ] - }, null, this.IMMUTABLE_FIELDS); - - this.isAvailableSource = (pubkey, type, number, fingerprint, amount) => { - let src = this.lokiExisting({ - pubkey: pubkey, - type: type, - number: number, - fingerprint: fingerprint, - amount: amount - }); - return Q(src ? !src.consumed : false); - }; - - this.consumeSource = (pubkey, type, number, fingerprint, amount) => { - let src = this.lokiExisting({ - pubkey: pubkey, - type: type, - number: number, - fingerprint: fingerprint, - amount: amount - }); - src.consumed = true; - return this.lokiSave(src); - }; - - this.addSource = (state, pubkey, type, number, fingerprint, amount, block_hash, time) => this.lokiSave({ - pubkey: pubkey, - type: type, - number: number, - fingerprint: fingerprint, - amount: amount, - time: time, - block_hash: block_hash, - consumed: false - }); - - this.unConsumeSource = (type, pubkey, number, fingerprint, amount, time, block_hash) => { - let src = this.lokiExisting({ - pubkey: pubkey, - type: type, - number: number, - fingerprint: fingerprint, - amount: amount - }); - if (src) { - src.consumed = false; - collection.update(src); - } else { - this.lokiSave({ - pubkey: pubkey, - type: type, - number: number, - fingerprint: fingerprint, - amount: amount, - time: time, - block_hash: block_hash, - consumed: false - }); - } - }; - - this.removeAllSourcesOfBlock = (number) => this.lokiRemoveWhere({ - number: number - }); -} \ No newline at end of file diff --git a/app/lib/dal/fileDALs/TxsDAL.js b/app/lib/dal/fileDALs/TxsDAL.js deleted file mode 100644 index 760118f7547ee8ad1f9e86156081d4c96ad539d6..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDALs/TxsDAL.js +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Created by cgeek on 22/08/15. - */ -var co = require('co'); -var Q = require('q'); -var _ = require('underscore'); -var AbstractLoki = require('./AbstractLoki'); - -module.exports = TxsDAL; - -function TxsDAL(loki) { - - "use strict"; - - let that = this; - let collection = loki.getCollection('txs') || loki.addCollection('txs', { indices: ['hash', 'block_number', 'written', 'signature', 'recipients'] }); - - AbstractLoki.call(this, collection); - - this.idKeys = ['hash', 'block_number']; - this.propsToSave = [ - 'inputs', - 'outputs', - 'issuers', - 'signatories', - 'signatures', - 'comment', - 'hash', - 'version', - 'currency', - 'block_number', - 'time', - 'recipients', - 'written', - 'removed' - ]; - - this.init = () => null; - - this.getAllPending = () => - this.lokiFindInAll({ - $and: [{ - written: false - },{ - removed: false - }] - }); - - this.getTX = (hash) => this.lokiFindOne({ - hash: hash - }, null, this.IMMUTABLE_FIELDS); - - this.removeTX = (hash) => co(function *() { - let tx = yield that.lokiFindOne({ - hash: hash - }); - if (tx) { - tx.removed = true; - return that.lokiSave(tx); - } - return Q(tx); - }); - - this.addLinked = (tx) => { - tx.written = true; - tx.removed = false; - tx.hash = tx.getHash(true); - tx.recipients = tx.outputs.map(function(out) { - return out.match('(.*):')[1]; - }); - return that.lokiSave(tx); - }; - - this.addPending = (tx) => { - tx.written = false; - tx.removed = false; - tx.hash = tx.getHash(true); - tx.recipients = tx.outputs.map(function(out) { - return out.match('(.*):')[1]; - }); - return this.lokiSave(tx); - }; - - this.getLinkedWithIssuer = (pubkey) => this.lokiFind({ - issuers: { $contains: pubkey } - },{ - written: true - }); - - this.getLinkedWithRecipient = (pubkey) => this.lokiFind({ - recipients: { $contains: pubkey } - },{ - written: true - }); - - this.getPendingWithIssuer = (pubkey) => this.lokiFind({ - issuers: { $contains: pubkey } - },{ - written: false - }); - - this.getPendingWithRecipient = (pubkey) => this.lokiFind({ - recipients: { $contains: pubkey } - },{ - written: false - }); -} \ No newline at end of file diff --git a/app/lib/dal/sqliteDAL/AbstractSQLite.js b/app/lib/dal/sqliteDAL/AbstractSQLite.js new file mode 100644 index 0000000000000000000000000000000000000000..54522c716f01fa7e416b45ff3c06cfe03adbdaaf --- /dev/null +++ b/app/lib/dal/sqliteDAL/AbstractSQLite.js @@ -0,0 +1,327 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var _ = require('underscore'); +var co = require('co'); +var colors = require('colors'); + +module.exports = AbstractSQLite; + +function AbstractSQLite(db) { + + "use strict"; + + let that = this; + + this.ASC = false; + this.DESC = true; + this.arrays = []; + this.booleans = []; + this.bigintegers = []; + this.translated = {}; + + this.query = (sql, params) => co(function *() { + try { + if (sql.match(/^INSERT/)) { + //console.log(sql, JSON.stringify(params || [])); + } + let start = new Date(); + let res = yield Q.nbind(db.all, db)(sql, params || []); + let duration = (new Date()) - start; + let entities = res.map(toEntity); + if (true || that.table == "source") { + let msg = sql + ' | %s\t==> %s rows in %s ms'; + if (duration <= 2) { + msg = colors.green(msg); + } else if(duration <= 5) { + msg = colors.yellow(msg); + } else if (duration <= 10) { + msg = colors.magenta(msg); + } else if (duration <= 100) { + msg = colors.red(msg); + } + if (duration > 10) { + //console.log(msg, JSON.stringify([] || []), entities.length, duration); + } + } + return entities; + } catch (e) { + console.error('ERROR >> %s', sql, JSON.stringify(params || []), e.stack || e.message || e); + throw e; + } + }); + + this.sqlListAll = () => this.query("SELECT * FROM " + this.table); + + this.sqlDeleteAll = () => this.exec("DELETE FROM " + this.table); + + this.sqlFind = (obj, sortObj) => co(function *() { + let conditions = toConditionsArray(obj).join(' and '); + let values = toParams(obj); + let sortKeys = _.keys(sortObj); + let sort = sortKeys.length ? ' ORDER BY ' + sortKeys.map((k) => "`" + k + "` " + (sortObj[k] ? 'DESC' : 'ASC')).join(',') : ''; + return that.query('SELECT * FROM ' + that.table + ' WHERE ' + conditions + sort, values); + }); + + this.sqlFindOne = (obj) => co(function *() { + let res = yield that.sqlFind(obj); + return res[0]; + }); + + this.sqlFindLikeAny = (obj, sort) => co(function *() { + let keys = _.keys(obj); + return that.query('SELECT * FROM ' + that.table + ' WHERE ' + keys.map((k) => '`' + k + '` like ?').join(' or '), keys.map((k) => obj[k].toUpperCase()), sort); + }); + + this.sqlUpdateWhere = (obj, where) => co(function *() { + // Valorizations + let setInstructions = toSetArray(obj).join(', '); + let setValues = toParams(obj); + // Conditions + let conditions = toConditionsArray(where).join(' AND '); + let condValues = toParams(where); + return that.query('UPDATE ' + that.table + ' SET ' + setInstructions + ' WHERE ' + conditions, setValues.concat(condValues)); + }); + + this.sqlRemoveWhere = (obj) => co(function *() { + let keys = _.keys(obj); + return that.query('DELETE FROM ' + that.table + ' WHERE ' + keys.map((k) => '`' + k + '` = ?').join(' and '), keys.map((k) => obj[k])); + }); + + this.sqlExisting = (entity) => that.getEntity(entity); + + this.saveEntity = (entity) => co(function *() { + let toSave = entity; + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let existing = yield that.getEntity(toSave); + if (existing) { + toSave = toRow(toSave); + let valorizations = that.fields.map((field) => '`' + field + '` = ?').join(', '); + let conditions = getPKFields().map((field) => '`' + field + '` = ?').join(' and '); + let setValues = that.fields.map((field) => toSave[field]); + let condValues = getPKFields().map((k) => toSave[k]); + return that.query('UPDATE ' + that.table + ' SET ' + valorizations + ' WHERE ' + conditions, setValues.concat(condValues)); + } + yield that.insert(toSave); + }); + + this.updateEntity = (entity, values) => co(function *() { + let toSave = toRow(entity); + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let valuesKeys = _.keys(values); + let valorizations = valuesKeys.map((field) => '`' + field + '` = ?').join(', '); + let conditions = getPKFields().map((field) => '`' + field + '` = ?').join(' and '); + let setValues = valuesKeys.map((field) => values[field]); + let condValues = getPKFields().map((k) => toSave[k]); + return that.query('UPDATE ' + that.table + ' SET ' + valorizations + ' WHERE ' + conditions, setValues.concat(condValues)); + }); + + this.deleteEntity = (entity) => co(function *() { + let toSave = toRow(entity); + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let conditions = getPKFields().map((field) => '`' + field + '` = ?').join(' and '); + let condValues = getPKFields().map((k) => toSave[k]); + return that.query('DELETE FROM ' + that.table + ' WHERE ' + conditions, condValues); + }); + + this.insert = (entity) => co(function *() { + let row = toRow(entity); + let values = that.fields.map((f) => row[f]); + yield that.query(that.getInsertQuery(), values); + }); + + this.getEntity = function(entity) { + return co(function *() { + let conditions = getPKFields().map((field) => '`' + field + '` = ?').join(' and '); + let params = toParams(entity, getPKFields()); + return (yield that.query('SELECT * FROM ' + that.table + ' WHERE ' + conditions, params))[0]; + }); + }; + + this.exec = (sql) => co(function *() { + try { + if (sql.match(/INSERT INTO source/)) { + //console.log('------------'); + //console.log(sql); + //console.log('------------'); + } + return Q.nbind(db.exec, db)(sql); + } catch (e) { + console.error('ERROR >> %s', sql); + throw e; + } + }); + + this.getInsertQuery = () => + "INSERT INTO " + that.table + " (" + getFields(that.fields).map(f => '`' + f + '`').join(',') + ") VALUES (" + "?,".repeat(that.fields.length - 1) + "?);"; + + this.getInsertHead = () => { + let valuesKeys = getFields(that.fields); + return 'INSERT INTO ' + that.table + " (" + valuesKeys.map(f => '`' + f + '`').join(',') + ") VALUES "; + }; + + this.getInsertValue = (toSave) => { + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let row = toRow(toSave); + let valuesKeys = getFields(that.fields); + let values = valuesKeys.map((field) => escapeToSQLite(row[field])); + return "(" + values.join(',') + ")"; + }; + + this.getUpdateRawQuery = (toSave, values) => { + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let valuesKeys = _.keys(values); + let valorizations = valuesKeys.map((field) => '`' + field + '` = ' + escapeToSQLite(values[field])).join(', '); + let conditions = getPKFields().map((field) => '`' + field + '` = ' + escapeToSQLite(toSave[field])).join(' and '); + return 'UPDATE ' + that.table + ' SET ' + valorizations + ' WHERE ' + conditions + ';'; + }; + + this.getDeleteRawQuery = (toSave) => { + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let conditions = getPKFields().map((field) => '`' + field + '` = ' + escapeToSQLite(toSave[field])).join(' and '); + return 'DELETE FROM ' + that.table + ' WHERE ' + conditions + ';'; + }; + + this.getDeleteHead = () => { + return 'DELETE FROM ' + that.table + " WHERE "; + }; + + this.getDeleteValues = (entities) => { + return entities.map((toSave) => { + if (that.beforeSaveHook) { + that.beforeSaveHook(toSave); + } + let conditions = getPKFields().map((field) => '`' + field + '` = ' + escapeToSQLite(toSave[field])).join(' and '); + return "(" + conditions + ")"; + }).join(' OR\n '); + }; + + this.toInsertValues = (entity) => { + let row = toRow(entity); + let values = that.fields.map((f) => row[f]); + let formatted = values.map(escapeToSQLite); + return "(" + formatted.join(',') + ")"; + }; + + function toConditionsArray(obj) { + return _.keys(obj).map((k) => { + if (obj[k].$lte !== undefined) { + return '`' + k + '` <= ?'; + } else if (obj[k].$gte !== undefined) { + return '`' + k + '` >= ?'; + } else if (obj[k].$contains !== undefined) { + return '`' + k + '` LIKE ?'; + } else { + return '`' + k + '` = ?'; + } + }); + } + + function toSetArray(obj) { + let row = toRow(obj); + return _.keys(row).map((k) => '`' + k + '` = ?'); + } + + function toParams(obj, fields) { + return (fields || _.keys(obj)).map((f) => { + if (obj[f].$lte !== undefined) { + return obj[f].$lte; + } else if (obj[f].$gte !== undefined) { + return obj[f].$gte; + } else if (obj[f].$contains !== undefined) { + return "%" + obj[f].$contains + "%"; + } else if (~that.bigintegers.indexOf(f) && typeof obj[f] !== "string") { + return String(obj[f]); + } else { + return obj[f]; + } + }); + } + + function escapeToSQLite(val) { + if (typeof val == "boolean") { + // SQLite specific: true => 1, false => 0 + return val ? 1 : 0; + } + else if (typeof val == "string") { + return "'" + val.replace(/'/g, "\\'") + "'"; + } + else if (val === undefined) { + return "null"; + } else { + return JSON.stringify(val); + } + } + + function getPKFields() { + return getFields(that.pkFields); + } + + function getFields(fields) { + return fields.map((f) => getField(f)); + } + + function getField(f) { + return that.translated[f] || f; + } + + function toEntity(row) { + for (let i = 0, len = that.arrays.length; i < len; i++) { + let arr = that.arrays[i]; + row[arr] = JSON.parse(row[arr]); + } + // Big integers are stored as strings to avoid data loss + for (let i = 0, len = that.bigintegers.length; i < len; i++) { + let bigint = that.bigintegers[i]; + row[bigint] = parseInt(row[bigint], 10); + } + // Translate some DB fields to obj fields + let toTranslate = that.translated || {}; + let toDBFields = _.keys(toTranslate); + for (let i = 0, len = toDBFields.length; i < len; i++) { + let objField = toDBFields[i]; + row[objField] = row[toTranslate[objField]]; + } + // Booleans + for (let i = 0, len = that.booleans.length; i < len; i++) { + let f = that.booleans[i]; + row[f] = Boolean(row[f]); + } + return row; + } + + function toRow(entity) { + let row = _.clone(entity); + for (let i = 0, len = that.arrays.length; i < len; i++) { + let arr = that.arrays[i]; + row[arr] = JSON.stringify(row[arr] || []); + } + // Big integers are stored as strings to avoid data loss + for (let i = 0, len = that.bigintegers.length; i < len; i++) { + let bigint = that.bigintegers[i]; + row[bigint] = String(entity[bigint]); + } + // Translate some obj fields to DB field name (because of DB keywords) + let toTranslate = that.translated || {}; + let toDBFields = _.keys(toTranslate); + for (let i = 0, len = toDBFields.length; i < len; i++) { + let objField = toDBFields[i]; + row[toTranslate[objField]] = row[objField]; + } + return row; + } +} \ No newline at end of file diff --git a/app/lib/dal/sqliteDAL/BlockDAL.js b/app/lib/dal/sqliteDAL/BlockDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..f305eee70a773268a4781eee7a94c3286f94a130 --- /dev/null +++ b/app/lib/dal/sqliteDAL/BlockDAL.js @@ -0,0 +1,135 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var co = require('co'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = BlockDAL; + +const IS_FORK = true; +const IS_NOT_FORK = false; + +function BlockDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let current = null; + let that = this; + + this.table = 'block'; + this.fields = ['fork', 'hash', 'signature', 'currency', 'issuer', 'parameters', 'previousHash', 'previousIssuer', 'version', 'membersCount', 'monetaryMass', 'UDTime', 'medianTime', 'dividend', 'time', 'powMin', 'number', 'nonce', 'transactions', 'certifications', 'identities', 'joiners', 'actives', 'leavers', 'excluded']; + this.arrays = ['identities','certifications','actives','excluded','leavers','joiners','transactions']; + this.bigintegers = ['monetaryMass','dividend']; + this.pkFields = ['number','hash']; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'fork BOOLEAN NOT NULL,' + + 'hash VARCHAR(40) NOT NULL,' + + 'signature VARCHAR(100) NOT NULL,' + + 'currency VARCHAR(50) NOT NULL,' + + 'issuer VARCHAR(50) NOT NULL,' + + 'parameters VARCHAR(255),' + + 'previousHash VARCHAR(50),' + + 'previousIssuer VARCHAR(50),' + + 'version INTEGER NOT NULL,' + + 'membersCount INTEGER NOT NULL,' + + 'monetaryMass INTEGER DEFAULT 0,' + + 'UDTime DATETIME,' + + 'medianTime DATETIME NOT NULL,' + + 'dividend INTEGER,' + + 'time DATETIME NOT NULL,' + + 'powMin INTEGER NOT NULL,' + + 'number INTEGER NOT NULL,' + + 'nonce INTEGER NOT NULL,' + + 'transactions TEXT,' + + 'certifications TEXT,' + + 'identities TEXT,' + + 'joiners TEXT,' + + 'actives TEXT,' + + 'leavers TEXT,' + + 'excluded TEXT,' + + 'created DATETIME DEFAULT NULL,' + + 'updated DATETIME DEFAULT NULL,' + + 'PRIMARY KEY (number,hash)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_block_hash ON block (hash);' + + 'CREATE INDEX IF NOT EXISTS idx_block_fork ON block (fork);' + + 'COMMIT;', []); + }); + + this.getCurrent = () => co(function *() { + if (!current) { + current = (yield that.query('SELECT * FROM block WHERE NOT fork ORDER BY number DESC LIMIT 1'))[0]; + } + return Q(current); + }); + + this.getBlock = (number) => co(function *() { + return (yield that.query('SELECT * FROM block WHERE number = ? and NOT fork', [parseInt(number)]))[0]; + }); + + this.getAbsoluteBlock = (number, hash) => co(function *() { + return (yield that.query('SELECT * FROM block WHERE number = ? and hash = ?', [parseInt(number), hash]))[0]; + }); + + this.getBlocks = (start, end) => { + return that.query('SELECT * FROM block WHERE number BETWEEN ? and ? and NOT fork ORDER BY number ASC', [start, end]); + }; + + this.lastBlockWithDividend = () => co(function *() { + return (yield that.query('SELECT * FROM block WHERE dividend > 0 and NOT fork ORDER BY number DESC LIMIT 1'))[0]; + }); + + this.lastBlockOfIssuer = (issuer) => co(function *() { + return (yield that.query('SELECT * FROM block WHERE issuer = ? and NOT fork ORDER BY number DESC LIMIT 1', [issuer]))[0]; + }); + + this.getForkBlocks = () => { + return that.query('SELECT * FROM block WHERE fork ORDER BY number'); + }; + + this.saveBunch = (blocks) => co(function *() { + let queries = "INSERT INTO block (" + that.fields.join(',') + ") VALUES "; + for (let i = 0, len = blocks.length; i < len; i++) { + let block = blocks[i]; + queries += that.toInsertValues(block); + if (i + 1 < len) { + queries += ",\n"; + } + } + yield that.exec(queries); + }); + + this.saveBlock = (block) => co(function *() { + if (!current || current.number < block.number) { + current = block; + } + return saveBlockAs(block, IS_NOT_FORK); + }); + + this.saveSideBlock = (block) => + saveBlockAs(block, IS_FORK); + + function saveBlockAs(block, fork) { + return co(function *() { + let existing = yield that.getEntity(block); + if (existing) { + return that.query('UPDATE block SET fork = ? WHERE number = ? and hash = ?', [fork, block.number, block.hash]); + } + yield that.insert(block); + }); + } + + this.setSideBlock = (block, previousBlock) => co(function *() { + yield that.query('UPDATE block SET fork = ? WHERE number = ? and hash = ?', [true, block.number, block.hash]); + current = previousBlock; + }); + + this.migrateOldBlocks = () => Q(); +} diff --git a/app/lib/dal/sqliteDAL/CertDAL.js b/app/lib/dal/sqliteDAL/CertDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..27ce340dfd48a791a0a52ee7d4c7792d629a3673 --- /dev/null +++ b/app/lib/dal/sqliteDAL/CertDAL.js @@ -0,0 +1,105 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var co = require('co'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = CertDAL; + +function CertDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'cert'; + this.fields = [ + 'linked', + 'written', + 'written_block', + 'written_hash', + 'sig', + 'block_number', + 'block_hash', + 'target', + 'to', + 'from', + 'block' + ]; + this.arrays = []; + this.booleans = ['linked', 'written']; + this.pkFields = ['from','target','sig']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + '`from` VARCHAR(50) NOT NULL,' + + '`to` VARCHAR(50) NOT NULL,' + + 'target CHAR(40) NOT NULL,' + + 'sig VARCHAR(100) NOT NULL,' + + 'block_number INTEGER NOT NULL,' + + 'block_hash VARCHAR(50),' + + 'block INTEGER NOT NULL,' + + 'linked BOOLEAN NOT NULL,' + + 'written BOOLEAN NOT NULL,' + + 'written_block INTEGER,' + + 'written_hash VARCHAR(50),' + + 'PRIMARY KEY (`from`, target, sig, written_block)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_cert_from ON cert (`from`);' + + 'CREATE INDEX IF NOT EXISTS idx_cert_target ON cert (target);' + + 'CREATE INDEX IF NOT EXISTS idx_cert_linked ON cert (linked);' + + 'COMMIT;', []); + }); + + this.beforeSaveHook = function(entity) { + entity.written = entity.written || !!(entity.written_hash); + }; + + this.getToTarget = (hash) => this.sqlFind({ + target: hash + }); + + this.getFromPubkey = (pubkey) => this.sqlFind({ + from: pubkey + }); + + this.getNotLinked = () => this.sqlFind({ + linked: false + }); + + this.getNotLinkedToTarget = (hash) => this.sqlFind({ + target: hash, + linked: false + }); + + this.listLocalPending = () => Q([]); + + this.saveOfficial = (cert) => { + cert.linked = true; + return this.saveEntity(cert); + }; + + this.saveCert = (cert) => this.saveEntity(cert); + + this.saveNewCertification = (cert) => this.saveEntity(cert); + + this.existsGivenCert = (cert) => Q(this.sqlExisting(cert)); + + this.updateBatchOfCertifications = (certs) => co(function *() { + let queries = []; + let insert = that.getInsertHead(); + let values = certs.map((cert) => that.getInsertValue(cert)); + if (certs.length) { + queries.push(insert + '\n' + values.join(',\n') + ';'); + } + if (queries.length) { + return that.exec(queries.join('\n')); + } + }); +} \ No newline at end of file diff --git a/app/lib/dal/fileDALs/IdentityDAL.js b/app/lib/dal/sqliteDAL/IdentityDAL.js similarity index 65% rename from app/lib/dal/fileDALs/IdentityDAL.js rename to app/lib/dal/sqliteDAL/IdentityDAL.js index 182501d2840dce3239d0d0b74add74d8b2f3afdb..e3213d98b2e11030d1b79ae67651997317d5c6ec 100644 --- a/app/lib/dal/fileDALs/IdentityDAL.js +++ b/app/lib/dal/sqliteDAL/IdentityDAL.js @@ -3,22 +3,21 @@ */ var Q = require('q'); -var _ = require('underscore'); var co = require('co'); -var AbstractLoki = require('./AbstractLoki'); +var AbstractSQLite = require('./AbstractSQLite'); module.exports = IdentityDAL; -function IdentityDAL(loki) { +function IdentityDAL(db) { "use strict"; - let collection = loki.getCollection('identities') || loki.addCollection('identities', { indices: ['uid', 'pubkey', 'timestamp', 'member', 'written'] }); + AbstractSQLite.call(this, db); + let that = this; - AbstractLoki.call(this, collection); - this.idKeys = ['pubkey', 'uid', 'hash']; - this.propsToSave = [ + this.table = 'idty'; + this.fields = [ 'revoked', 'currentMSN', 'memberships', @@ -33,8 +32,39 @@ function IdentityDAL(loki) { 'hash', 'written' ]; - - this.init = () => null; + this.arrays = ['memberships']; + this.booleans = ['revoked', 'member', 'kick', 'leaving', 'wasMember', 'written']; + this.pkFields = ['pubkey', 'uid', 'hash']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'revoked BOOLEAN NOT NULL,' + + 'currentMSN INTEGER NOT NULL,' + + 'memberships TEXT,' + + 'time DATETIME NOT NULL,' + + 'member BOOLEAN NOT NULL,' + + 'kick BOOLEAN NOT NULL,' + + 'leaving BOOLEAN NOT NULL,' + + 'wasMember BOOLEAN NOT NULL,' + + 'pubkey VARCHAR(50) NOT NULL,' + + 'uid VARCHAR(255) NOT NULL,' + + 'sig VARCHAR(100) NOT NULL,' + + 'hash VARCHAR(40) NOT NULL,' + + 'written BOOLEAN NOT NULL,' + + 'PRIMARY KEY (pubkey,uid,hash)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_idty_pubkey ON idty (pubkey);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_uid ON idty (uid);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_kick ON idty (kick);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_member ON idty (member);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_wasMember ON idty (wasMember);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_hash ON idty (hash);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_written ON idty (written);' + + 'CREATE INDEX IF NOT EXISTS idx_idty_currentMSN ON idty (currentMSN);' + + 'COMMIT;', []); + }); this.excludeIdentity = (pubkey) => { return co(function *() { @@ -150,38 +180,47 @@ function IdentityDAL(loki) { }); }; + this.removeUnWrittenWithPubkey = (pubkey) => this.sqlRemoveWhere({ + pubkey: pubkey, + written: false + }); + + this.removeUnWrittenWithUID = (uid) => this.sqlRemoveWhere({ + uid: uid, + written: false + }); + this.getFromPubkey = function(pubkey) { - return that.lokiFindOne({ - pubkey: pubkey - }, { + return that.sqlFindOne({ + pubkey: pubkey, wasMember: true - }, that.IMMUTABLE_FIELDS); + }); }; this.getFromUID = function(uid) { - return that.lokiFindOne({ - uid: uid - }, { + return that.sqlFindOne({ + uid: uid, wasMember: true - }, that.IMMUTABLE_FIELDS); + }); }; this.getByHash = function(hash) { - return that.lokiFindOne({ + return that.sqlFindOne({ hash: hash }); }; - this.saveIdentity = (idty) => this.lokiSave(idty); + this.saveIdentity = (idty) => + this.saveEntity(idty); this.getWhoIsOrWasMember = function() { - return that.lokiFindInAll({ + return that.sqlFind({ wasMember: true }); }; this.getPendingIdentities = function() { - return that.lokiFindInAll({ + return that.sqlFind({ wasMember: false }); }; @@ -189,26 +228,22 @@ function IdentityDAL(loki) { this.listLocalPending = () => Q([]); this.searchThoseMatching = function(search) { - return that.lokiFind({ - $or: [{ - pubkey: { $regex: new RegExp(search, 'i') } - },{ - uid: { $regex: new RegExp(search, 'i') } - }] + return that.sqlFindLikeAny({ + pubkey: "%" + search + "%", + uid: "%" + search + "%" }); }; this.kickMembersForMembershipBelow = (maxNumber) => co(function *() { - let toKick = yield that.lokiFind({ - currentMSN: { $lte: maxNumber } - },{ + let toKick = yield that.sqlFind({ + currentMSN: { $lte: maxNumber }, kick: false, member: true }); for (let i = 0; i < toKick.length; i++) { let idty = toKick[i]; idty.kick = true; - collection.update(idty); + yield that.saveEntity(idty); } }); } \ No newline at end of file diff --git a/app/lib/dal/sqliteDAL/LinksDAL.js b/app/lib/dal/sqliteDAL/LinksDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..90da01b68e5a01e71eaaac3d5cf0088c8b654cb7 --- /dev/null +++ b/app/lib/dal/sqliteDAL/LinksDAL.js @@ -0,0 +1,105 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var co = require('co'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = LinksDAL; + +function LinksDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'link'; + this.fields = [ + 'source', + 'target', + 'timestamp', + 'block_number', + 'block_hash', + 'obsolete' + ]; + this.arrays = []; + this.booleans = ['obsolete']; + this.pkFields = ['source', 'target', 'block_number', 'block_hash']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'source VARCHAR(50) NOT NULL,' + + 'target CHAR(40) NOT NULL,' + + 'timestamp INTEGER NOT NULL,' + + 'block_number INTEGER NOT NULL,' + + 'block_hash VARCHAR(50),' + + 'obsolete BOOLEAN NOT NULL,' + + 'PRIMARY KEY (source,target,block_number,block_hash)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_link_source ON link (source);' + + 'CREATE INDEX IF NOT EXISTS idx_link_obsolete ON link (obsolete);' + + 'CREATE INDEX IF NOT EXISTS idx_link_target ON link (target);' + + 'CREATE INDEX IF NOT EXISTS idx_link_timestamp ON link (timestamp);' + + 'COMMIT;', []); + }); + + this.getValidLinksFrom = (pubkey) => this.sqlFind({ + source: pubkey, + obsolete: false + }); + + this.getSimilarLinksFromDate = (from, to, minDate) => this.sqlFind({ + source: from, + target: to, + timestamp: { $gte: minDate } + }); + + this.getValidLinksTo = (pubkey) => this.sqlFind({ + target: pubkey, + obsolete: false + }); + + this.getLinksWithPath = (from, to) => + this.sqlFind({ + source: from, + target: to + }); + + this.obsoletesLinks = (minTimestamp) => co(function *() { + return that.sqlUpdateWhere({ obsolete: true }, { + timestamp: { $lte: minTimestamp }, + obsolete: false + }); + }); + + this.unObsoletesLinks = (minTimestamp) => co(function *() { + return that.sqlUpdateWhere({ obsolete: false }, { + timestamp: { $gte: minTimestamp } + }); + }); + + this.addLink = (link) => { + link.obsolete = false; + return that.saveEntity(link); + }; + + this.removeLink = (link) => + this.deleteEntity(link); + + this.updateBatchOfLinks = (links) => co(function *() { + let queries = []; + let insert = that.getInsertHead(); + let values = links.map((cert) => that.getInsertValue(cert)); + if (links.length) { + queries.push(insert + '\n' + values.join(',\n') + ';'); + } + if (queries.length) { + return that.exec(queries.join('\n')); + } + }); +} \ No newline at end of file diff --git a/app/lib/dal/sqliteDAL/MembershipDAL.js b/app/lib/dal/sqliteDAL/MembershipDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..a41d8e96c3e91d4d0658054145184ed74932ac3a --- /dev/null +++ b/app/lib/dal/sqliteDAL/MembershipDAL.js @@ -0,0 +1,121 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var co = require('co'); +var _ = require('underscore'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = MembershipDAL; + +function MembershipDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'membership'; + this.fields = [ + 'membership', + 'issuer', + 'number', + 'blockNumber', + 'blockHash', + 'userid', + 'certts', + 'block', + 'fpr', + 'idtyHash', + 'written', + 'signature' + ]; + this.arrays = []; + this.booleans = ['written']; + this.pkFields = ['issuer','signature']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS membership (' + + 'membership CHAR(2) NOT NULL,' + + 'issuer VARCHAR(50) NOT NULL,' + + 'number INTEGER NOT NULL,' + + 'blockNumber INTEGER,' + + 'blockHash VARCHAR(40) NOT NULL,' + + 'userid VARCHAR(255) NOT NULL,' + + 'certts DATETIME NOT NULL,' + + 'block INTEGER,' + + 'fpr VARCHAR(50),' + + 'idtyHash VARCHAR(40),' + + 'written BOOLEAN NOT NULL,' + + 'signature VARCHAR(50),' + + 'PRIMARY KEY (issuer,signature)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_mmembership_idtyHash ON membership (idtyHash);' + + 'CREATE INDEX IF NOT EXISTS idx_mmembership_membership ON membership (membership);' + + 'CREATE INDEX IF NOT EXISTS idx_mmembership_written ON membership (written);' + + 'COMMIT;', []); + }); + + this.getMembershipOfIssuer = (ms) => this.sqlExisting(ms); + + this.getMembershipsOfIssuer = (issuer) => this.sqlFind({ + issuer: issuer + }); + + this.getPendingINOfTarget = (hash) => + this.sqlFind({ + idtyHash: hash, + membership: 'IN', + written: false + }); + + this.getPendingIN = () => this.sqlFind({ + membership: 'IN', + written: false + }); + + this.getPendingOUT = () => this.sqlFind({ + membership: 'OUT', + written: false + }); + + this.unwriteMS = (ms) => co(function *() { + let existing = yield that.sqlExisting({ + issuer: ms.issuer, + signature: ms.signature + }); + if (existing) { + existing.written = false; + that.saveEntity(existing); + } + }); + + this.saveOfficialMS = (type, ms) => { + let obj = _.extend({}, ms); + obj.membership = type.toUpperCase(); + obj.written = true; + return this.saveEntity(_.pick(obj, 'membership', 'issuer', 'number', 'blockNumber', 'blockHash', 'userid', 'certts', 'block', 'fpr', 'idtyHash', 'written', 'signature')); + }; + + this.savePendingMembership = (ms) => { + ms.membership = ms.membership.toUpperCase(); + ms.written = false; + return this.saveEntity(_.pick(ms, 'membership', 'issuer', 'number', 'blockNumber', 'blockHash', 'userid', 'certts', 'block', 'fpr', 'idtyHash', 'written', 'signature')); + }; + + this.updateBatchOfMemberships = (mss) => co(function *() { + let queries = []; + let insert = that.getInsertHead(); + let values = mss.map((cert) => that.getInsertValue(cert)); + if (mss.length) { + queries.push(insert + '\n' + values.join(',\n') + ';'); + } + if (queries.length) { + return that.exec(queries.join('\n')); + } + }); +} diff --git a/app/lib/dal/sqliteDAL/PeerDAL.js b/app/lib/dal/sqliteDAL/PeerDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..b721b0c1670992195b2a364f2635342c7785466c --- /dev/null +++ b/app/lib/dal/sqliteDAL/PeerDAL.js @@ -0,0 +1,66 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var co = require('co'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = PeerDAL; + +function PeerDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'peer'; + this.fields = [ + 'version', + 'currency', + 'status', + 'statusTS', + 'hash', + 'first_down', + 'last_try', + 'pubkey', + 'block', + 'signature', + 'endpoints', + 'raw' + ]; + this.arrays = ['endpoints']; + this.booleans = []; + this.pkFields = ['pubkey']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'version INTEGER NOT NULL,' + + 'currency VARCHAR(50) NOT NULL,' + + 'status VARCHAR(10),' + + 'statusTS INTEGER NOT NULL,' + + 'hash CHAR(40),' + + 'first_down INTEGER,' + + 'last_try INTEGER,' + + 'pubkey VARCHAR(50) NOT NULL,' + + 'block VARCHAR(60) NOT NULL,' + + 'signature VARCHAR(100),' + + 'endpoints TEXT NOT NULL,' + + 'raw TEXT,' + + 'PRIMARY KEY (pubkey)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_link_source ON peer (pubkey);' + + 'COMMIT;', []); + }); + + this.listAll = () => this.sqlListAll(); + + this.getPeer = (pubkey) => this.sqlFindOne({ pubkey: pubkey }); + + this.savePeer = (peer) => this.saveEntity(peer); + + this.removeAll = () => this.sqlDeleteAll(); +} diff --git a/app/lib/dal/sqliteDAL/SourcesDAL.js b/app/lib/dal/sqliteDAL/SourcesDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..4a784ac921c11085078bb7b71e704a4ee5de8095 --- /dev/null +++ b/app/lib/dal/sqliteDAL/SourcesDAL.js @@ -0,0 +1,161 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var co = require('co'); +var _ = require('underscore'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = SourcesDAL; + +function SourcesDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'source'; + this.fields = [ + 'pubkey', + 'type', + 'number', + 'time', + 'fingerprint', + 'amount', + 'block_hash', + 'consumed' + ]; + this.arrays = []; + this.bigintegers = ['amount']; + this.booleans = ['consumed']; + this.pkFields = ['pubkey', 'type', 'number', 'fingerprint', 'amount']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'pubkey VARCHAR(50) NOT NULL,' + + 'type VARCHAR(1) NOT NULL,' + + 'number INTEGER NOT NULL,' + + 'time DATETIME,' + + 'fingerprint VARCHAR(40) NOT NULL,' + + 'amount VARCHAR(50) NOT NULL,' + + 'block_hash VARCHAR(40) NOT NULL,' + + 'consumed BOOLEAN NOT NULL,' + + 'PRIMARY KEY (pubkey,type,number,fingerprint,amount)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_source_pubkey ON source (pubkey);' + + 'CREATE INDEX IF NOT EXISTS idx_source_type ON source (type);' + + 'CREATE INDEX IF NOT EXISTS idx_source_number ON source (number);' + + 'CREATE INDEX IF NOT EXISTS idx_source_fingerprint ON source (fingerprint);' + + 'COMMIT;', []); + }); + + this.getAvailableForPubkey = (pubkey) => this.sqlFind({ + pubkey: pubkey, + consumed: false + }); + + this.getUDSources = (pubkey) => this.sqlFind({ + pubkey: pubkey, + type: 'D' + }); + + this.getSource = (pubkey, type, number) => this.sqlFindOne({ + pubkey: pubkey, + type: type, + number: number + }); + + this.isAvailableSource = (pubkey, type, number, fingerprint, amount) => co(function *() { + let src = yield that.sqlExisting({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount + }); + return src ? !src.consumed : false; + }); + + this.consumeSource = (pubkey, type, number, fingerprint, amount) => co(function *() { + return that.updateEntity({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount + },{ + consumed: true + }); + }); + + this.addSource = (state, pubkey, type, number, fingerprint, amount, block_hash, time) => this.saveEntity({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount, + time: time, + block_hash: block_hash, + consumed: false + }); + + this.unConsumeSource = (type, pubkey, number, fingerprint, amount, time, block_hash) => co(function *() { + let src = yield that.sqlExisting({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount + }); + if (!src) { + return this.saveEntity({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount, + time: time, + block_hash: block_hash, + consumed: false + }); + } else { + return that.updateEntity({ + pubkey: pubkey, + type: type, + number: number, + fingerprint: fingerprint, + amount: amount, + block_hash: block_hash + },{ + consumed: false + }); + } + }); + + this.updateBatchOfSources = (sources) => co(function *() { + let inserts = _.filter(sources, { toConsume: false }); + let updates = _.filter(sources, { toConsume: true }); + let queries = []; + if (inserts.length) { + let insert = that.getInsertHead(); + let values = inserts.map((src) => that.getInsertValue(_.extend(src, { consumed: false }))); + queries.push(insert + '\n' + values.join(',\n') + ';'); + } + if (updates.length) { + let del = that.getDeleteHead(); + let values = that.getDeleteValues(updates); + queries.push(del + '\n' + values + ';'); + } + if (queries.length) { + return that.exec(queries.join('\n')); + } + }); + + this.removeAllSourcesOfBlock = (number) => this.sqlRemoveWhere({ + number: number + }); +} \ No newline at end of file diff --git a/app/lib/dal/sqliteDAL/TxsDAL.js b/app/lib/dal/sqliteDAL/TxsDAL.js new file mode 100644 index 0000000000000000000000000000000000000000..bbfdfcc55262c4b47a2cd28100f4b2b499dc0a95 --- /dev/null +++ b/app/lib/dal/sqliteDAL/TxsDAL.js @@ -0,0 +1,138 @@ +/** + * Created by cgeek on 22/08/15. + */ + +var Q = require('q'); +var co = require('co'); +var AbstractSQLite = require('./AbstractSQLite'); + +module.exports = TxsDAL; + +function TxsDAL(db) { + + "use strict"; + + AbstractSQLite.call(this, db); + + let that = this; + + this.table = 'txs'; + this.fields = [ + 'hash', + 'block_number', + 'version', + 'currency', + 'comment', + 'time', + 'written', + 'removed', + 'inputs', + 'outputs', + 'issuers', + 'signatories', + 'signatures', + 'recipients' + ]; + this.arrays = ['inputs','outputs','issuers','signatories','signatures','recipients']; + this.booleans = ['written','removed']; + this.pkFields = ['hash']; + this.translated = {}; + + this.init = () => co(function *() { + return that.exec('BEGIN;' + + 'CREATE TABLE IF NOT EXISTS ' + that.table + ' (' + + 'hash CHAR(40) NOT NULL,' + + 'block_number INTEGER,' + + 'version INTEGER NOT NULL,' + + 'currency VARCHAR(50) NOT NULL,' + + 'comment VARCHAR(255) NOT NULL,' + + 'time DATETIME,' + + 'inputs TEXT NOT NULL,' + + 'outputs TEXT NOT NULL,' + + 'issuers TEXT NOT NULL,' + + 'signatories TEXT NOT NULL,' + + 'signatures TEXT NOT NULL,' + + 'recipients TEXT NOT NULL,' + + 'written BOOLEAN NOT NULL,' + + 'removed BOOLEAN NOT NULL,' + + 'PRIMARY KEY (hash)' + + ');' + + 'CREATE INDEX IF NOT EXISTS idx_txs_issuers ON txs (issuers);' + + 'CREATE INDEX IF NOT EXISTS idx_txs_written ON txs (written);' + + 'CREATE INDEX IF NOT EXISTS idx_txs_removed ON txs (removed);' + + 'CREATE INDEX IF NOT EXISTS idx_txs_hash ON txs (hash);' + + 'COMMIT;', []); + }); + + this.getAllPending = () => this.sqlFind({ + written: false, + removed: false + }); + + this.getTX = (hash) => this.sqlFindOne({ + hash: hash + }); + + this.removeTX = (hash) => co(function *() { + let tx = yield that.sqlFindOne({ + hash: hash + }); + if (tx) { + tx.removed = true; + return that.saveEntity(tx); + } + return Q(tx); + }); + + this.addLinked = (tx) => { + tx.written = true; + tx.removed = false; + tx.hash = tx.getHash(true); + tx.recipients = tx.outputs.map(function(out) { + return out.match('(.*):')[1]; + }); + return that.saveEntity(tx); + }; + + this.addPending = (tx) => { + tx.written = false; + tx.removed = false; + tx.hash = tx.getHash(true); + tx.recipients = tx.outputs.map(function(out) { + return out.match('(.*):')[1]; + }); + return this.saveEntity(tx); + }; + + this.getLinkedWithIssuer = (pubkey) => this.sqlFind({ + issuers: { $contains: pubkey }, + written: true + }); + + this.getLinkedWithRecipient = (pubkey) => this.sqlFind({ + recipients: { $contains: pubkey }, + written: true + }); + + this.getPendingWithIssuer = (pubkey) => this.sqlFind({ + issuers: { $contains: pubkey }, + written: false + }); + + this.getPendingWithRecipient = (pubkey) => this.sqlFind({ + recipients: { $contains: pubkey }, + written: false + }); + + this.updateBatchOfTxs = (txs) => co(function *() { + let queries = []; + let insert = that.getInsertHead(); + let values = txs.map((cert) => that.getInsertValue(cert)); + if (txs.length) { + queries.push(insert + '\n' + values.join(',\n') + ';'); + } + if (queries.length) { + return that.exec(queries.join('\n')); + } + }); +} \ No newline at end of file diff --git a/app/lib/directory.js b/app/lib/directory.js new file mode 100644 index 0000000000000000000000000000000000000000..4627c1e23d4defc700fdce7bece8cfeaa38cb41d --- /dev/null +++ b/app/lib/directory.js @@ -0,0 +1,27 @@ +"use strict"; + +var opts = require('optimist').argv; +var path = require('path'); + +const DEFAULT_DOMAIN = "ucoin_default"; +const DEFAULT_HOME = ((process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE) + '/.config/ucoin/'); + +module.exports = { + + INSTANCE_NAME: getDomain(opts.mdb), + INSTANCE_HOME: getHomePath(opts.mdb, opts.home), + + getHome: (profile, dir) => getHomePath(profile, dir) +}; + +function getHomePath(profile, dir) { + return path.normalize(getUserHome(dir) + '/') + getDomain(profile); +} + +function getUserHome(dir) { + return dir || DEFAULT_HOME; +} + +function getDomain(profile) { + return profile || DEFAULT_DOMAIN; +} diff --git a/app/lib/entity/block.js b/app/lib/entity/block.js index b5a52fbbf8bb95df682193f7d49e8644471a7843..6e7a8cc3f761c5d10ebf64a8fb635193ae41d818 100644 --- a/app/lib/entity/block.js +++ b/app/lib/entity/block.js @@ -67,11 +67,6 @@ function Block(json) { ].forEach(function(field){ json[field] = parseInt(that[field]) || null; }); - [ - "membersChanges" - ].forEach(function(field){ - json[field] = that[field] || []; - }); [ "identities", "joiners", diff --git a/app/lib/entity/peer.js b/app/lib/entity/peer.js index ebe8c4cd6465f7c8cdcc6f6ad2f79e664de2796b..5ee76f8b2ea318ca0095ffadc1f36e0de449a7e5 100644 --- a/app/lib/entity/peer.js +++ b/app/lib/entity/peer.js @@ -1,4 +1,5 @@ "use strict"; +var Q = require('q'); var _ = require('underscore'); var vucoin = require('vucoin'); var rawer = require('../rawer'); @@ -143,6 +144,17 @@ function Peer(json) { }); }; + that.connectP = (timeout) => { + return Q.Promise(function(resolve, reject){ + vucoin(that.getDns() || that.getIPv6() || that.getIPv4(), that.getPort(), (err, node) => { + if (err) return reject(err); + resolve(node); + }, { + timeout: timeout || 2000 + }); + }); + }; + that.isReachable = function () { return that.getURL() ? true : false; }; diff --git a/app/lib/globalValidator.js b/app/lib/globalValidator.js index eb1e94318f715a9f6e804550956bcae1b516470b..c4b038726fe7df8c94b672b9ede7f07858f83ba3 100644 --- a/app/lib/globalValidator.js +++ b/app/lib/globalValidator.js @@ -221,7 +221,7 @@ function GlobalValidator (conf, dao) { next(null, { hash: 'DA39A3EE5E6B4B0D3255BFEF95601890AFD80709', medianTime: moment.utc().startOf('minute').unix() }); // Valid for root block } else { dao.getBlock(cert.block_number, function (err, basedBlock) { - next(err && 'Certification based on an unexisting block', basedBlock); + next((err || !basedBlock) && 'Certification based on an unexisting block', basedBlock); }); } }, @@ -256,7 +256,7 @@ function GlobalValidator (conf, dao) { var targetId = [cert.block_number, res.target.hash].join('-'); crypto.isValidCertification(selfCert, res.idty.sig, cert.from, cert.sig, targetId, next); } - }, + } ], done); } diff --git a/app/lib/localValidator.js b/app/lib/localValidator.js index 3e3e8a13a3cb3882879e47e7d44e8fe142793187..3e61eecb5b29fb9940473e045f4fa77a00f1a136 100644 --- a/app/lib/localValidator.js +++ b/app/lib/localValidator.js @@ -458,16 +458,17 @@ function hasEachIdentityMatchesANewcomer (block) { // identities and/or joiners, this is another test responsability var pubkeys = []; block.identities.forEach(function(inline){ - var sp = inline.split(':'); - var pubk = sp[0], ts = sp[2], uid = sp[3]; - pubkeys.push(pubk); + let sp = inline.split(':'); + let pubk = sp[0], ts = sp[2], uid = sp[3]; + pubkeys.push([pubk, uid, ts].join('-')); }); var matchCount = 0; var i = 0; while (i < block.joiners.length) { - var sp = block.joiners[i].split(':'); - var pubk = sp[0], ts = sp[3], uid = sp[4]; - if (~pubkeys.indexOf(pubk)) matchCount++; + let sp = block.joiners[i].split(':'); + let pubk = sp[0], ts = sp[4], uid = sp[5]; + let idty = [pubk, uid, ts].join('-'); + if (~pubkeys.indexOf(idty)) matchCount++; i++; } return matchCount != pubkeys.length; diff --git a/app/lib/logger.js b/app/lib/logger.js index e281d307d0fce17b1f044e96bcd63b77bbc42171..9f86d9477b6c09e398adae4c474e0018b778e9dc 100644 --- a/app/lib/logger.js +++ b/app/lib/logger.js @@ -1,15 +1,67 @@ "use strict"; -var log4js = require('log4js'); +var moment = require('moment'); var path = require('path'); +var winston = require('winston'); +var directory = require('../lib/directory'); -log4js.configure(path.join(__dirname, '/../../conf/logs.json'), { reloadSecs: 60 }); +var customLevels = { + levels: { + debug: 0, + info: 1, + warn: 2, + error: 3 + }, + colors: { + debug: 'cyan', + info: 'green', + warn: 'yellow', + error: 'red' + } +}; + +// create the logger +var logger = new (winston.Logger)({ + level: 'debug', + levels: customLevels.levels, + handleExceptions: false, + colors: customLevels.colors, + transports: [ + // setup console logging + new (winston.transports.Console)({ + level: 'error', + levels: customLevels.levels, + handleExceptions: false, + colorize: true, + timestamp: function() { + return moment().format(); + } + }) + ] +}); + +logger.addHomeLogs = (home) => { + logger.add(winston.transports.File, { + level: 'error', + levels: customLevels.levels, + handleExceptions: false, + colorize: true, + tailable: true, + maxsize: 50 * 1024 * 1024, // 50 MB + maxFiles: 3, + //zippedArchive: true, + json: false, + filename: path.join(home, 'ucoin.log'), + timestamp: function() { + return moment().format(); + } + }); +}; + +logger.mute = () => { + logger.remove(winston.transports.Console); +}; /** * Convenience function to get logger directly */ -module.exports = function (name) { - - var logger = log4js.getLogger(name || 'default'); - logger.setLevel('DEBUG'); - return logger; -}; +module.exports = () => logger; diff --git a/app/lib/rawer.js b/app/lib/rawer.js index 08e09d5b06641e3c494e444f9c4a93a1358fbe2e..d95f460f09a048e5f97da51e06dfa5c096ce1ef7 100644 --- a/app/lib/rawer.js +++ b/app/lib/rawer.js @@ -211,8 +211,7 @@ module.exports = new function() { }; function signed (raw, json) { - if (json.signature) - raw += json.signature + '\n'; + raw += json.signature + '\n'; return raw; } -} +}; diff --git a/app/lib/sanitize.js b/app/lib/sanitize.js new file mode 100644 index 0000000000000000000000000000000000000000..5fa468542434dcd20d1149700933f6751a1a3753 --- /dev/null +++ b/app/lib/sanitize.js @@ -0,0 +1,97 @@ +"use strict"; + +let _ = require('underscore'); + +module.exports = function sanitize (json, contract) { + + // Tries to sanitize only if contract is given + if (contract) { + + if (Object.prototype.toString.call(contract) === "[object Array]") { + // Contract is an array + + if (Object.prototype.toString.call(json) !== "[object Array]") { + json = []; + } + + for (let i = 0, len = json.length; i < len; i++) { + json[i] = sanitize(json[i], contract[0]); + } + } else { + // Contract is an object or native type + + // Return type is either a string, a number or an object + if (typeof json != typeof contract) { + try { + // Cast value + json = contract(json); + } catch (e) { + // Cannot be casted: create empty value + json = contract(); + } + } + + let contractFields = _(contract).keys(); + let objectFields = _(json).keys(); + let toDeleteFromObj = _.difference(objectFields, contractFields); + + // Remove unwanted fields + for (let i = 0, len = toDeleteFromObj.length; i < len; i++) { + let field = toDeleteFromObj[i]; + delete json[field]; + } + + // Format wanted fields + for (let i = 0, len = contractFields.length; i < len; i++) { + let prop = contractFields[i]; + let propType = contract[prop]; + let t = ""; + if (propType.name) { + t = propType.name; + } else if (propType.length != undefined) { + t = 'Array'; + } else { + t = 'Object'; + } + // Test json member type + let tjson = typeof json[prop]; + if (~['Array', 'Object'].indexOf(t)) { + if (tjson == 'object') { + tjson = json[prop].length == undefined ? 'Object' : 'Array'; + } + } + // Check coherence & alter member if needed + if (!_(json[prop]).isNull() && t.toLowerCase() != tjson.toLowerCase()) { + try { + if (t == "String" || t == "Number") { + let s = json[prop] == undefined ? '' : json[prop]; + eval('json[prop] = new ' + t + '(' + s + ').valueOf()'); + } + else { + eval('json[prop] = new ' + t + '()'); + } + } catch (ex) { + eval('json[prop] = new ' + t + '()'); + } + } + // Arrays + if (t == 'Array') { + let subt = propType[0]; + for (let j = 0, len2 = json[prop].length; j < len2; j++) { + if (subt == "String" || subt == "Number") { + eval('item = new ' + t + '(' + (json[prop] + '') + ').valueOf()'); + } + else { + json[prop][j] = sanitize(json[prop][j], subt); + } + } + } + // Recursivity + if (t == 'Object') { + json[prop] = sanitize(json[prop], contract[prop]); + } + } + } + } + return json; +}; diff --git a/app/lib/streams/bma.js b/app/lib/streams/bma.js index fb7406bf74ecfba3183ad2037c93e9690d9a5b54..0835e476eaf9f9615c8aa49dbf81fbaa75b0a967 100644 --- a/app/lib/streams/bma.js +++ b/app/lib/streams/bma.js @@ -1,18 +1,20 @@ +var _ = require('underscore'); var http = require('http'); var express = require('express'); -var log4js = require('log4js'); var co = require('co'); var Q = require('q'); var cors = require('express-cors'); var es = require('event-stream'); - +var morgan = require('morgan'); +var constants = require('../../lib/constants'); +var dtos = require('../../lib/streams/dtos'); +var sanitize = require('../../lib/sanitize'); var logger = require('../../lib/logger')('bma'); module.exports = function(server, interfaces, httpLogs) { "use strict"; - var httpLogger = log4js.getLogger(); var app = express(); if (!interfaces) { @@ -30,14 +32,14 @@ module.exports = function(server, interfaces, httpLogs) { // all environments if (httpLogs) { - app.use(log4js.connectLogger(httpLogger, { - format: '\x1b[90m:remote-addr - :method :url HTTP/:http-version :status :res[content-length] - :response-time ms\x1b[0m' + app.use(morgan('\x1b[90m:remote-addr - :method :url HTTP/:http-version :status :res[content-length] - :response-time ms\x1b[0m', { + stream: { + write: function(message){ + message && logger.info(message.replace(/\n$/,'')); + } + } })); } - //app.use(function(req, res, next) { - // console.log('\x1b[90mDEBUG URL - %s\x1b[0m', req.url); - // next(); - //}); app.use(cors({ allowedOrigins: [ @@ -57,71 +59,99 @@ module.exports = function(server, interfaces, httpLogs) { app.use(express.errorHandler()); } - var node = require('../../controllers/node')(server); - answerForGet('/node/summary', node.summary); - - var blockchain = require('../../controllers/blockchain')(server); - answerForGet( '/blockchain/parameters', blockchain.parameters); - answerForPost('/blockchain/membership', blockchain.parseMembership); - answerForGet( '/blockchain/memberships/:search', blockchain.memberships); - answerForPost('/blockchain/block', blockchain.parseBlock); - answerForGet( '/blockchain/block/:number', blockchain.promoted); - answerForGet( '/blockchain/blocks/:count/:from', blockchain.blocks); - answerForGet( '/blockchain/current', blockchain.current); - answerForGet( '/blockchain/hardship/:search', blockchain.hardship); - answerForGet( '/blockchain/with/newcomers', blockchain.with.newcomers); - answerForGet( '/blockchain/with/certs', blockchain.with.certs); - answerForGet( '/blockchain/with/joiners', blockchain.with.joiners); - answerForGet( '/blockchain/with/actives', blockchain.with.actives); - answerForGet( '/blockchain/with/leavers', blockchain.with.leavers); - answerForGet( '/blockchain/with/excluded', blockchain.with.excluded); - answerForGet( '/blockchain/with/ud', blockchain.with.ud); - answerForGet( '/blockchain/with/tx', blockchain.with.tx); - answerForGet( '/blockchain/branches', blockchain.branches); - - var net = require('../../controllers/network')(server, server.conf); - answerForGet( '/network/peering', net.peer); - answerForGet( '/network/peering/peers', net.peersGet); - answerForPost('/network/peering/peers', net.peersPost); - answerForGet('/network/peers', net.peers); - - var wot = require('../../controllers/wot')(server); - answerForPost('/wot/add', wot.add); - answerForPost('/wot/revoke', wot.revoke); - answerForGet( '/wot/lookup/:search', wot.lookup); - answerForGet( '/wot/members', wot.members); - answerForGet( '/wot/requirements/:pubkey', wot.requirements); - answerForGet( '/wot/certifiers-of/:search', wot.certifiersOf); - answerForGet( '/wot/certified-by/:search', wot.certifiedBy); - answerForGet( '/wot/identity-of/:search', wot.identityOf); - + var node = require('../../controllers/node')(server); + var blockchain = require('../../controllers/blockchain')(server); + var net = require('../../controllers/network')(server, server.conf); + var wot = require('../../controllers/wot')(server); var transactions = require('../../controllers/transactions')(server); var dividend = require('../../controllers/uds')(server); - answerForPost('/tx/process', transactions.parseTransaction); - answerForGet( '/tx/sources/:pubkey', transactions.getSources); - answerForGet( '/tx/history/:pubkey', transactions.getHistory); - answerForGet( '/tx/history/:pubkey/blocks/:from/:to', transactions.getHistoryBetweenBlocks); - answerForGet( '/tx/history/:pubkey/times/:from/:to', transactions.getHistoryBetweenTimes); - answerForGet( '/tx/history/:pubkey/pending', transactions.getPendingForPubkey); - answerForGet( '/tx/pending', transactions.getPending); - answerForGet( '/ud/history/:pubkey', dividend.getHistory); - answerForGet( '/ud/history/:pubkey/blocks/:from/:to', dividend.getHistoryBetweenBlocks); - answerForGet( '/ud/history/:pubkey/times/:from/:to', dividend.getHistoryBetweenTimes); - - function answerForGet(uri, callback) { - app.get(uri, function(req, res) { - res.set('Access-Control-Allow-Origin', '*'); - callback(req, res); - }); + answerForGetP( '/node/summary', node.summary, dtos.Summary); + answerForGetP( '/blockchain/parameters', blockchain.parameters, dtos.Parameters); + answerForPostP( '/blockchain/membership', blockchain.parseMembership, dtos.Membership); + answerForGetP( '/blockchain/memberships/:search', blockchain.memberships, dtos.Memberships); + answerForPostP( '/blockchain/block', blockchain.parseBlock, dtos.Block); + answerForGetP( '/blockchain/block/:number', blockchain.promoted, dtos.Block); + answerForGetP( '/blockchain/blocks/:count/:from', blockchain.blocks, dtos.Blocks); + answerForGetP( '/blockchain/current', blockchain.current, dtos.Block); + answerForGetP( '/blockchain/hardship/:search', blockchain.hardship, dtos.Hardship); + answerForGetP( '/blockchain/with/newcomers', blockchain.with.newcomers, dtos.Stat); + answerForGetP( '/blockchain/with/certs', blockchain.with.certs, dtos.Stat); + answerForGetP( '/blockchain/with/joiners', blockchain.with.joiners, dtos.Stat); + answerForGetP( '/blockchain/with/actives', blockchain.with.actives, dtos.Stat); + answerForGetP( '/blockchain/with/leavers', blockchain.with.leavers, dtos.Stat); + answerForGetP( '/blockchain/with/excluded', blockchain.with.excluded, dtos.Stat); + answerForGetP( '/blockchain/with/ud', blockchain.with.ud, dtos.Stat); + answerForGetP( '/blockchain/with/tx', blockchain.with.tx, dtos.Stat); + answerForGetP( '/blockchain/branches', blockchain.branches, dtos.Branches); + answerForGetP( '/network/peering', net.peer, dtos.Peer); + answerForGetP( '/network/peering/peers', net.peersGet, dtos.Merkle); + answerForPostP( '/network/peering/peers', net.peersPost, dtos.Peer); + answerForGetP( '/network/peers', net.peers, dtos.Peers); + answerForPostP( '/wot/add', wot.add, dtos.Identity); + answerForPostP( '/wot/revoke', wot.revoke, dtos.Result); + answerForGetP( '/wot/lookup/:search', wot.lookup, dtos.Lookup); + answerForGetP( '/wot/members', wot.members, dtos.Members); + answerForGetP( '/wot/requirements/:search', wot.requirements, dtos.Requirements); + answerForGetP( '/wot/certifiers-of/:search', wot.certifiersOf, dtos.Certifications); + answerForGetP( '/wot/certified-by/:search', wot.certifiedBy, dtos.Certifications); + answerForGetP( '/wot/identity-of/:search', wot.identityOf, dtos.SimpleIdentity); + answerForPostP( '/tx/process', transactions.parseTransaction, dtos.Transaction); + answerForGetP( '/tx/sources/:pubkey', transactions.getSources, dtos.Sources); + answerForGetP( '/tx/history/:pubkey', transactions.getHistory, dtos.TxHistory); + answerForGetP( '/tx/history/:pubkey/blocks/:from/:to', transactions.getHistoryBetweenBlocks, dtos.TxHistory); + answerForGetP( '/tx/history/:pubkey/times/:from/:to', transactions.getHistoryBetweenTimes, dtos.TxHistory); + answerForGetP( '/tx/history/:pubkey/pending', transactions.getPendingForPubkey, dtos.TxHistory); + answerForGetP( '/tx/pending', transactions.getPending, dtos.TxPending); + answerForGetP( '/ud/history/:pubkey', dividend.getHistory, dtos.UDHistory); + answerForGetP( '/ud/history/:pubkey/blocks/:from/:to', dividend.getHistoryBetweenBlocks, dtos.UDHistory); + answerForGetP( '/ud/history/:pubkey/times/:from/:to', dividend.getHistoryBetweenTimes, dtos.UDHistory); + + function answerForGetP(uri, promiseFunc, dtoContract) { + handleRequest(app.get.bind(app), uri, promiseFunc, dtoContract); + } + + function answerForPostP(uri, promiseFunc, dtoContract) { + handleRequest(app.post.bind(app), uri, promiseFunc, dtoContract); } - function answerForPost(uri, callback) { - app.post(uri, function(req, res) { + function handleRequest(method, uri, promiseFunc, dtoContract) { + method(uri, function(req, res) { res.set('Access-Control-Allow-Origin', '*'); - callback(req, res); + res.type('application/json'); + co(function *() { + try { + let result = yield promiseFunc(req); + // Ensure of the answer format + result = sanitize(result, dtoContract); + // HTTP answer + res.send(200, JSON.stringify(result, null, " ")); + } catch (e) { + let error = getResultingError(e); + // HTTP error + res.send(error.httpCode, JSON.stringify(error.uerr, null, " ")); + } + }); }); } + function getResultingError(e) { + // Default is 500 unknown error + let error = constants.ERRORS.UNKNOWN; + if (e) { + // Print eventual stack trace + e.stack && logger.error(e.stack); + e.message && logger.warn(e.message); + // BusinessException + if (e.uerr) { + error = e; + } else { + error = _.clone(constants.ERRORS.UNHANDLED); + error.uerr.message = e.message || error.uerr.message; + } + } + return error; + } + var httpServers = []; return co(function *() { diff --git a/app/lib/streams/dtos.js b/app/lib/streams/dtos.js new file mode 100644 index 0000000000000000000000000000000000000000..0a5fcbb585a93978dc7213ae606400f4f2b19c5a --- /dev/null +++ b/app/lib/streams/dtos.js @@ -0,0 +1,328 @@ +"use strict"; + +let dtos; + +module.exports = dtos = {}; + +dtos.Summary = { + ucoin: { + "software": String, + "version": String, + "forkWindowSize": Number + } +}; + +dtos.Parameters = { + currency: String, + c: Number, + dt: Number, + ud0: Number, + sigDelay: Number, + sigValidity: Number, + sigQty: Number, + sigWoT: Number, + msValidity: Number, + stepMax: Number, + medianTimeBlocks: Number, + avgGenTime: Number, + dtDiffEval: Number, + blocksRot: Number, + percentRot: Number +}; + +dtos.Membership = { + "signature": String, + "membership": { + "version": Number, + "currency": String, + "issuer": String, + "membership": String, + "date": Number, + "sigDate": Number, + "raw": String + } +}; + +dtos.Memberships = { + "pubkey": String, + "uid": String, + "sigDate": Number, + "memberships": [ + { + "version": Number, + "currency": String, + "membership": String, + "blockNumber": Number, + "blockHash": String + } + ] +}; + +dtos.TransactionOfBlock = { + "version": Number, + "currency": String, + "comment": String, + "signatures": [String], + "outputs": [String], + "inputs": [String], + "signatories": [String], + "block_number": Number, + "time": Number, + "issuers": [String] +}; + +dtos.Block = { + "version": Number, + "currency": String, + "nonce": Number, + "number": Number, + "issuer": String, + "parameters": String, + "membersCount": Number, + "monetaryMass": Number, + "powMin": Number, + "time": Number, + "medianTime": Number, + "dividend": Number, + "hash": String, + "previousHash": String, + "previousIssuer": String, + "identities": [String], + "certifications": [String], + "joiners": [String], + "actives": [String], + "leavers": [String], + "excluded": [String], + "transactions": [dtos.TransactionOfBlock], + "signature": String, + "raw": String +}; + +dtos.Hardship = { + "block": Number, + "level": Number +}; + +dtos.Blocks = [dtos.Block]; + +dtos.Stat = { + "result": { + "blocks": [Number] + } +}; + +dtos.Branches = { + "blocks": [dtos.Block] +}; + +dtos.Peer = { + "version": Number, + "currency": String, + "pubkey": String, + "block": String, + "endpoints": [String], + "signature": String, + "raw": String +}; + +dtos.DBPeer = { + "version": Number, + "currency": String, + "pubkey": String, + "block": String, + "status": String, + "first_down": Number, + "last_try": Number, + "endpoints": [String], + "signature": String +}; + +dtos.Peers = { + "peers": [dtos.DBPeer] +}; + +dtos.Merkle = { + "depth": Number, + "nodesCount": Number, + "leavesCount": Number, + "root": String, + "leaves": [String], + "leaf": { + "hash": String, + "value": {} + } +}; + +dtos.Other = { + "pubkey": String, + "meta": { + "block_number": Number + }, + "uids": [String], + "isMember": Boolean, + "wasMember": Boolean, + "signature": String +}; + +dtos.UID = { + "uid": String, + "meta": { + "timestamp": Number + }, + "self": String, + "others": [dtos.Other] +}; + +dtos.Signed = { + "uid": String, + "pubkey": String, + "meta": { + "timestamp": Number + }, + "isMember": Boolean, + "wasMember": Boolean, + "signature": String +}; + +dtos.Identity = { + "pubkey": String, + "uids": [dtos.UID], + "signed": [dtos.Signed] +}; + +dtos.Result = { + "result": Boolean +}; + +dtos.Lookup = { + "partial": Boolean, + "results": [dtos.Identity] +}; + +dtos.Members = { + "results": [{ + pubkey: String, + uid: String + }] +}; + +dtos.RequirementsCert = { + from: String, + to: String, + expiresIn: Number +}; + +dtos.Requirements = { + "identities": [{ + pubkey: String, + uid: String, + meta: { + timestamp: Number + }, + outdistanced: [String], + certifications: [dtos.RequirementsCert], + membershipPendingExpiresIn: Number, + membershipExpiresIn: Number + }] +}; + +dtos.Certification = { + "pubkey": String, + "uid": String, + "isMember": Boolean, + "wasMember": Boolean, + "cert_time": { + "block": Number, + "medianTime": Number + }, + "sigDate": Number, + "written": { + "number": Number, + "hash": String + }, + "signature": String +}; + +dtos.Certifications = { + "pubkey": String, + "uid": String, + "sigDate": Number, + "isMember": Boolean, + "certifications": [dtos.Certification] +}; + +dtos.SimpleIdentity = { + "pubkey": String, + "uid": String, + "sigDate": Number +}; + +dtos.Transaction = { + "version": Number, + "currency": String, + "issuers": [String], + "inputs": [String], + "outputs": [String], + "comment": String, + "signatures": [String], + "raw": String, + "hash": String +}; + +dtos.Source = { + "pubkey": String, + "type": String, + "number": Number, + "fingerprint": String, + "amount": Number +}; + +dtos.Sources = { + "currency": String, + "pubkey": String, + "sources": [dtos.Source] +}; + +dtos.TxOfHistory = { + "version": Number, + "issuers": [String], + "inputs": [String], + "outputs": [String], + "comment": String, + "signatures": [String], + "hash": String, + "block_number": Number, + "time": Number +}; + +dtos.TxHistory = { + "currency": String, + "pubkey": String, + "history": { + "sent": [dtos.TxOfHistory], + "received": [dtos.TxOfHistory], + "sending": [dtos.TxOfHistory], + "receiving": [dtos.TxOfHistory], + "pending": [dtos.TxOfHistory] + } +}; + +dtos.TxPending = { + "currency": String, + "pending": [dtos.Transaction] +}; + +dtos.UD = { + "block_number": Number, + "consumed": Boolean, + "time": Number, + "amount": Number +}; + +dtos.UDHistory = { + "currency": String, + "pubkey": String, + "history": { + "history": [dtos.UD] + } +}; diff --git a/app/lib/streams/parsers/doc/GenericParser.js b/app/lib/streams/parsers/doc/GenericParser.js index 4246fa7a7dec4f0d94f4026f6fcb75bc4ca400e7..58f355f5fa95355e74366d3f100e4a69edb83b12 100644 --- a/app/lib/streams/parsers/doc/GenericParser.js +++ b/app/lib/streams/parsers/doc/GenericParser.js @@ -2,6 +2,7 @@ var sha1 = require('sha1'); var util = require('util'); var stream = require('stream'); +var constants = require('../../../constants'); var simpleLineExtract = require('../../../simpleLineExtract'); var multipleLinesExtract = require('../../../multipleLinesExtract'); @@ -51,7 +52,7 @@ function GenericParser (captures, multipleLinesFields, rawerFunc, onError) { if (!error) { var raw = that.rawerFunc(obj); if (sha1(str) != sha1(raw)) - error = 'Document has unkown fields or wrong line ending format'; + error = constants.ERRORS.WRONG_DOCUMENT; if (error) { // console.log(error); // console.log('-----------------'); @@ -60,10 +61,9 @@ function GenericParser (captures, multipleLinesFields, rawerFunc, onError) { // console.log('-----------------'); } } - if (typeof done == 'function') - done(error, obj); - else - return obj; + if (typeof done == 'function') return done(error, obj); + if (error) throw constants.ERRORS.WRONG_DOCUMENT; + return obj; } this._clean = function (obj) { diff --git a/app/lib/streams/parsers/doc/index.js b/app/lib/streams/parsers/doc/index.js index 3856f121dcadc9559b009215c87a8d5807144ac7..6ed2dfa2cdca95adbd9f3770065d9985018b2075 100644 --- a/app/lib/streams/parsers/doc/index.js +++ b/app/lib/streams/parsers/doc/index.js @@ -1,14 +1,10 @@ "use strict"; module.exports = { - parseIdentity: instanciate.bind(instanciate, require('./identity')), - parseRevocation: instanciate.bind(instanciate, require('./revocation')), - parseTransaction: instanciate.bind(instanciate, require('./transaction')), - parsePeer: instanciate.bind(instanciate, require('./peer')), - parseMembership: instanciate.bind(instanciate, require('./membership')), - parseBlock: instanciate.bind(instanciate, require('./block')) -}; - -function instanciate (constructorFunc, onError) { - return new constructorFunc(onError); + parseIdentity: (new (require('./identity'))), + parseRevocation: (new (require('./revocation'))), + parseTransaction: (new (require('./transaction'))), + parsePeer: (new (require('./peer'))), + parseMembership: (new (require('./membership'))), + parseBlock: (new (require('./block'))) }; diff --git a/app/lib/streams/parsers/doc/membership.js b/app/lib/streams/parsers/doc/membership.js index ae04ef58f2cbaa71229ff1659492573319fbd56c..43e01cbac66a3176525f1acc60c2df71d092f64b 100644 --- a/app/lib/streams/parsers/doc/membership.js +++ b/app/lib/streams/parsers/doc/membership.js @@ -58,7 +58,7 @@ function MembershipParser (onError) { err = {code: codes.BAD_ISSUER, message: "Incorrect issuer field"}; } if(!err){ - if(obj.membership && !obj.membership.match(/^(IN|OUT)$/)) + if(!(obj.membership || "").match(/^(IN|OUT)$/)) err = {code: codes.BAD_MEMBERSHIP, message: "Incorrect Membership field: must be either IN or OUT"}; } if(!err){ diff --git a/app/lib/streams/parsers/http2raw.js b/app/lib/streams/parsers/http2raw.js index 83567d6b9ea1a69b06b5831ce092e3855079c74e..a9fcfdc7fc83c2eb860bcfc3ee349a7056942a37 100644 --- a/app/lib/streams/parsers/http2raw.js +++ b/app/lib/streams/parsers/http2raw.js @@ -1,146 +1,84 @@ "use strict"; -var stream = require('stream'); -var util = require('util'); +var constants = require('../../constants'); module.exports = { - identity: instanciate.bind(null, Http2RawIdentity), - revocation: instanciate.bind(null, Http2RawRevocation), - transaction: instanciate.bind(null, Http2RawTransaction), - peer: instanciate.bind(null, Http2RawPeer), - membership: instanciate.bind(null, Http2RawMembership), - block: instanciate.bind(null, Http2RawBlock) + identity: Http2RawIdentity, + revocation: Http2RawRevocation, + transaction: Http2RawTransaction, + peer: Http2RawPeer, + membership: Http2RawMembership, + block: Http2RawBlock }; -function instanciate (constructorFunc, req, onError) { - return new constructorFunc(req, onError); -}; - -function Http2RawIdentity (req, onError) { - - stream.Readable.call(this); - - this._read = function () { - if(!req.body || !req.body.pubkey){ - onError('Parameter `pubkey` is required'); - } - else if(!req.body || !req.body.self){ - onError('Parameter `self` is required'); - } - else { - var pubkey = req.body.pubkey; - // Add trailing LF to pubkey - if (!req.body.pubkey.match(/\n$/)) { - pubkey += '\n'; - } - var selfCert = req.body.self; - // Add trailing LF to self - if (!req.body.self.match(/\n$/)) { - selfCert += '\n'; - } - var raw = pubkey + selfCert + (req.body.other || ''); - this.push(raw); - } - this.push(null); +function Http2RawIdentity (req) { + if(!req.body || !req.body.pubkey){ + throw constants.ERRORS.HTTP_PARAM_PUBKEY_REQUIRED; + } + if(!req.body || !req.body.self){ + throw constants.ERRORS.HTTP_PARAM_SELF_REQUIRED; } + let pubkey = req.body.pubkey; + // Add trailing LF to pubkey + if (!req.body.pubkey.match(/\n$/)) { + pubkey += '\n'; + } + let selfCert = req.body.self; + // Add trailing LF to self + if (!req.body.self.match(/\n$/)) { + selfCert += '\n'; + } + return pubkey + selfCert + (req.body.other || ''); } -function Http2RawRevocation (req, onError) { - - stream.Readable.call(this); - - this._read = function () { - if(!req.body || !req.body.pubkey){ - onError('Parameter `pubkey` is required'); - } - else if(!req.body || !req.body.self){ - onError('Parameter `self` is required'); - } - else if(!req.body || !req.body.sig){ - onError('Parameter `sig` is required'); - } - else { - var pubkey = req.body.pubkey; - // Add trailing LF to pubkey - if (!req.body.pubkey.match(/\n$/)) { - pubkey += '\n'; - } - var selfCert = req.body.self; - // Add trailing LF to self - if (!req.body.self.match(/\n$/)) { - selfCert += '\n'; - } - var revocationLine = 'META:REVOKE\n'; - var raw = pubkey + selfCert + revocationLine + (req.body.sig || ''); - this.push(raw); - } - this.push(null); +function Http2RawRevocation (req) { + if(!req.body || !req.body.pubkey){ + throw constants.ERRORS.HTTP_PARAM_PUBKEY_REQUIRED; + } + else if(!req.body || !req.body.self){ + throw constants.ERRORS.HTTP_PARAM_SELF_REQUIRED; } + else if(!req.body || !req.body.sig){ + throw constants.ERRORS.HTTP_PARAM_SIG_REQUIRED; + } + var pubkey = req.body.pubkey; + // Add trailing LF to pubkey + if (!req.body.pubkey.match(/\n$/)) { + pubkey += '\n'; + } + var selfCert = req.body.self; + // Add trailing LF to self + if (!req.body.self.match(/\n$/)) { + selfCert += '\n'; + } + var revocationLine = 'META:REVOKE\n'; + return pubkey + selfCert + revocationLine + (req.body.sig || ''); } -function Http2RawTransaction (req, onError) { - - stream.Readable.call(this); - - this._read = function () { - if(!(req.body && req.body.transaction)){ - onError('Requires a transaction'); - } - else { - this.push(req.body.transaction); - } - this.push(null); +function Http2RawTransaction (req) { + if(!(req.body && req.body.transaction)){ + throw constants.ERRORS.HTTP_PARAM_TX_REQUIRED; } + return req.body.transaction; } -function Http2RawPeer (req, onError) { - - stream.Readable.call(this); - - this._read = function () { - if(!(req.body && req.body.peer)){ - onError('Requires a peer'); - } - else { - this.push(req.body.peer); - } - this.push(null); +function Http2RawPeer (req) { + if(!(req.body && req.body.peer)){ + throw constants.ERRORS.HTTP_PARAM_PEER_REQUIRED; } + return req.body.peer; } -function Http2RawMembership (req, onError) { - - stream.Readable.call(this); - - this._read = function () { - if(!(req.body && req.body.membership)){ - onError('Requires a membership'); - } - else { - this.push(req.body.membership); - } - this.push(null); +function Http2RawMembership (req) { + if(!(req.body && req.body.membership)){ + throw constants.ERRORS.HTTP_PARAM_MEMBERSHIP_REQUIRED; } + return req.body.membership; } -function Http2RawBlock (req, onError) { - - stream.Readable.call(this); - - this._read = function () { - if(!(req.body && req.body.block)){ - onError('Requires a block'); - } - else { - this.push(req.body.block); - } - this.push(null); +function Http2RawBlock (req) { + if(!(req.body && req.body.block)){ + throw constants.ERRORS.HTTP_PARAM_BLOCK_REQUIRED; } + return req.body.block; } - -util.inherits(Http2RawIdentity, stream.Readable); -util.inherits(Http2RawRevocation, stream.Readable); -util.inherits(Http2RawTransaction, stream.Readable); -util.inherits(Http2RawPeer, stream.Readable); -util.inherits(Http2RawMembership, stream.Readable); -util.inherits(Http2RawBlock, stream.Readable); diff --git a/app/lib/streams/router.js b/app/lib/streams/router.js index 5ed1176b7f22f9d325b1a84d2c24e251c3d3ca51..bb37988ddc164645de1c7c8adde050dd765412b5 100644 --- a/app/lib/streams/router.js +++ b/app/lib/streams/router.js @@ -1,8 +1,8 @@ "use strict"; +var co = require('co'); var _ = require('underscore'); var async = require('async'); -var sha1 = require('sha1'); var util = require('util'); var stream = require('stream'); var Peer = require('../entity/peer'); @@ -75,41 +75,21 @@ function Router (serverPubkey, conf, dal) { function getValidUpPeers (without) { return function (done) { - var members = []; - var nonmembers = []; - async.waterfall([ - function(next) { - dal.getCurrentBlockOrNull(next); - }, - function (current, next) { - dal.getRandomlyUPsWithout(without, next); // Peers with status UP - }, - function (peers, next) { - async.forEachSeries(peers, function (p, callback) { - async.waterfall([ - function (next) { - dal.isMember(p.pubkey, next); - }, - function (isMember, next) { - isMember ? members.push(p) : nonmembers.push(p); - next(); - } - ], callback); - }, next); - }, - function (next) { - logger.info('New document to send to %s member and %s non-member peers', members.length, nonmembers.length); - async.parallel({ - members: async.apply(choose4in, members), // Choose up to 4 member peers - nonmembers: async.apply(choose4in, nonmembers) // Choose up to 4 non-member peers - }, next); - }, - function (res, next) { - var chosen = res.members.concat(res.nonmembers); - next(null, chosen); + return co(function *() { + let members = []; + let nonmembers = []; + let peers = yield dal.getRandomlyUPsWithout(without); // Peers with status UP + for (let i = 0, len = peers.length; i < len; i++) { + let p = peers[i]; + let isMember = yield dal.isMember(p.pubkey); + isMember ? members.push(p) : nonmembers.push(p); } - ], done); - } + members = chooseXin(members, constants.NETWORK.MAX_MEMBERS_TO_FORWARD_TO); + nonmembers = chooseXin(nonmembers, constants.NETWORK.MAX_NON_MEMBERS_TO_FORWARD_TO); + return members.concat(nonmembers); + }) + .then(_.partial(done, null)).catch(done); + }; } /** @@ -121,25 +101,21 @@ function Router (serverPubkey, conf, dal) { done(null, []); } else { dal.getPeer(to) - .then(function(peer){ - done(null, [peer]); - }) - .catch(function(err){ - done(err); - }); + .then((peer) => done(null, [peer])) + .catch((err) => done(err)); } }; } - function choose4in (peers, done) { + function chooseXin (peers, max) { var chosen = []; var nbPeers = peers.length; - for (var i = 0; i < Math.min(nbPeers, 4); i++) { - var randIndex = Math.max(Math.floor(Math.random()*10) - (10 - nbPeers) - i, 0); + for (var i = 0; i < Math.min(nbPeers, max); i++) { + var randIndex = Math.max(Math.floor(Math.random() * 10) - (10 - nbPeers) - i, 0); chosen.push(peers[randIndex]); peers.splice(randIndex, 1); } - done(null, chosen); + return chosen; } } diff --git a/app/lib/sync.js b/app/lib/sync.js index 11c87d51c256b10207625350ff160e9f6cad81f5..c5c655e612a8faefeb0679e53a43869df95ba674 100644 --- a/app/lib/sync.js +++ b/app/lib/sync.js @@ -13,20 +13,17 @@ var constants = require('../lib/constants'); var Peer = require('../lib/entity/peer'); var multimeter = require('multimeter'); -var CONST_BLOCKS_CHUNK = 500; -var EVAL_REMAINING_INTERVAL = 1000; +const CONST_BLOCKS_CHUNK = 500; +const EVAL_REMAINING_INTERVAL = 1000; +const COMPUTE_SPEED_ON_COUNT_CHUNKS = 8; module.exports = function Synchroniser (server, host, port, conf, interactive) { - var speed = 0, syncStart = new Date(), blocksApplied = 0; + var speed = 0, syncStart = new Date(), times = [syncStart], blocksApplied = 0; var watcher = interactive ? new MultimeterWatcher() : new LoggerWatcher(); if (interactive) { - require('log4js').configure({ - "appenders": [ - //{ category: "db1", type: "console" } - ] - }); + logger.mute(); } // Services @@ -88,7 +85,6 @@ module.exports = function Synchroniser (server, host, port, conf, interactive) { co(function *() { // Download blocks and save them watcher.downloadPercent(Math.floor(chunk[0] / remoteNumber * 100)); - logger.info('Blocks #%s to #%s...', chunk[0], chunk[1]); var blocks = yield Q.nfcall(node.blockchain.blocks, chunk[1] - chunk[0] + 1, chunk[0]); watcher.downloadPercent(Math.floor(chunk[1] / remoteNumber * 100)); chunk[2] = blocks; @@ -104,7 +100,17 @@ module.exports = function Synchroniser (server, host, port, conf, interactive) { function incrementBlocks(increment) { blocksApplied += increment; - speed = blocksApplied / Math.round(Math.max((new Date() - syncStart) / 1000, 1)); + let now = new Date(); + if (times.length == COMPUTE_SPEED_ON_COUNT_CHUNKS) { + times.splice(0, 1); + } + times.push(now); + let duration = times.reduce(function(sum, t, index) { + return index == 0 ? sum : (sum + (times[index] - times[index - 1])); + }, 0); + speed = (CONST_BLOCKS_CHUNK * (times.length - 1)) / Math.round(Math.max(duration / 1000, 1)); + // Reset chrono + syncStart = new Date(); if (watcher.appliedPercent() != Math.floor((blocksApplied + localNumber) / remoteNumber * 100)) { watcher.appliedPercent(Math.floor((blocksApplied + localNumber) / remoteNumber * 100)); } @@ -117,7 +123,6 @@ module.exports = function Synchroniser (server, host, port, conf, interactive) { let chunk = yield toApplyNoCautious[i].promise; let blocks = chunk[2]; blocks = _.sortBy(blocks, 'number'); - logger.info("Applying blocks #%s to #%s...", blocks[0].number, blocks[blocks.length - 1].number); if (cautious) { for (let j = 0, len = blocks.length; j < len; j++) { yield applyGivenBlock(cautious, remoteNumber)(blocks[j]); @@ -126,6 +131,14 @@ module.exports = function Synchroniser (server, host, port, conf, interactive) { } else { yield BlockchainService.saveBlocksInMainBranch(blocks, remoteNumber); incrementBlocks(blocks.length); + // Free memory + if (i >= 0 && i < toApplyNoCautious.length - 1) { + blocks.splice(0, blocks.length); + chunk.splice(0, chunk.length); + } + if (i - 1 >= 0) { + delete toApplyNoCautious[i - 1]; + } } } @@ -228,7 +241,7 @@ module.exports = function Synchroniser (server, host, port, conf, interactive) { if (watcher.appliedPercent() != Math.floor(block.number / remoteCurrentNumber * 100)) { watcher.appliedPercent(Math.floor(block.number / remoteCurrentNumber * 100)); } - return BlockchainService.submitBlock(_.omit(block, '$loki', 'meta'), cautious); + return BlockchainService.submitBlock(block, cautious); }; } @@ -358,19 +371,30 @@ function MultimeterWatcher() { function LoggerWatcher() { - var downPct = 0, appliedPct = 0; + var downPct = 0, appliedPct = 0, lastMsg; + + this.showProgress = function() { + logger.info('Downloaded %s%, Applied %s%', downPct, appliedPct); + }; this.writeStatus = function(str) { - logger.info(str); + if (str != lastMsg) { + lastMsg = str; + logger.info(str); + } }; this.downloadPercent = function(pct) { + let changed = pct != downPct; downPct = pct; + if (changed) this.showProgress(); return downPct; }; this.appliedPercent = function(pct) { + let changed = pct !== undefined && pct != appliedPct; appliedPct = pct; + if (changed) this.showProgress(); return appliedPct; }; diff --git a/app/service/BlockchainService.js b/app/service/BlockchainService.js index 7bcd2fbb64249d4fffd20168bf470b9b165a3aef..966107e07ab97752b8c5fc1419aa5e6a30fe87da 100644 --- a/app/service/BlockchainService.js +++ b/app/service/BlockchainService.js @@ -9,7 +9,6 @@ var sha1 = require('sha1'); var moment = require('moment'); var inquirer = require('inquirer'); var childProcess = require('child_process'); -var usage = require('usage'); var base58 = require('../lib/base58'); var signature = require('../lib/signature'); var constants = require('../lib/constants'); @@ -82,17 +81,11 @@ function BlockchainService (conf, mainDAL, pair) { this.current = (done) => mainDAL.getCurrentBlockOrNull(done); - this.promoted = (number, done) => - co(function *() { - let bb = yield mainDAL.getPromoted(number); - if (!bb) throw 'Block not found'; - done && done(null, bb); - return bb; - }) - .catch(function(err) { - done && done(err); - throw err; - }); + this.promoted = (number) => co(function *() { + let bb = yield mainDAL.getPromoted(number); + if (!bb) throw constants.ERRORS.BLOCK_NOT_FOUND; + return bb; + }); this.checkBlock = function(block) { return mainContext.checkBlock(block); @@ -730,66 +723,80 @@ function BlockchainService (conf, mainDAL, pair) { }); } - this.requirementsOfIdentity = function(idty) { - return Q.all([ - that.currentDal.getMembershipsForIssuer(idty.pubkey), - that.currentDal.getCurrent(), - that.currentDal.getMembers() - .then(function(members){ - return Q.Promise(function(resolve, reject){ - getSentryMembers(that.currentDal, members, function(err, sentries) { - if (err) return reject(err); - resolve(sentries); - }); - }); - }) - ]) - .spread(function(mss, current, sentries){ - var ms = _.chain(mss).where({ membership: 'IN' }).sortBy(function(ms) { return -ms.number; }).value()[0]; - return Q.nfcall(getSinglePreJoinData, that.currentDal, current, idty.hash) - .then(function(join){ - join.ms = ms; - var joinData = {}; - joinData[join.identity.pubkey] = join; - return joinData; - }) - .then(function(joinData){ - var pubkey = _.keys(joinData)[0]; - var join = joinData[pubkey]; - // Check WoT stability - var someNewcomers = [join.identity.pubkey]; - var nextBlockNumber = current ? current.number + 1 : 0; - return Q.nfcall(computeNewLinks, nextBlockNumber, that.currentDal, someNewcomers, joinData, {}) - .then(function(newLinks){ - return Q.all([ - that.getValidCertsCount(pubkey, newLinks), - that.isOver3Hops(pubkey, newLinks, sentries, current) - ]) - .spread(function(certs, outdistanced) { - return { - uid: join.identity.uid, - meta: { - timestamp: parseInt(new Date(join.identity.time).getTime() / 1000) - }, - outdistanced: outdistanced, - certifications: certs, - membershipMissing: !join.ms - }; - }); - }); - }); - }, Q.reject); - }; + this.requirementsOfIdentities = (identities) => co(function *() { + let all = []; + let current = yield that.currentDal.getCurrent(); + let members = yield that.currentDal.getMembers(); + let sentries = yield Q.nfcall(getSentryMembers, that.currentDal, members); + for (let i = 0, len = identities.length; i < len; i++) { + let idty = new Identity(identities[i]); + let reqs = yield that.requirementsOfIdentity(idty, current, sentries); + all.push(reqs); + } + return all; + }); + + this.requirementsOfIdentity = (idty, current, sentries) => co(function *() { + let join = yield Q.nfcall(getSinglePreJoinData, that.currentDal, current, idty.hash); + let pubkey = join.identity.pubkey; + // Check WoT stability + let someNewcomers = [join.identity.pubkey]; + let nextBlockNumber = current ? current.number + 1 : 0; + let joinData = {}; + joinData[join.identity.pubkey] = join; + let updates = {}; + let newCerts = yield computeNewCerts(nextBlockNumber, that.currentDal, someNewcomers, joinData, updates); + let newLinks = newCertsToLinks(newCerts, updates); + let certs = yield that.getValidCerts(pubkey, newCerts); + let outdistanced = yield that.isOver3Hops(pubkey, newLinks, sentries, current); + let expiresMS = 0; + let currentTime = current ? current.medianTime : 0; + // Expiration of current membershship + if (join.identity.currentMSN >= 0) { + let msBlock = yield that.currentDal.getBlockOrNull(join.identity.currentMSN); + expiresMS = Math.max(0, (msBlock.medianTime + conf.msValidity - currentTime)); + } + // Expiration of pending membership + let expiresPending = 0; + let lastJoin = yield that.currentDal.lastJoinOfIdentity(idty.hash); + if (lastJoin) { + let msBlock = yield that.currentDal.getBlockOrNull(lastJoin.blockNumber); + expiresPending = Math.max(0, (msBlock.medianTime + conf.msValidity - currentTime)); + } + // Expiration of certifications + for (let i = 0, len = certs.length; i < len; i++) { + let cert = certs[i]; + cert.expiresIn = Math.max(0, cert.timestamp + conf.sigValidity - currentTime); + } + return { + pubkey: join.identity.pubkey, + uid: join.identity.uid, + meta: { + timestamp: parseInt(new Date(join.identity.time).getTime() / 1000) + }, + outdistanced: outdistanced, + certifications: certs, + membershipPendingExpiresIn: expiresPending, + membershipExpiresIn: expiresMS + }; + }); - this.getValidCertsCount = function(newcomer, newLinks) { - return that.currentDal.getValidLinksTo(newcomer) - .then(function(links){ - var count = links.length; - if (newLinks && newLinks.length) - count += newLinks.length; - return count; + this.getValidCerts = (newcomer, newCerts) => co(function *() { + let links = yield that.currentDal.getValidLinksTo(newcomer); + let certsFromLinks = links.map((lnk) => { return { from: lnk.source, to: lnk.target, timestamp: lnk.timestamp }; }); + let certsFromCerts = []; + let certs = newCerts[newcomer] || []; + for (let i = 0, len = certs.length; i < len; i++) { + let cert = certs[i]; + let block = yield that.currentDal.getBlockOrNull(cert.block_number); + certsFromCerts.push({ + from: cert.from, + to: cert.to, + timestamp: block.medianTime }); - }; + } + return certsFromLinks.concat(certsFromCerts); + }); this.isOver3Hops = function(newcomer, newLinks, sentries, current) { if (!current) { @@ -861,54 +868,68 @@ function BlockchainService (conf, mainDAL, pair) { .catch(done); } - function computeNewLinks (forBlock, dal, theNewcomers, joinData, updates, done) { - var newLinks = {}, certifiers = []; - var certsByKey = _.mapObject(joinData, function(val){ return val.certs; }); - async.waterfall([ - function (next){ - async.forEach(theNewcomers, function(newcomer, callback){ - // New array of certifiers - newLinks[newcomer] = newLinks[newcomer] || []; - // Check wether each certification of the block is from valid newcomer/member - async.forEach(certsByKey[newcomer], function(cert, callback){ - var isAlreadyCertifying = certifiers.indexOf(cert.from) !== -1; - if (isAlreadyCertifying && forBlock > 0) { - return callback(); - } + function computeNewCerts(forBlock, dal, theNewcomers, joinData) { + return co(function *() { + var newCerts = {}, certifiers = []; + var certsByKey = _.mapObject(joinData, function(val){ return val.certs; }); + for (let i = 0, len = theNewcomers.length; i < len; i++) { + let newcomer = theNewcomers[i]; + // New array of certifiers + newCerts[newcomer] = newCerts[newcomer] || []; + // Check wether each certification of the block is from valid newcomer/member + for (let j = 0, len2 = certsByKey[newcomer].length; j < len2; j++) { + let cert = certsByKey[newcomer][j]; + let isAlreadyCertifying = certifiers.indexOf(cert.from) !== -1; + if (!(isAlreadyCertifying && forBlock > 0)) { if (~theNewcomers.indexOf(cert.from)) { // Newcomer to newcomer => valid link - newLinks[newcomer].push(cert.from); + newCerts[newcomer].push(cert); certifiers.push(cert.from); - callback(); } else { - async.waterfall([ - function (next){ - dal.isMember(cert.from, next); - }, - function (isMember, next){ - // Member to newcomer => valid link - if (isMember) { - newLinks[newcomer].push(cert.from); - certifiers.push(cert.from); - } - next(); - } - ], callback); + let isMember = yield dal.isMember(cert.from); + // Member to newcomer => valid link + if (isMember) { + newCerts[newcomer].push(cert); + certifiers.push(cert.from); + } } - }, callback); - }, next); - }, - function (next){ - _.mapObject(updates, function(certs, pubkey) { - newLinks[pubkey] = (newLinks[pubkey] || []).concat(_.pluck(certs, 'pubkey')); - }); - next(); + } + } } - ], function (err) { - done(err, newLinks); + return newCerts; }); } + function newCertsToLinks(newCerts, updates) { + let newLinks = {}; + _.mapObject(newCerts, function(certs, pubkey) { + newLinks[pubkey] = _.pluck(certs, 'from'); + }); + _.mapObject(updates, function(certs, pubkey) { + newLinks[pubkey] = (newLinks[pubkey] || []).concat(_.pluck(certs, 'pubkey')); + }); + return newLinks; + } + + function computeNewLinksP(forBlock, dal, theNewcomers, joinData, updates) { + return co(function *() { + let newCerts = yield computeNewCerts(forBlock, dal, theNewcomers, joinData); + return newCertsToLinks(newCerts, updates); + }); + } + + function computeNewLinks (forBlock, dal, theNewcomers, joinData, updates, done) { + return computeNewLinksP(forBlock, dal, theNewcomers, joinData, updates) + .catch((err) => { + done && done(err); + throw err; + }) + .then((res) => { + done && done(null, res); + return res; + }); + } + function createBlock (dal, current, joinData, leaveData, updates, exclusions, lastUDBlock, transactions) { // Prevent writing joins/updates for excluded members exclusions.forEach(function (excluded) { @@ -923,6 +944,7 @@ function BlockchainService (conf, mainDAL, pair) { var block = new Block(); block.version = 1; block.currency = current ? current.currency : conf.currency; + block.nonce = 0; block.number = current ? current.number + 1 : 0; block.parameters = block.number > 0 ? '' : [ conf.c, conf.dt, conf.ud0, @@ -1036,30 +1058,6 @@ function BlockchainService (conf, mainDAL, pair) { } var powWorker; - this.getPoWProcessStats = function(done) { - if (powWorker) - usage.lookup(powWorker.powProcess.pid, done); - else - done(null, { memory: 0, cpu: 0 }); - }; - - var askedStop = null; - - this.stopProof = function(done) { - if (!newKeyblockCallback) { - askedStop = 'Stopping node.'; - newKeyblockCallback = function() { - newKeyblockCallback = null; - // Definitely kill the process for this BlockchainService instance - if (powWorker) { - powWorker.kill(); - } - done(); - }; - } - else done(); - }; - this.prove = function (block, sigFunc, nbZeros, done) { return Q.Promise(function(resolve){ @@ -1164,7 +1162,6 @@ function BlockchainService (conf, mainDAL, pair) { if (!selfPubkey) { throw 'No self pubkey found.'; } - askedStop = null; var block, current; var dal = mainDAL; var isMember = yield dal.isMember(selfPubkey); @@ -1274,6 +1271,15 @@ function BlockchainService (conf, mainDAL, pair) { if (blocks[0].number == 0) { yield that.saveParametersForRootBlock(blocks[0]); } + // Helper to retrieve a block with local cache + let getBlockOrNull = (number) => { + let firstLocalNumber = blocks[0].number; + if (number >= firstLocalNumber) { + let offset = number - firstLocalNumber; + return Q(blocks[offset]); + } + return mainDAL.getBlockOrNull(number); + }; // Insert a bunch of blocks let lastPrevious = blocks[0].number == 0 ? null : yield mainDAL.getBlock(blocks[0].number - 1); let rootBlock = (blocks[0].number == 0 ? blocks[0] : null) || (yield mainDAL.getBlockOrNull(0)); @@ -1298,50 +1304,26 @@ function BlockchainService (conf, mainDAL, pair) { } else { block.UDTime = previousBlock.UDTime; } - // Transactions & Memberships recording - yield mainDAL.saveTxsInFiles(block.transactions, { block_number: block.number, time: block.medianTime }); - yield mainDAL.saveMemberships('join', block.joiners); - yield mainDAL.saveMemberships('active', block.actives); - yield mainDAL.saveMemberships('leave', block.leavers); yield Q.Promise(function(resolve, reject){ - // Compute resulting entities - async.waterfall([ - function (next) { - // Create/Update members (create new identities if do not exist) - mainContext.updateMembers(block, next); - }, - function (next) { - // Create/Update certifications - mainContext.updateCertifications(block, next); - }, - function (next) { - // Create/Update memberships - mainContext.updateMemberships(block, next); - }, - function (next){ - // Save links - mainContext.updateLinks(block, next, (number) => { - let firstLocalNumber = blocks[0].number; - if (number >= firstLocalNumber) { - let offset = number - firstLocalNumber; - return Q(blocks[offset]); - } - return mainDAL.getBlockOrNull(number); - }); - }, - function (next){ - // Update consumed sources & create new ones - mainContext.updateTransactionSources(block, next); - } - ], function (err) { + // Create/Update members (create new identities if do not exist) + mainContext.updateMembers(block, function (err) { if (err) return reject(err); resolve(); }); }); } + // Transactions recording + yield mainContext.updateTransactionsForBlocks(blocks); + // Create certifications + yield mainContext.updateMembershipsForBlocks(blocks); + // Create certifications + yield mainContext.updateLinksForBlocks(blocks, getBlockOrNull); + // Create certifications + yield mainContext.updateCertificationsForBlocks(blocks); + // Create / Update sources + yield mainContext.updateTransactionSourcesForBlocks(blocks); yield mainDAL.blockDAL.saveBunch(blocks, (targetLastNumber - lastBlockToSave.number) > maxBlock); yield pushStatsForBlocks(blocks); - //console.log('Saved'); }); function pushStatsForBlocks(blocks) { diff --git a/app/service/HTTPService.js b/app/service/HTTPService.js deleted file mode 100644 index 3d6b88a385a187c8ed8606108ced24dd7029582e..0000000000000000000000000000000000000000 --- a/app/service/HTTPService.js +++ /dev/null @@ -1,16 +0,0 @@ -"use strict"; -var log4js = require('log4js'); -var logger = require('../lib/logger')('http'); - -module.exports = new function () { - - this.answer = function(res, code, err, done) { - if (err) { - logger.warn(err); - res.send(code, err); - } - else done(); - } - - return this; -} \ No newline at end of file diff --git a/app/service/IdentityService.js b/app/service/IdentityService.js index 108806ce94d5afc2951dc034e4f397e38368554b..adb16fa332a6f368c5768ff3cdc892cfdb279d45 100644 --- a/app/service/IdentityService.js +++ b/app/service/IdentityService.js @@ -40,7 +40,7 @@ function IdentityService (conf, dal) { return dal.searchJustIdentities(search); }; - this.findMember = (search, done) => co(function *() { + this.findMember = (search) => co(function *() { let idty = null; if (search.match(constants.PUBLIC_KEY)) { idty = yield dal.getWrittenIdtyByPubkey(search); @@ -49,16 +49,11 @@ function IdentityService (conf, dal) { idty = yield dal.getWrittenIdtyByUID(search); } if (!idty) { - throw 'No member matching this pubkey or uid'; + throw constants.ERRORS.NO_MEMBER_MATCHING_PUB_OR_UID; } yield dal.fillInMembershipsOfIdentity(Q(idty)); return new Identity(idty); - }) - .then(function(idty){ - done && done(null, idty); - return idty; - }) - .catch(done); + }); this.findMemberWithoutMemberships = (search) => co(function *() { let idty = null; @@ -69,7 +64,7 @@ function IdentityService (conf, dal) { idty = yield dal.getWrittenIdtyByUID(search); } if (!idty) { - throw 'No member matching this pubkey or uid'; + throw constants.ERRORS.NO_MEMBER_MATCHING_PUB_OR_UID; } return new Identity(idty); }); @@ -105,7 +100,7 @@ function IdentityService (conf, dal) { // Check signature's validity let verified = crypto.verify(selfCert, idty.sig, idty.pubkey); if (!verified) { - throw 'Signature does not match'; + throw constants.ERRORS.SIGNATURE_DOES_NOT_MATCH; } // CERTS for (let i = 0; i < certs.length; i++) { @@ -148,22 +143,22 @@ function IdentityService (conf, dal) { } } } else { - logger.info('✘ CERT %s wrong signature', cert.from); + logger.info('✘ CERT %s %s', cert.from, cert.err); } } let existing = yield dal.getIdentityByHashWithCertsOrNull(obj.hash); if (existing && !aCertWasSaved) { - throw 'Already up-to-date'; + throw constants.ERRORS.ALREADY_UP_TO_DATE; } else if (!existing) { // Create if not already written uid/pubkey let used = yield dal.getWrittenIdtyByPubkey(idty.pubkey); if (used) { - throw 'Pubkey already used in the blockchain'; + throw constants.ERRORS.PUBKEY_ALREADY_USED; } used = yield dal.getWrittenIdtyByUID(idty.uid); if (used) { - throw 'UID already used in the blockchain'; + throw constants.ERRORS.UID_ALREADY_USED; } idty = new Identity(idty); yield dal.savePendingIdentity(idty); diff --git a/app/service/MembershipService.js b/app/service/MembershipService.js index 4b068d89b2ee800fa00fe74b9e0d8de93e55996f..61def5192a600c642fce5242a7580c11ecef73dd 100644 --- a/app/service/MembershipService.js +++ b/app/service/MembershipService.js @@ -1,8 +1,13 @@ "use strict"; +var Q = require('q'); +var co = require('co'); var _ = require('underscore'); +var sha1 = require('sha1'); var async = require('async'); -var localValidator = require('../lib/localValidator'); +var moment = require('moment'); +var constants = require('../lib/constants'); +var localValidator = require('../lib/localValidator'); var globalValidator = require('../lib/globalValidator'); var blockchainDao = require('../lib/blockchainDao'); @@ -26,57 +31,42 @@ function MembershipService (conf, dal) { dal.getCurrentBlockOrNull(done); }; + let submitMembershipP = (ms) => co(function *() { + let entry = new Membership(ms); + entry.idtyHash = (sha1(entry.userid + moment(entry.certts).unix() + entry.issuer) + "").toUpperCase(); + let globalValidation = globalValidator(conf, blockchainDao(null, dal)); + logger.info('⬇ %s %s', entry.issuer, entry.membership); + if (!localValidator().checkSingleMembershipSignature(entry)) { + throw constants.ERRORS.WRONG_SIGNATURE_MEMBERSHIP; + } + // Get already existing Membership with same parameters + let found = yield dal.getMembershipForHashAndIssuer(entry); + if (found) { + throw constants.ERRORS.ALREADY_RECEIVED_MEMBERSHIP; + } + let isMember = yield dal.isMember(entry.issuer); + let isJoin = entry.membership == 'IN'; + if (!isMember && !isJoin) { + // LEAVE + throw constants.ERRORS.MEMBERSHIP_A_NON_MEMBER_CANNOT_LEAVE; + } + let current = yield dal.getCurrentBlockOrNull(); + yield Q.nbind(globalValidation.checkMembershipBlock, globalValidation)(entry, current); + // Saves entry + yield dal.savePendingMembership(entry); + logger.info('✔ %s %s', entry.issuer, entry.membership); + return entry; + }); + this.submitMembership = function (ms, done) { - var entry = new Membership(ms); - var globalValidation = globalValidator(conf, blockchainDao(null, dal)); - async.waterfall([ - function (next){ - logger.info('⬇ %s %s', entry.issuer, entry.membership); - if (!localValidator().checkSingleMembershipSignature(entry)) { - return next('wrong signature for membership'); - } - // Get already existing Membership with same parameters - dal.getMembershipForHashAndIssuer(entry).then(_.partial(next, null)).catch(next); - }, - function (found, next){ - if (found) { - next('Already received membership'); - } - else dal.isMember(entry.issuer, next); - }, - function (isMember, next){ - var isJoin = entry.membership == 'IN'; - if (!isMember && isJoin) { - // JOIN - next(); - } - else if (isMember && !isJoin) { - // LEAVE - next(); - } else { - if (isJoin) - // RENEW - next(); - else - next('A non-member cannot leave.'); - } - }, - function (next) { - dal.getCurrentBlockOrNull(next); - }, - function (current, next) { - globalValidation.checkMembershipBlock(entry, current, next); - }, - function (next){ - // Saves entry - dal.savePendingMembership(entry).then(function() { - next(); - }).catch(next); - }, - function (next){ - logger.info('✔ %s %s', entry.issuer, entry.membership); - next(null, entry); - } - ], done); + return submitMembershipP(ms) + .then((saved) => { + done && done(null, saved); + return saved; + }) + .catch((err) => { + done && done(err); + throw err; + }); }; } diff --git a/app/service/MerkleService.js b/app/service/MerkleService.js index 901cdfd909125f075131c634bc93f5970f3309fe..296f196128ec97d8dd355b1c10f04f045e1af755 100644 --- a/app/service/MerkleService.js +++ b/app/service/MerkleService.js @@ -2,13 +2,6 @@ module.exports = { - merkleDone: function (req, res, json) { - if(req.query.nice){ - res.end(JSON.stringify(json, null, " ")); - } - else res.end(JSON.stringify(json)); - }, - processForURL: function (req, merkle, valueCB, done) { // Result var json = { @@ -27,7 +20,7 @@ module.exports = { var hashes = [req.query.leaf]; // This code is in a loop for historic reasons. Should be set to non-loop style. valueCB(hashes, function (err, values) { - hashes.forEach(function (hash, index){ + hashes.forEach(function (hash){ json.leaf = { "hash": hash, "value": values[hash] || "" @@ -39,4 +32,4 @@ module.exports = { done(null, json); } } -} +}; diff --git a/app/service/ParametersService.js b/app/service/ParametersService.js index 21c89c9566630d7fc473d85e510b641f899e2188..5a25f8b095cbb449e0b63215b693b3cc36cd6922 100644 --- a/app/service/ParametersService.js +++ b/app/service/ParametersService.js @@ -55,6 +55,8 @@ function ParameterNamespace () { callback(null, matches[0]); }; + this.getPubkeyP = (req) => Q.nbind(this.getPubkey, this)(req); + this.getFrom = function (req, callback){ if(!req.params.from){ callback('Parameter `from` is required'); @@ -68,6 +70,8 @@ function ParameterNamespace () { callback(null, matches[0]); }; + this.getFromP = (req) => Q.nbind(this.getFrom, this)(req); + this.getTo = function (req, callback){ if(!req.params.to){ callback('Parameter `to` is required'); @@ -81,6 +85,8 @@ function ParameterNamespace () { callback(null, matches[0]); }; + this.getToP = (req) => Q.nbind(this.getTo, this)(req); + this.getFingerprint = function (req, callback){ if(!req.params.fpr){ callback("Fingerprint is required"); @@ -107,6 +113,8 @@ function ParameterNamespace () { callback(null, parseInt(matches[1])); }; + this.getNumberP = (req) => Q.nbind(this.getNumber, this)(req); + this.getCount = function (req, callback){ if(!req.params.count){ callback("Count is required"); diff --git a/app/service/PeeringService.js b/app/service/PeeringService.js index 41000a897fff3d02312d43c9a87dc51da2479bf4..68bf4787115e2bd1e3916917ea7bc976aea5f5a3 100644 --- a/app/service/PeeringService.js +++ b/app/service/PeeringService.js @@ -14,6 +14,8 @@ var constants = require('../lib/constants'); var localValidator = require('../lib/localValidator'); var blockchainCtx = require('../lib/blockchainContext'); +const DONT_IF_MORE_THAN_FOUR_PEERS = true; + function PeeringService(server, pair, dal) { var conf = server.conf; @@ -50,7 +52,7 @@ function PeeringService(server, pair, dal) { this.submitP = function(peering, eraseIfAlreadyRecorded, cautious){ let thePeer = new Peer(peering); let sp = thePeer.block.split('-'); - let blockNumber = sp[0]; + let blockNumber = parseInt(sp[0]); let blockHash = sp[1]; let sigTime = 0; let block; @@ -82,7 +84,7 @@ function PeeringService(server, pair, dal) { if(found){ // Already existing peer var sp2 = found.block.split('-'); - var previousBlockNumber = sp2[0]; + var previousBlockNumber = parseInt(sp2[0]); if(blockNumber <= previousBlockNumber && !eraseIfAlreadyRecorded){ throw constants.ERROR.PEER.ALREADY_RECORDED; } @@ -115,12 +117,21 @@ function PeeringService(server, pair, dal) { generateSelfPeer(conf, signalTimeInterval, done); }; + var crawlPeersFifo = async.queue((task, callback) => task(callback), 1); + var crawlPeersInterval = null; + this.regularCrawlPeers = function (done) { + if (crawlPeersInterval) + clearInterval(crawlPeersInterval); + crawlPeersInterval = setInterval(() => crawlPeersFifo.push(crawlPeers), 1000 * conf.avgGenTime * constants.NETWORK.SYNC_PEERS_INTERVAL); + crawlPeers(DONT_IF_MORE_THAN_FOUR_PEERS, done); + }; + var syncBlockFifo = async.queue((task, callback) => task(callback), 1); var syncBlockInterval = null; this.regularSyncBlock = function (done) { if (syncBlockInterval) clearInterval(syncBlockInterval); - syncBlockInterval = setInterval(() => syncBlockFifo.push(syncBlock), 1000*conf.avgGenTime*constants.NETWORK.SYNC_BLOCK_INTERVAL); + syncBlockInterval = setInterval(() => syncBlockFifo.push(syncBlock), 1000 * conf.avgGenTime * constants.NETWORK.SYNC_BLOCK_INTERVAL); syncBlock(done); }; @@ -130,7 +141,7 @@ function PeeringService(server, pair, dal) { this.regularTestPeers = function (done) { if (testPeerFifoInterval) clearInterval(testPeerFifoInterval); - testPeerFifoInterval = setInterval(() => testPeerFifo.push(testPeers.bind(null, !FIRST_CALL)), 1000 * conf.avgGenTime * constants.NETWORK.TEST_PEERS_INTERVAL); + testPeerFifoInterval = setInterval(() => testPeerFifo.push(testPeers.bind(null, !FIRST_CALL)), 1000 * constants.NETWORK.TEST_PEERS_INTERVAL); testPeers(FIRST_CALL, done); }; @@ -207,6 +218,79 @@ function PeeringService(server, pair, dal) { .catch(done); } + function crawlPeers(dontCrawlIfEnoughPeers, done) { + if (arguments.length == 1) { + done = dontCrawlIfEnoughPeers; + dontCrawlIfEnoughPeers = false; + } + logger.info('Crawling the network...'); + return co(function *() { + let peers = yield dal.listAllPeersWithStatusNewUPWithtout(selfPubkey); + if (peers.length > constants.NETWORK.COUNT_FOR_ENOUGH_PEERS && dontCrawlIfEnoughPeers == DONT_IF_MORE_THAN_FOUR_PEERS) { + return; + } + let peersToTest = peers.slice().map((p) => Peer.statics.peerize(p)); + let tested = []; + let found = []; + while (peersToTest.length > 0) { + let results = yield peersToTest.map(crawlPeer); + tested = tested.concat(peersToTest.map((p) => p.pubkey)); + // End loop condition + peersToTest.splice(0); + // Eventually continue the loop + for (let i = 0, len = results.length; i < len; i++) { + let res = results[i]; + for (let j = 0, len2 = res.length; j < len2; j++) { + try { + let subpeer = res[j].leaf.value; + if (subpeer.currency && tested.indexOf(subpeer.pubkey) === -1) { + let p = Peer.statics.peerize(subpeer); + peersToTest.push(p); + found.push(p); + } + } catch (e) { + logger.warn('Invalid peer %s', res[j]); + } + } + } + // Make unique list + peersToTest = _.uniq(peersToTest, false, (p) => p.pubkey); + } + logger.info('Crawling done.'); + for (let i = 0, len = found.length; i < len; i++) { + let p = found[i]; + try { + // Try to write it + p.documentType = 'peer'; + yield server.singleWritePromise(p); + } catch(e) { + // Silent error + } + } + }) + .then(() => done()).catch(done); + } + + function crawlPeer(aPeer) { + return co(function *() { + let subpeers = []; + try { + logger.info('Crawling peers of %s %s', aPeer.pubkey.substr(0, 6), aPeer.getNamedURL()); + let node = yield aPeer.connectP(); + //let remotePeer = yield Q.nbind(node.network.peering.get)(); + let json = yield Q.nbind(node.network.peering.peers.get, node)({ leaves: true }); + for (let i = 0, len = json.leaves.length; i < len; i++) { + let leaf = json.leaves[i]; + let subpeer = yield Q.nbind(node.network.peering.peers.get, node)({ leaf: leaf }); + subpeers.push(subpeer); + } + return subpeers; + } catch (e) { + return subpeers; + } + }); + } + function testPeers(displayDelays, done) { return co(function *() { let peers = yield dal.listAllPeers(); @@ -224,8 +308,15 @@ function PeeringService(server, pair, dal) { // We try to reconnect only with peers marked as DOWN try { logger.info('Checking if node %s (%s:%s) is UP...', p.pubkey.substr(0, 6), p.getHostPreferDNS(), p.getPort()); + // We register the try anyway + yield dal.setPeerDown(p.pubkey); + // Now we test let node = yield Q.nfcall(p.connect); let peering = yield Q.nfcall(node.network.peering.get); + // The node answered, it is no more DOWN! + logger.info('Node %s (%s:%s) is UP!', p.pubkey.substr(0, 6), p.getHostPreferDNS(), p.getPort()); + yield dal.setPeerUP(p.pubkey); + // We try to forward its peering entry let sp1 = peering.block.split('-'); let currentBlockNumber = sp1[0]; let currentBlockHash = sp1[1]; @@ -289,24 +380,29 @@ function PeeringService(server, pair, dal) { let peers = yield dal.findAllPeersNEWUPBut([selfPubkey]); peers = _.shuffle(peers); for (let i = 0, len = peers.length; i < len; i++) { - var p = new Peer(peers[i]); + let p = new Peer(peers[i]); logger.info("Try with %s %s", p.getURL(), p.pubkey.substr(0, 6)); - let node = yield Q.nfcall(p.connect); - let okUP = yield processAscendingUntilNoBlock(p, node, current); - if (okUP) { - let remoteCurrent = yield Q.nfcall(node.blockchain.current); - // We check if our current block has changed due to ascending pulling - let nowCurrent = yield dal.getCurrentBlockOrNull(); - logger.debug("Remote #%s Local #%s", remoteCurrent.number, nowCurrent.number); - if (remoteCurrent.number != nowCurrent.number) { - yield processLastTen(p, node, nowCurrent); - } - } try { + let node = yield Q.nfcall(p.connect); + let okUP = yield processAscendingUntilNoBlock(p, node, current); + if (okUP) { + let remoteCurrent = yield Q.nfcall(node.blockchain.current); + // We check if our current block has changed due to ascending pulling + let nowCurrent = yield dal.getCurrentBlockOrNull(); + logger.debug("Remote #%s Local #%s", remoteCurrent.number, nowCurrent.number); + if (remoteCurrent.number != nowCurrent.number) { + yield processLastTen(p, node, nowCurrent); + } + } // Try to fork as a final treatment let nowCurrent = yield dal.getCurrentBlockOrNull(); yield server.BlockchainService.tryToFork(nowCurrent); } catch (e) { + if (isConnectionError(e)) { + logger.info("Peer %s unreachable: now considered as DOWN.", p.pubkey); + yield dal.setPeerDown(p.pubkey); + return false; + } logger.warn(e); } } @@ -320,7 +416,7 @@ function PeeringService(server, pair, dal) { } function isConnectionError(err) { - return err && (err.code == "EINVAL" || err.code == "ECONNREFUSED"); + return err && (err.code == "EINVAL" || err.code == "ECONNREFUSED" || err.code == "ETIMEDOUT"); } function processAscendingUntilNoBlock(p, node, current) { @@ -331,7 +427,6 @@ function PeeringService(server, pair, dal) { yield dal.setPeerDown(p.pubkey); } while (downloaded) { - logger.info("Downloaded block #%s from peer %s", downloaded.number, p.getNamedURL()); downloaded = rawifyTransactions(downloaded); try { let res = yield server.BlockchainService.submitBlock(downloaded, true); @@ -340,10 +435,10 @@ function PeeringService(server, pair, dal) { yield server.BlockchainService.tryToFork(nowCurrent); } } catch (err) { - console.log(err); if (isConnectionError(err)) { throw err; } + logger.info("Downloaded block #%s from peer %s => %s", downloaded.number, p.getNamedURL(), err.code || err.message || err); } if (downloaded.number == 0) { downloaded = null; @@ -370,7 +465,6 @@ function PeeringService(server, pair, dal) { yield dal.setPeerDown(p.pubkey); } while (downloaded) { - logger.info("Downloaded block #%s from peer %s", downloaded.number, p.getNamedURL()); downloaded = rawifyTransactions(downloaded); try { let res = yield server.BlockchainService.submitBlock(downloaded, true); @@ -379,10 +473,10 @@ function PeeringService(server, pair, dal) { yield server.BlockchainService.tryToFork(nowCurrent); } } catch (err) { - console.log(err); if (isConnectionError(err)) { throw err; } + logger.info("Downloaded block #%s from peer %s => %s", downloaded.number, p.getNamedURL(), err.code || err.message || err || "Unknown error"); } if (downloaded.number == 0 || downloaded.number <= current.number - 10) { downloaded = null; diff --git a/bin/daemon b/bin/daemon new file mode 100755 index 0000000000000000000000000000000000000000..051bd70ecaa1c77a82c13370920e09aa352430c0 --- /dev/null +++ b/bin/daemon @@ -0,0 +1,32 @@ +#!/usr/bin/env node +"use strict"; + +var directory = require('../app/lib/directory'); +var path = require('path'); + +var daemon = require("daemonize2").setup({ + main: "ucoind", + name: directory.INSTANCE_NAME, + pidfile: path.join(directory.INSTANCE_HOME, "app.pid") +}); + +switch (process.argv[2]) { + + case "start": + daemon.start(); + break; + + case "stop": + daemon.stop(); + break; + + case "restart": + daemon.stop(function(err) { + err && console.error(err); + daemon.start(); + }); + break; + + default: + console.log("Usage: [start|stop|restart]"); +} diff --git a/bin/ucoind b/bin/ucoind index 4b2cc2c66d0fcfe3711db867e0422d8e985046a0..946803536d8677e167ca0bb8b2cb49f616289420 100755 --- a/bin/ucoind +++ b/bin/ucoind @@ -1,23 +1,24 @@ #!/usr/bin/env node "use strict"; -if (!~process.execArgv.indexOf('--harmony')) { - throw Error('This version requires --harmony flag. Use \'node --harmony\' instead of just \'node\'.'); -} +process.on('uncaughtException', function (err) { + logger.error(err); + process.exit(1); +}); +var logger = require('../app/lib/logger')('ucoind'); var async = require('async'); var co = require('co'); var Q = require('q'); var _ = require('underscore'); var program = require('commander'); var vucoin = require('vucoin'); +var directory = require('../app/lib/directory'); var wizard = require('../app/lib/wizard'); var multicaster = require('../app/lib/streams/multicaster'); -var logger = require('../app/lib/logger')('ucoind'); var signature = require('../app/lib/signature'); var crypto = require('../app/lib/crypto'); var base58 = require('../app/lib/base58'); -var constants = require('../app/lib/constants'); var Synchroniser = require('../app/lib/sync'); var bma = require('../app/lib/streams/bma'); var upnp = require('../app/lib/upnp'); @@ -38,6 +39,7 @@ program .version(pjson.version) .usage('<command> [options]') + .option('-h, --home <path>', 'Path to uCoin HOME (defaults to "$HOME/.config/ucoin").') .option('-d, --mdb <name>', 'Database name (defaults to "ucoin_default").') .option('--autoconf', 'With `init` command, will guess the best network and key options witout aksing for confirmation') @@ -53,7 +55,6 @@ program .option('--salt <salt>', 'Key salt to generate this key\'s secret key') .option('--passwd <password>', 'Password to generate this key\'s secret key') - .option('--member', 'With `bootstrap` command, ensures we have a member keypair') .option('--participate <Y|N>', 'Participate to writing the blockchain') .option('--cpu <percent>', 'Percent of CPU usage for proof-of-work computation', parsePercent) @@ -71,7 +72,6 @@ program .option('--growth <number>', 'Universal Dividend %growth. Aka. \'c\' parameter in RTM', parsePercent) .option('--ud0 <number>', 'Universal Dividend initial value') .option('--dt <number>', 'Number of seconds between two UD') - .option('--udid2', 'Enable udid2 format for user id') .option('--rootoffset <number>', 'Allow to give a time offset for first block (offset in the past)') .option('--show', 'With gen-next or gen-root commands, displays the generated block') @@ -99,12 +99,6 @@ program .description('Launch the configuration Wizard') .action(function (step) { // Only show message "Saved" - require('log4js').configure({ - "appenders": [{ - category: "ucoind", - type: "console" - }] - }); connect(function(step, server, conf){ async.series([ function(next) { @@ -184,7 +178,7 @@ program program .command('init [host] [port]') .description('Setup a node configuration and sync data with given node') - .action(connect(bootstrapServer, true, true)); + .action(connect(bootstrapServer, true)); program .command('forward [host] [port] [to]') @@ -201,7 +195,7 @@ program }, function(current, next) { remoteCurrent = current; - console.log(remoteCurrent.number); + logger.info(remoteCurrent.number); server.dal.getBlockFrom(remoteCurrent.number - 10).then(_.partial(next, null)).catch(next); }, function (blocks, next){ @@ -420,7 +414,7 @@ program server.disconnect(); process.exit(); }); - }, true)); + })); program .command('reset [config|data|peers|tx|stats|all]') @@ -467,11 +461,6 @@ function resetDone(server, msg) { function serverStart(server, conf) { - if (conf.udid2) { - // UserID must match udid2 format - constants.setUDID2Format(); - } - return co(function *() { // Enabling HTTP yield bma(server, null, conf.httplogs); @@ -567,7 +556,7 @@ function getBootstrapOperations(host, port, server, conf) { async.doWhilst(function(next){ async.waterfall([ function(next) { - if (!program.member && !conf.salt && !conf.passwd) { + if (!conf.salt && !conf.passwd) { if (program.autoconf) { conf.salt = ~~(Math.random() * 2147483647) + ""; conf.passwd = ~~(Math.random() * 2147483647) + ""; @@ -582,43 +571,6 @@ function getBootstrapOperations(host, port, server, conf) { function(){ startWizard("key", server, conf, next); }); - } else if(program.member) { - async.waterfall([ - function(next) { - async.parallel({ - node: function(callback){ - vucoin(host, port, callback, { timeout: server.conf.timeout }); - }, - keys: function(callback){ - crypto.getKeyPair(conf.passwd, conf.salt, callback); - } - }, next); - }, - function(res, next) { - var node = res.node; - var keys = res.keys; - // Look for existing member with this key - node.wot.certifiersOf(base58.encode(keys.publicKey), function(err) { - next(null, !err); - }); - }, - function(matchesMember, next) { - if (!matchesMember){ - wiz.choose('Your key does not match an existing member. Retry?', true, - function(){ - keyChosen = false; - startWizard("key", server, conf, next); - }, - function(){ - logger.warn('This node will not be able to compute blocks.'); - next(); - }); - } else { - keyChosen = true; - next(); - } - } - ], next); } else { next(); } @@ -660,7 +612,8 @@ function commandLineConf(conf) { }, db: { mport: program.mport, - mdb: program.mdb + mdb: program.mdb, + home: program.home }, net: { upnp: program.upnp, @@ -683,7 +636,6 @@ function commandLineConf(conf) { ud0: program.ud0, c: program.growth, dt: program.dt, - udid2: program.udid2, incDateMin: program.incDateMin, medtblocks: program.medtblocks, dtdiffeval: program.dtdiffeval, @@ -719,7 +671,6 @@ function commandLineConf(conf) { if (cli.ucp.dt) conf.dt = cli.ucp.dt; if (cli.ucp.c) conf.c = cli.ucp.c; if (cli.ucp.ud0) conf.ud0 = cli.ucp.ud0; - if (cli.ucp.udid2) conf.udid2 = cli.ucp.udid2; if (cli.ucp.incDateMin) conf.incDateMin = cli.ucp.incDateMin; if (cli.ucp.medtblocks) conf.medianTimeBlocks = cli.ucp.medtblocks; if (cli.ucp.avgGenTime) conf.avgGenTime = cli.ucp.avgGenTime; @@ -730,6 +681,7 @@ function commandLineConf(conf) { if (cli.logs.http) conf.httplogs = true; if (cli.logs.nohttp) conf.httplogs = false; if (cli.db.mport) conf.mport = cli.db.mport; + if (cli.db.home) conf.home = cli.db.home; if (cli.db.mdb) conf.mdb = cli.db.mdb; if (cli.isolate) conf.isolate = cli.isolate; if (cli.timeout) conf.timeout = cli.timeout; @@ -740,15 +692,16 @@ function commandLineConf(conf) { return _(conf).extend({ routing: true }); } -function connect(callback, forConf, useDefaultConf) { +function connect(callback, useDefaultConf) { return function () { var cbArgs = arguments; var dbName = program.mdb || "ucoin_default"; + var dbHome = program.home; - var server = ucoin({ name: dbName }, commandLineConf()); + var server = ucoin({ home: dbHome, name: dbName }, commandLineConf()); // Initialize server (db connection, ...) - server.connectDB(forConf, useDefaultConf) + server.connectDB(useDefaultConf) .then(function(){ cbArgs.length--; cbArgs[cbArgs.length++] = server; @@ -786,16 +739,18 @@ function service(callback, nologs) { if (nologs) { // Disable logs - require('log4js').configure({ - "appenders": [ - ] - }); + require('../app/lib/logger')().mute(); } var cbArgs = arguments; - var dbName = program.mdb || "ucoin_default"; + var dbName = program.mdb; + var dbHome = program.home; - var server = ucoin({ name: dbName, memory: program.memory }, commandLineConf()); + // Add log files for this instance + logger.addHomeLogs(directory.getHome(dbName, dbHome)); + logger.info(">> NODE STARTING"); + + var server = ucoin({ home: dbHome, name: dbName, memory: program.memory }, commandLineConf()); // Initialize server (db connection, ...) server.initWithServices() @@ -837,6 +792,9 @@ function exit (server) { function logIfErrorAndExit (server, prefix) { return function (err) { + if (err && err.uerr) { + err = err.uerr.message; + } err && logger.error((prefix ? prefix : "") + (err.message || err)); server.disconnect(); process.exit(); diff --git a/install.sh b/install.sh index ec151dea497ca73b306f1a96b1d0a054fbc7bc8a..a2fa0b6b1dfb443a095e4c9ea14068d66566d93a 100644 --- a/install.sh +++ b/install.sh @@ -11,7 +11,7 @@ if [ -z "$UCOIN_DIR" ]; then fi ucoin_latest_version() { - echo "v0.12.10" + echo "v0.13.0" } ucoin_repo_url() { @@ -155,7 +155,7 @@ install_ucoin_from_git() { fi # Download Nodejs - local NVER="0.12.7"; + local NVER="4.2.0"; local ARCH="86" local X64=`uname -a | grep "x86_64"` if [ ! -z "$X64" ]; then diff --git a/package.json b/package.json index 9514919e6146de6adc0afdce32dad530fbe77dc6..d3ad1f57882e102d98e74d7b94e0458b9d37b728 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "ucoin", - "version": "0.12.10", + "version": "0.13.0", "engines": { - "node": ">=0.12 <0.13", + "node": ">=4.2.0", "npm": ">=2.11" }, "engineStrict": true, @@ -13,9 +13,9 @@ "test": "test" }, "scripts": { - "test": "mocha --harmony --growl --timeout 20000 test test/fast test/fast/block test/medium test/integration test/", - "start": "node --harmony bin/ucoind start", - "test-travis": "node --harmony ./node_modules/istanbul/lib/cli.js cover ./node_modules/mocha/bin/_mocha --report lcovonly -- -R spec --timeout 20000 test test/fast test/fast/block test/medium test/integration test/" + "test": "mocha --growl --timeout 20000 test test/fast test/fast/block test/integration test/", + "start": "node bin/ucoind start", + "test-travis": "node ./node_modules/istanbul/lib/cli.js cover ./node_modules/mocha/bin/_mocha --report lcovonly -- -R spec --timeout 20000 test test/fast test/fast/block test/integration test/" }, "repository": { "type": "git", @@ -36,30 +36,32 @@ "async": "0.2.9", "bindings": "1.2.1", "co": "4.6.0", + "colors": "1.1.2", "commander": "2.1.0", + "daemonize2": "0.4.2", "event-stream": "3.1.5", "express": "3.4.7", "express-cors": "0.0.3", "inquirer": "0.8.5", - "log4js": "0.6.9", - "lokijs": "1.3.10", "merkle": "0.1.0", "moment": "2.6.0", + "morgan": "^1.6.1", "multimeter": "0.1.1", - "naclb": "0.2.2", + "naclb": "1.0.0", "nnupnp": "1.0.1", - "pm2": "0.15.10", + "optimist": "^0.6.1", "q": "1.1.2", - "q-io": "1.13.1", + "q-io": "1.13.2", "request": "2.31.0", - "scrypt": "4.0.7", + "scrypt": "5.4.1", "sha1": "1.1.0", - "socket.io": "1.3.4", + "socket.io": "1.3.7", + "sqlite3": "3.1.1", "superagent": "1.4.0", "tweetnacl": "0.11.2", "underscore": "1.8.3", - "usage": "0.6.0", - "vucoin": "0.25.1" + "vucoin": "0.26.0", + "winston": "^2.1.1" }, "devDependencies": { "coveralls": "2.11.4", diff --git a/server.js b/server.js index 7adfc7ef12e3af10a7cc99726ebf956dd4cf724c..4966eb584c24766928b6fc48af8ffc641280fac1 100644 --- a/server.js +++ b/server.js @@ -16,6 +16,7 @@ var crypto = require('./app/lib/crypto'); var Peer = require('./app/lib/entity/peer'); var signature = require('./app/lib/signature'); var common = require('./app/lib/common'); +var directory = require('./app/lib/directory'); var INNER_WRITE = true; // Add methods to String and Date @@ -25,6 +26,7 @@ function Server (dbConf, overrideConf) { stream.Duplex.call(this, { objectMode: true }); + let home = directory.getHome(dbConf.name, dbConf.home); var logger = require('./app/lib/logger')('server'); var that = this; var connectionPromise, servicesPromise; @@ -48,9 +50,9 @@ function Server (dbConf, overrideConf) { }); }; - this.connectDB = function (forConf, useDefaultConf) { + this.connectDB = function (useDefaultConf) { // Connect only once - return connectionPromise || (connectionPromise = that.connect(forConf, useDefaultConf)); + return connectionPromise || (connectionPromise = that.connect(useDefaultConf)); }; this.initWithServices = function (done) { @@ -65,43 +67,41 @@ function Server (dbConf, overrideConf) { }; this.submit = function (obj, isInnerWrite, done) { - async.waterfall([ - function (next){ - if (!obj.documentType) { - return next('Document type not given'); - } - var action = documentsMapping[obj.documentType]; + return co(function *() { + if (!obj.documentType) { + throw 'Document type not given'; + } + var action = documentsMapping[obj.documentType]; + try { + let res; if (typeof action == 'function') { // Handle the incoming object - action(obj, next); + res = yield Q.nbind(action, this)(obj); } else { - next('Unknown document type \'' + obj.documentType + '\''); + throw 'Unknown document type \'' + obj.documentType + '\''; } - } - ], function (err, res) { - err && logger.debug(err); - if (res != null && res != undefined && !err) { - // Only emit valid documents - that.emit(obj.documentType, res); - that.push(res); - } - if (isInnerWrite) { - done(err, res); - } else { - done(); + if (res) { + // Only emit valid documents + that.emit(obj.documentType, res); + that.push(res); + } + isInnerWrite ? done(null, res) : done(); + } catch (err) { + logger.debug(err); + isInnerWrite ? done(err, null) : done(); } }); }; this.submitP = (obj, isInnerWrite) => Q.nbind(this.submit, this)(obj, isInnerWrite); - this.connect = function (forConf, useDefaultConf) { + this.connect = function (useDefaultConf) { // Init connection if (that.dal) { return Q(); } var dbType = dbConf && dbConf.memory ? fileDAL.memory : fileDAL.file; - return dbType(dbConf.name || "default", forConf) + return dbType(home) .then(function(dal){ that.dal = dal; return that.dal.init(overrideConf, useDefaultConf); @@ -158,6 +158,9 @@ function Server (dbConf, overrideConf) { function (next){ that.checkConfig().then(next).catch(next); }, + function (next){ + that.PeeringService.regularCrawlPeers(next); + }, function (next){ logger.info('Storing self peer...'); that.PeeringService.regularPeerSignal(next); @@ -286,7 +289,6 @@ function Server (dbConf, overrideConf) { }, function(next) { that.servicesInited = true; - that.HTTPService = require("./app/service/HTTPService"); that.MerkleService = require("./app/service/MerkleService"); that.ParametersService = require("./app/service/ParametersService")(); that.IdentityService = require('./app/service/IdentityService')(that.conf, that.dal); @@ -299,9 +301,13 @@ function Server (dbConf, overrideConf) { 'identity': that.IdentityService.submitIdentity, 'revocation': that.IdentityService.submitRevocation, 'membership': that.MembershipService.submitMembership, - 'peer': that.PeeringService.submit, + 'peer': (obj, done) => { + that.PeeringService.submitP(obj) + .then((res) => done(null, res)) + .catch(done); + }, 'transaction': that.TransactionsService.processTx, - 'block': function (obj, done) { + 'block': (obj, done) => { that.BlockchainService.submitBlock(obj, true) .then(function(block){ that.dal = that.BlockchainService.currentDal; @@ -345,7 +351,7 @@ function Server (dbConf, overrideConf) { this.checkBlock = function(block) { return that.initWithServices() .then(function(){ - var parsed = parsers.parseBlock().syncWrite(block.getRawSigned()); + var parsed = parsers.parseBlock.syncWrite(block.getRawSigned()); return that.BlockchainService.checkBlock(parsed, false); }); }; @@ -363,10 +369,6 @@ function Server (dbConf, overrideConf) { that.BlockchainService.revertCurrentBlock(); }); - this.singleWriteStream = function (onError, onSuccess) { - return new TempStream(that, onError, onSuccess); - }; - this.singleWritePromise = function (obj) { return Q.Promise(function(resolve, reject){ new TempStream(that, reject, resolve).write(obj); diff --git a/test/dal/dal.js b/test/dal/dal.js index b28430f0d8bb37ab4af78bcfefee65e49ed5427e..17adf43516c16078a8ed37cac32e98d404b544b4 100644 --- a/test/dal/dal.js +++ b/test/dal/dal.js @@ -116,22 +116,22 @@ var mocks = { "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw" ], "leavers" : [ - "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:ccJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", - "HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk:1lFIiaR0QX0jibr5zQpXVGzBvMGqcsTRlmHiwGz5HOAZT8PTdVUb5q6YGZ6qAUZjdMjPmhLaiMIpYc47wUnzBA==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421786393:cgeek", - "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:ctyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", - "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:uoiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" + "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:3cJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", + "HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk:3lFIiaR0QX0jibr5zQpXVGzBvMGqcsTRlmHiwGz5HOAZT8PTdVUb5q6YGZ6qAUZjdMjPmhLaiMIpYc47wUnzBA==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421786393:cgeek", + "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:3tyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", + "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:3oiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" ], "actives" : [ - "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:ccJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", - "HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk:1lFIiaR0QX0jibr5zQpXVGzBvMGqcsTRlmHiwGz5HOAZT8PTdVUb5q6YGZ6qAUZjdMjPmhLaiMIpYc47wUnzBA==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421786393:cgeek", - "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:ctyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", - "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:uoiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" + "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:2cJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", + "HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk:2lFIiaR0QX0jibr5zQpXVGzBvMGqcsTRlmHiwGz5HOAZT8PTdVUb5q6YGZ6qAUZjdMjPmhLaiMIpYc47wUnzBA==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421786393:cgeek", + "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:2tyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", + "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:2oiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" ], "joiners" : [ - "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:ccJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", + "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:1cJm3F44eLMhQtnQY/7+14SWCDqVTL3Miw65hBVpV+YiUSUknIGhBNN0C0Cf+Pf0/pa1tjucW8Us3z5IklFSDg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787800:inso", "HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk:1lFIiaR0QX0jibr5zQpXVGzBvMGqcsTRlmHiwGz5HOAZT8PTdVUb5q6YGZ6qAUZjdMjPmhLaiMIpYc47wUnzBA==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421786393:cgeek", - "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:ctyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", - "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:uoiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" + "BMAVuMDcGhYAV4wA27DL1VXX2ZARZGJYaMwpf7DJFMYH:1tyAhpTRrAAOhFJukWI8RBr//nqYYdQibVzjOfaCdcWLb3TNFKrNBBothNsq/YrYHr7gKrpoftucf/oxLF8zAg==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421790376:moul", + "37qBxM4hLV2jfyYo2bNzAjkeLngLr2r7G2HpdpKieVxw:1oiGaC5b7kWqtqdPxwatPk9QajZHCNT9rf8/8ud9Rli24z/igcOf0Zr4A6RTAIKWUq9foW39VqJe+Y9R3rhACw==:0:DA39A3EE5E6B4B0D3255BFEF95601890AFD80709:1421787461:galuel" ], "identities" : [ "8Fi1VSTbjkXguwThF4v2ZxC5whK7pwG2vcGTkPUPjPGU:Ot3zIp/nsHT3zgJy+2YcXPL6vaM5WFsD+F8w3qnJoBRuBG6lv761zoaExp2iyUnm8fDAyKPpMxRK2kf437QSCw==:1421787800:inso", @@ -169,7 +169,7 @@ describe("DAL", function(){ it('should have no peer in a first time', function(){ return fileDAL.listAllPeers().then(function(peers){ peers.should.have.length(0); - }) + }); }); it('should have 1 peer if 1 is created', function(){ @@ -182,19 +182,19 @@ describe("DAL", function(){ peers[0].should.have.property('currency').equal('bb'); peers[0].should.have.property('endpoints').length(1); peers[0].endpoints[0].should.equal('BASIC_MERKLED_API localhost 7777'); - }) + }); }); it('should have no current block', function(){ return fileDAL.getCurrentBlockOrNull().then(function(current){ should.not.exist(current); - }) + }); }); it('should have no blocks in first time', function(){ return fileDAL.getCurrentBlockOrNull().then(function(block){ should.not.exist(block); - }) + }); }); it('should be able to save a Block', function(){ diff --git a/test/fast/block/block_format.js b/test/fast/block/block_format.js index 67d42066a24eab54ac0847286ee7dd5240c5bf7f..96ad389b1d3982fae64abc0674ffba27b3e000c4 100644 --- a/test/fast/block/block_format.js +++ b/test/fast/block/block_format.js @@ -35,7 +35,7 @@ var raw = "Version: 1\n" + describe("Block format", function(){ - var parser = parsers.parseBlock(); + var parser = parsers.parseBlock; it('a valid block should be well formatted', function(done){ parser.asyncWrite(raw, function (err, obj) { diff --git a/test/fast/block/block_global.js b/test/fast/block/block_global.js index 24877766ae162c827ca4b31ae305c7a646407c7a..8577cdeb9fb3815614139a75def8bc22566ff11e 100644 --- a/test/fast/block/block_global.js +++ b/test/fast/block/block_global.js @@ -6,7 +6,7 @@ var assert = require('assert'); var parsers = require('../../../app/lib/streams/parsers/doc'); var blocks = require('../../data/blocks'); var validator = require('../../../app/lib/globalValidator'); -var parser = parsers.parseBlock(); +var parser = parsers.parseBlock; var Block = require('../../../app/lib/entity/block'); var Identity = require('../../../app/lib/entity/identity'); diff --git a/test/fast/block/block_local.js b/test/fast/block/block_local.js index f7c09ac8186d1beb35d469af1e68f43230228f72..0ccfeffa8c49f584aff2fb9669d86e83f4b1e38b 100644 --- a/test/fast/block/block_local.js +++ b/test/fast/block/block_local.js @@ -5,7 +5,7 @@ var assert = require('assert'); var parsers = require('../../../app/lib/streams/parsers/doc'); var blocks = require('../../data/blocks'); var localValidator = require('../../../app/lib/localValidator'); -var parser = parsers.parseBlock(); +var parser = parsers.parseBlock; var Block = require('../../../app/lib/entity/block'); var Configuration = require('../../../app/lib/entity/configuration'); diff --git a/test/fast/peering.js b/test/fast/peering.js index f1f524b48df7b9c07eb824fcc8d3739a75f0e4ea..cceaed997ddfd74adb63440cbf5b102f229ed621 100644 --- a/test/fast/peering.js +++ b/test/fast/peering.js @@ -27,10 +27,8 @@ describe('Peer', function(){ var pr; before(function(done) { - var parser = parsers.parsePeer().asyncWrite(rawPeer, function (err, obj) { - pr = new Peer(obj); - done(err); - }); + pr = new Peer(parsers.parsePeer.syncWrite(rawPeer)); + done(); }); it('should be version 1', function(){ diff --git a/test/integration/branches.js b/test/integration/branches.js index 051f11c89cec8dbdebe9ee0eba2eaa55b3b066c1..614406f35578a9583d1f8f5fe7ae92d15325e027 100644 --- a/test/integration/branches.js +++ b/test/integration/branches.js @@ -16,10 +16,7 @@ var expectJSON = httpTest.expectJSON; var expectAnswer = httpTest.expectAnswer; var expectHttpCode = httpTest.expectHttpCode; -require('log4js').configure({ - "appenders": [ - ] -}); +require('../../app/lib/logger')().mute(); var MEMORY_MODE = true; var commonConf = { @@ -228,7 +225,7 @@ describe("Branches", function() { it('should have a 3 blocks fork window size', function() { return expectAnswer(rp('http://127.0.0.1:7778/node/summary', { json: true }), function(res) { res.should.have.property('ucoin').property('software').equal('ucoind'); - res.should.have.property('ucoin').property('version').equal('0.12.10'); + res.should.have.property('ucoin').property('version').equal('0.13.0'); res.should.have.property('ucoin').property('forkWindowSize').equal(3); }); }); diff --git a/test/integration/branches_revert.js b/test/integration/branches_revert.js index a4c13c7ed9c145208d79501d7f4fb083015dffe7..cce686433070f0b4fc6f5a4f15736fe83b43eeef 100644 --- a/test/integration/branches_revert.js +++ b/test/integration/branches_revert.js @@ -74,7 +74,7 @@ describe("Revert root", function() { }); it('/block/2 should exist', function() { - return expectHttpCode(404, 'Block not found', rp('http://127.0.0.1:7711/blockchain/block/2', { json: true })); + return httpTest.expectError(404, 'Block not found', rp('http://127.0.0.1:7711/blockchain/block/2', { json: true })); }); // Revert then Commit should be a neutral operation diff --git a/test/integration/forwarding.js b/test/integration/forwarding.js index bc1490049a15d283799a7267cee487b145f5ce1c..b8f2528e64c7f98a84fad002ca04708b355b2f2e 100644 --- a/test/integration/forwarding.js +++ b/test/integration/forwarding.js @@ -8,12 +8,6 @@ var node = require('./tools/node'); var user = require('./tools/user'); var jspckg = require('../../package'); -//require('log4js').configure({ -// "appenders": [ -// //{ category: "db1", type: "console" } -// ] -//}); - var MEMORY_MODE = true; describe("Forwarding", function() { diff --git a/test/integration/http_api.js b/test/integration/http_api.js index 1666face16a815db5687ac8a3767f1cdf7960fd9..fc68ab0176772fc028a2adc926f96e75b92a9ac2 100644 --- a/test/integration/http_api.js +++ b/test/integration/http_api.js @@ -7,6 +7,7 @@ var assert = require('assert'); var ucoin = require('./../../index'); var bma = require('./../../app/lib/streams/bma'); var user = require('./tools/user'); +var http = require('./tools/http'); var constants = require('../../app/lib/constants'); var rp = require('request-promise'); @@ -81,7 +82,7 @@ describe("HTTP API", function() { }); it('/block/88 should not exist', function() { - return expectHttpCode(404, rp('http://127.0.0.1:7777/blockchain/block/88')); + return http.expectError(404, rp('http://127.0.0.1:7777/blockchain/block/88')); }); it('/current should exist', function() { @@ -91,7 +92,7 @@ describe("HTTP API", function() { }); it('/membership should not accept wrong signature', function() { - return expectHttpCode(400, 'wrong signature for membership', rp.post('http://127.0.0.1:7777/blockchain/membership', { + return http.expectError(400, 'wrong signature for membership', rp.post('http://127.0.0.1:7777/blockchain/membership', { json: { membership: 'Version: 1\n' + 'Type: Membership\n' + @@ -106,8 +107,8 @@ describe("HTTP API", function() { })); }); - it('/membership should not accept wrong signature', function() { - return expectHttpCode(400, 'Document has unkown fields or wrong line ending format', rp.post('http://127.0.0.1:7777/blockchain/membership', { + it('/membership should not accept wrong signature 2', function() { + return http.expectError(400, 'Document has unkown fields or wrong line ending format', rp.post('http://127.0.0.1:7777/blockchain/membership', { json: { membership: 'Version: 1\n' + 'Type: Membership\n' + @@ -121,8 +122,8 @@ describe("HTTP API", function() { })); }); - it('/membership should not accept wrong signature', function() { - return expectHttpCode(400, 'Document has unkown fields or wrong line ending format', rp.post('http://127.0.0.1:7777/blockchain/membership', { + it('/membership should not accept wrong signature 3', function() { + return http.expectError(400, 'Document has unkown fields or wrong line ending format', rp.post('http://127.0.0.1:7777/blockchain/membership', { json: { membership: 'Version: 1\n' + 'Type: Membership\n' + @@ -139,26 +140,6 @@ describe("HTTP API", function() { }); }); -function expectHttpCode(code, message, promise) { - if (arguments.length == 2) { - promise = arguments[1]; - message = undefined; - } - return promise - .then(function(){ - assert.equal(200, code); - }) - .catch(function(err){ - if (err.response) { - assert.equal(err.response.statusCode, code); - if (message) { - assert.equal(err.error || err.cause, message); - } - } - else throw err; - }); -} - function expectJSON(promise, json) { return promise .then(function(resJson){ diff --git a/test/integration/identity-clean-test.js b/test/integration/identity-clean-test.js new file mode 100644 index 0000000000000000000000000000000000000000..24b7e1a0b227f4828726dda138871d164665f519 --- /dev/null +++ b/test/integration/identity-clean-test.js @@ -0,0 +1,101 @@ +"use strict"; + +var _ = require('underscore'); +var co = require('co'); +var should = require('should'); +var ucoin = require('./../../index'); +var bma = require('./../../app/lib/streams/bma'); +var user = require('./tools/user'); +var constants = require('../../app/lib/constants'); +var rp = require('request-promise'); +var httpTest = require('./tools/http'); +var commit = require('./tools/commit'); + +var expectAnswer = httpTest.expectAnswer; + +var MEMORY_MODE = true; +var commonConf = { + ipv4: '127.0.0.1', + currency: 'bb', + httpLogs: true, + forksize: 3, + sigWoT: 2, + msValidity: 10000, + parcatipate: false, // TODO: to remove when startGeneration will be an explicit call + sigQty: 1 +}; + +var s1 = ucoin({ + memory: MEMORY_MODE, + name: 'bb12' +}, _.extend({ + port: '7733', + pair: { + pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', + sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP' + } +}, commonConf)); + +var cat = user('cat', { pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'}, { server: s1 }); +var tic = user('tic', { pub: 'DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV', sec: '468Q1XtTq7h84NorZdWBZFJrGkB18CbmbHr9tkp9snt5GiERP7ySs3wM8myLccbAAGejgMRC9rqnXuW3iAfZACm7'}, { server: s1 }); +var toc = user('cat', { pub: 'DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo', sec: '64EYRvdPpTfLGGmaX5nijLXRqWXaVz8r1Z1GtaahXwVSJGQRn7tqkxLb288zwSYzELMEG5ZhXSBYSxsTsz1m9y8F'}, { server: s1 }); + +var now = Math.round(new Date().getTime() / 1000); + +describe("Identities cleaned", function() { + + before(function() { + + var commitS1 = commit(s1); + + return co(function *() { + yield s1.initWithServices().then(bma); + yield cat.selfCertPromise(now); + yield tic.selfCertPromise(now); + yield toc.selfCertPromise(now); + + yield expectAnswer(rp('http://127.0.0.1:7733/wot/lookup/cat', { json: true }), function(res) { + res.should.have.property('results').length(2); + res.results[0].should.have.property('uids').length(1); + res.results[0].uids[0].should.have.property('uid').equal('cat'); // This is cat + res.results[1].uids[0].should.have.property('uid').equal('cat'); // This is toc + }); + + yield cat.certPromise(tic); + yield tic.certPromise(cat); + yield cat.joinPromise(); + yield tic.joinPromise(); + yield tic.selfCertPromise(now + 1); + yield commitS1(); + + // We have the following WoT (diameter 1): + + /** + * cat <-> tic + */ + }); + }); + + it('should have 2 members', function() { + return expectAnswer(rp('http://127.0.0.1:7733/wot/members', { json: true }), function(res) { + res.should.have.property('results').length(2); + _.pluck(res.results, 'uid').sort().should.deepEqual(['cat', 'tic']); + }); + }); + + it('lookup should give only 1 cat', function() { + return expectAnswer(rp('http://127.0.0.1:7733/wot/lookup/cat', { json: true }), function(res) { + res.should.have.property('results').length(1); + res.results[0].should.have.property('uids').length(1); + res.results[0].uids[0].should.have.property('uid').equal('cat'); + }); + }); + + it('lookup should give only 1 tic', function() { + return expectAnswer(rp('http://127.0.0.1:7733/wot/lookup/tic', { json: true }), function(res) { + res.should.have.property('results').length(1); + res.results[0].should.have.property('uids').length(1); + res.results[0].uids[0].should.have.property('uid').equal('tic'); + }); + }); +}); diff --git a/test/integration/identity-test.js b/test/integration/identity-test.js index 247a705deac8a57fd62a84c4e008c39d45675a1d..aa51ed95c8ec7fa02f9e4c3f2e8aa4e8421bcb0c 100644 --- a/test/integration/identity-test.js +++ b/test/integration/identity-test.js @@ -19,6 +19,8 @@ var commonConf = { currency: 'bb', httpLogs: true, forksize: 3, + sigWoT: 2, + msValidity: 10000, parcatipate: false, // TODO: to remove when startGeneration will be an explicit call sigQty: 1 }; @@ -35,11 +37,15 @@ var s1 = ucoin({ }, commonConf)); var cat = user('cat', { pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'}, { server: s1 }); +var tac = user('tac', { pub: '2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc', sec: '2HuRLWgKgED1bVio1tdpeXrf7zuUszv1yPHDsDj7kcMC4rVSN9RC58ogjtKNfTbH1eFz7rn38U1PywNs3m6Q7UxE'}, { server: s1 }); var toc = user('toc', { pub: 'DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo', sec: '64EYRvdPpTfLGGmaX5nijLXRqWXaVz8r1Z1GtaahXwVSJGQRn7tqkxLb288zwSYzELMEG5ZhXSBYSxsTsz1m9y8F'}, { server: s1 }); var tic = user('tic', { pub: 'DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV', sec: '468Q1XtTq7h84NorZdWBZFJrGkB18CbmbHr9tkp9snt5GiERP7ySs3wM8myLccbAAGejgMRC9rqnXuW3iAfZACm7'}, { server: s1 }); var tic2 = user('tic', { pub: '4KEA63RCFF7AXUePPg5Q7JX9RtzXjywai1iKmE7LcoEC', sec: '48vHGE2xkhnC81ChSu7dHaNv8JqnYubyyHRbkmkeAPKNg8Tv2BE7kVi3voh2ZhfVpQhEJLzceufzqpJ2dqnyXNSp'}, { server: s1 }); +var man1 = user('man1', { pub: '12AbjvYY5hxV4v2KrN9pnGzgFxogwrzgYyncYHHsyFDK', sec: '2h8UNKE4YRnjmTGQTrgf4DZp2h3F5LqjnecxP8AgU6aH1x4dvbNVirsNeBiSR2UQfExuLAbdXiyM465hb5qUxYC1'}, { server: s1 }); +var man2 = user('man2', { pub: 'E44RxG9jKZQsaPLFSw2ZTJgW7AVRqo1NGy6KGLbKgtNm', sec: 'pJRwpaCWshKZNWsbDxAHFQbVjk6X8gz9eBy9jaLnVY9gUZRqotrZLZPZe68ag4vEX1Y8mX77NhPXV2hj9F1UkX3'}, { server: s1 }); +var man3 = user('man3', { pub: '5bfpAfZJ4xYspUBYseASJrofhRm6e6JMombt43HBaRzW', sec: '2VFQtEcYZRwjoc8Lxwfzcejtw9VP8VAi47WjwDDjCJCXu7g1tXUAbVZN3QmvG6NJqaSuLCuYP7WDHWkFmTrUEMaE'}, { server: s1 }); -var now = Math.round(new Date().getTime()/1000); +var now = Math.round(new Date().getTime() / 1000); describe("Identities", function() { @@ -50,34 +56,58 @@ describe("Identities", function() { return co(function *() { yield s1.initWithServices().then(bma); yield cat.selfCertPromise(now); + yield tac.selfCertPromise(now); yield toc.selfCertPromise(now); yield tic.selfCertPromise(now); yield toc.certPromise(cat); yield cat.certPromise(toc); yield cat.certPromise(tic); + yield tic.certPromise(tac); yield cat.joinPromise(); yield toc.joinPromise(); yield tic.joinPromise(); + yield tac.joinPromise(); yield commitS1(); + + // We have the following WoT (diameter 3): + + /** + * toc <=> cat -> tic -> tac + */ + + // cat is the sentry + + // Man1 is someone who just needs a commit to join + yield man1.selfCertPromise(now); + yield man1.joinPromise(); + yield tac.certPromise(man1); + + // Man2 is someone who has no certifications yet has sent a JOIN + yield man2.selfCertPromise(now); + yield man2.joinPromise(); + + // Man3 is someone who has only published its identity + yield man3.selfCertPromise(now); + try { yield tic.selfCertPromise(now + 2); throw 'Should have thrown an error for already used pubkey'; } catch (e) { - e.should.equal('Pubkey already used in the blockchain'); + JSON.parse(e).message.should.equal('Pubkey already used in the blockchain'); } try { yield tic2.selfCertPromise(now); throw 'Should have thrown an error for already used uid'; } catch (e) { - e.should.equal('UID already used in the blockchain'); + JSON.parse(e).message.should.equal('UID already used in the blockchain'); } }); }); - it('should have 3 identities', function() { + it('should have 4 members', function() { return expectAnswer(rp('http://127.0.0.1:7799/wot/members', { json: true }), function(res) { - res.should.have.property('results').length(3); - _.pluck(res.results, 'uid').sort().should.deepEqual(['cat', 'tic', 'toc']); + res.should.have.property('results').length(4); + _.pluck(res.results, 'uid').sort().should.deepEqual(['cat', 'tac', 'tic', 'toc']); }); }); @@ -106,7 +136,7 @@ describe("Identities", function() { }); it('should have identity-of/aaa', function() { - return httpTest.expectHttpCode(400, "No member matching this pubkey or uid", rp('http://127.0.0.1:7799/wot/identity-of/tac')); + return httpTest.expectError(404, "No member matching this pubkey or uid", rp('http://127.0.0.1:7799/wot/identity-of/aaa')); }); it('should have certifiers-of/cat giving results', function() { @@ -172,12 +202,64 @@ describe("Identities", function() { }); }); + it('requirements of cat', function() { + return expectAnswer(rp('http://127.0.0.1:7799/wot/requirements/cat', { json: true }), function(res) { + res.should.have.property('identities').be.an.Array; + res.should.have.property('identities').have.length(1); + res.identities[0].should.have.property('pubkey').equal('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd'); + res.identities[0].should.have.property('uid').equal('cat'); + res.identities[0].should.have.property('meta').property('timestamp'); + res.identities[0].should.have.property('outdistanced').have.length(0); + res.identities[0].should.have.property('certifications').have.length(1); + res.identities[0].should.have.property('membershipPendingExpiresIn').equal(0); + res.identities[0].should.have.property('membershipExpiresIn').greaterThan(9000); + }); + }); + + it('requirements of man1', function() { + return expectAnswer(rp('http://127.0.0.1:7799/wot/requirements/man1', { json: true }), function(res) { + res.should.have.property('identities').be.an.Array; + res.should.have.property('identities').have.length(1); + res.identities[0].should.have.property('pubkey').equal('12AbjvYY5hxV4v2KrN9pnGzgFxogwrzgYyncYHHsyFDK'); + res.identities[0].should.have.property('uid').equal('man1'); + res.identities[0].should.have.property('meta').property('timestamp'); + res.identities[0].should.have.property('outdistanced').have.length(0); + res.identities[0].should.have.property('certifications').length(1); + res.identities[0].certifications[0].should.have.property('from').equal('2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc'); + res.identities[0].certifications[0].should.have.property('to').equal('12AbjvYY5hxV4v2KrN9pnGzgFxogwrzgYyncYHHsyFDK'); + res.identities[0].certifications[0].should.have.property('expiresIn').greaterThan(0); + res.identities[0].should.have.property('membershipPendingExpiresIn').greaterThan(9000); + res.identities[0].should.have.property('membershipExpiresIn').equal(0); + }); + }); + it('should have certified-by/tic giving results', function() { return expectAnswer(rp('http://127.0.0.1:7799/wot/certified-by/tic', { json: true }), function(res) { res.should.have.property('pubkey').equal('DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV'); res.should.have.property('uid').equal('tic'); res.should.have.property('isMember').equal(true); res.should.have.property('sigDate').be.a.Number; + res.should.have.property('certifications').length(1); + let certs = res.certifications; + certs[0].should.have.property('pubkey').equal('2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc'); + certs[0].should.have.property('uid').equal('tac'); + certs[0].should.have.property('isMember').equal(true); + certs[0].should.have.property('wasMember').equal(true); + certs[0].should.have.property('sigDate').be.a.Number; + certs[0].should.have.property('cert_time').property('block').be.a.Number; + certs[0].should.have.property('cert_time').property('medianTime').be.a.Number; + certs[0].should.have.property('written').property('number').equal(0); + certs[0].should.have.property('written').property('hash').not.equal(''); + certs[0].should.have.property('signature').not.equal(''); + }); + }); + + it('should have certified-by/tac giving results', function() { + return expectAnswer(rp('http://127.0.0.1:7799/wot/certified-by/tac', { json: true }), function(res) { + res.should.have.property('pubkey').equal('2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc'); + res.should.have.property('uid').equal('tac'); + res.should.have.property('isMember').equal(true); + res.should.have.property('sigDate').be.a.Number; res.should.have.property('certifications').length(0); }); }); @@ -212,4 +294,34 @@ describe("Identities", function() { certs[1].should.have.property('signature').not.equal(''); }); }); + + it('requirements of man2', function() { + return expectAnswer(rp('http://127.0.0.1:7799/wot/requirements/man2', { json: true }), function(res) { + res.should.have.property('identities').be.an.Array; + res.should.have.property('identities').have.length(1); + res.identities[0].should.have.property('pubkey').equal('E44RxG9jKZQsaPLFSw2ZTJgW7AVRqo1NGy6KGLbKgtNm'); + res.identities[0].should.have.property('uid').equal('man2'); + res.identities[0].should.have.property('meta').property('timestamp'); + res.identities[0].should.have.property('outdistanced').have.length(1); + res.identities[0].outdistanced[0].should.equal('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd'); + res.identities[0].should.have.property('certifications').length(0); + res.identities[0].should.have.property('membershipPendingExpiresIn').greaterThan(9000); + res.identities[0].should.have.property('membershipExpiresIn').equal(0); + }); + }); + + it('requirements of man3', function() { + return expectAnswer(rp('http://127.0.0.1:7799/wot/requirements/man3', { json: true }), function(res) { + res.should.have.property('identities').be.an.Array; + res.should.have.property('identities').have.length(1); + res.identities[0].should.have.property('pubkey').equal('5bfpAfZJ4xYspUBYseASJrofhRm6e6JMombt43HBaRzW'); + res.identities[0].should.have.property('uid').equal('man3'); + res.identities[0].should.have.property('meta').property('timestamp'); + res.identities[0].should.have.property('outdistanced').have.length(1); + res.identities[0].outdistanced[0].should.equal('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd'); + res.identities[0].should.have.property('certifications').length(0); + res.identities[0].should.have.property('membershipPendingExpiresIn').equal(0); + res.identities[0].should.have.property('membershipExpiresIn').equal(0); + }); + }); }); diff --git a/test/integration/peerings.js b/test/integration/peerings.js index 50b9aae0c32a849af2d3c77e10a917e231b7b463..f4cb542c9ff730dc853a6a93599f8c0d6e85dcca 100644 --- a/test/integration/peerings.js +++ b/test/integration/peerings.js @@ -1,5 +1,6 @@ "use strict"; +var co = require('co'); var Q = require('q'); var _ = require('underscore'); var should = require('should'); @@ -102,106 +103,60 @@ describe("Network", function() { }, Q()) .then(function(){ - nodeS1 = vucoin_p('127.0.0.1', s1.conf.port); - nodeS2 = vucoin_p('127.0.0.1', s2.conf.port); - nodeS3 = vucoin_p('127.0.0.1', s3.conf.port); - // Server 1 - return Q() - .then(function() { - return cat.selfCertPromise(now); - }) - .then(function() { - return toc.selfCertPromise(now); - }) - .then(function() { - return tic.selfCertPromise(now); - }) - .then(_.partial(toc.certPromise, cat)) - .then(_.partial(cat.certPromise, toc)) - .then(_.partial(cat.certPromise, tic)) - .then(cat.joinPromise) - .then(toc.joinPromise) - .then(tic.joinPromise) - .then(commitS1); - }) - - .then(function(){ - return Q() - .then(function(){ - // Server 2 syncs block 0 - return sync(0, 0, s1, s2); - }) - .then(function(){ - // Server 3 syncs block 0 - return sync(0, 0, s1, s3); - }) - .then(function(){ - return nodeS1.getPeer().then(function(peer) { - return nodeS2.postPeer(new Peer(peer).getRawSigned()); - }); - }) - .then(function(){ - return nodeS2.getPeer().then(function(peer) { - return nodeS1.postPeer(new Peer(peer).getRawSigned()); - }); - }) - .then(function(){ - return nodeS3.getPeer().then(function(peer) { - return nodeS1.postPeer(new Peer(peer).getRawSigned()); - }); - }) - .then(commitS1) - .then(function(){ - return Q.all([ - until(s2, 'block', 1), - until(s3, 'block', 1) - ]); - }) - .then(function(){ - // A block was successfully spread accross the network - s2.bma.closeConnections(); - }) - .then(commitS1) - .then(function(){ - return Q.all([ - until(s3, 'block', 1) - ]); - }) - .then(function(){ - s2.bma.reopenConnections(); - // Server 2 syncs block number 2 (it did not have it) - return sync(2, 2, s1, s2); - }) - .then(function(){ - return s2.recomputeSelfPeer(); - }) - .then(function(){ - return nodeS2.getPeer().then(function(peer) { - return nodeS1.postPeer(new Peer(peer).getRawSigned()); - }); - }) - .then(function(){ - return Q.all([ - until(s2, 'block', 2), - until(s3, 'block', 2), - commitS1() - .then(commitS1) - ]); - }) - .then(commitS3) - .then(function(){ - return Q.all([ - until(s1, 'block', 1), - until(s2, 'block', 1) - ]); - }) - .then(commitS2) - .then(function(){ - return Q.all([ - until(s1, 'block', 1), - until(s3, 'block', 1) - ]); - }); + return co(function *() { + nodeS1 = vucoin_p('127.0.0.1', s1.conf.port); + nodeS2 = vucoin_p('127.0.0.1', s2.conf.port); + nodeS3 = vucoin_p('127.0.0.1', s3.conf.port); + // Server 1 + yield cat.selfCertPromise(now); + yield toc.selfCertPromise(now); + yield tic.selfCertPromise(now); + yield toc.certPromise(cat); + yield cat.certPromise(toc); + yield cat.certPromise(tic); + yield cat.joinPromise(); + yield toc.joinPromise(); + yield tic.joinPromise(); + yield commitS1(); + // Server 2 syncs block 0 + yield sync(0, 0, s1, s2); + // Server 3 syncs block 0 + yield sync(0, 0, s1, s3); + yield nodeS1.getPeer().then((peer) => nodeS2.postPeer(new Peer(peer).getRawSigned())); + yield nodeS2.getPeer().then((peer) => nodeS1.postPeer(new Peer(peer).getRawSigned())); + yield nodeS3.getPeer().then((peer) => nodeS1.postPeer(new Peer(peer).getRawSigned())); + yield commitS1(); + yield [ + until(s2, 'block', 1), + until(s3, 'block', 1) + ]; + // A block was successfully spread accross the network + s2.bma.closeConnections(); + yield commitS1(); + yield [ + until(s3, 'block', 1) + ]; + s2.bma.reopenConnections(); + // Server 2 syncs block number 2 (it did not have it) + yield sync(2, 2, s1, s2); + yield s2.recomputeSelfPeer(); + yield [ + until(s2, 'block', 2), + until(s3, 'block', 2), + commitS1() + .then(commitS1) + ]; + yield commitS3(); + yield [ + until(s1, 'block', 1), + until(s2, 'block', 1) + ]; + yield commitS2(); + yield [ + until(s1, 'block', 1), + until(s3, 'block', 1) + ]; + }); }) ; }); diff --git a/test/integration/scenarios/wot-lookup.js b/test/integration/scenarios/wot-lookup.js deleted file mode 100644 index 37b02dcfdf4486676842c1e0340aeb6181c6ca1e..0000000000000000000000000000000000000000 --- a/test/integration/scenarios/wot-lookup.js +++ /dev/null @@ -1,26 +0,0 @@ -"use strict"; -var user = require('./../tools/user'); - -module.exports = function(node1) { - - var cat = user('cat', { pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'}, node1); - var tac = user('tac', { pub: '2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc', sec: '2HuRLWgKgED1bVio1tdpeXrf7zuUszv1yPHDsDj7kcMC4rVSN9RC58ogjtKNfTbH1eFz7rn38U1PywNs3m6Q7UxE'}, node1); - var tic = user('tic', { pub: 'DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV', sec: '468Q1XtTq7h84NorZdWBZFJrGkB18CbmbHr9tkp9snt5GiERP7ySs3wM8myLccbAAGejgMRC9rqnXuW3iAfZACm7'}, node1); - var toc = user('toc', { pub: 'DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo', sec: '64EYRvdPpTfLGGmaX5nijLXRqWXaVz8r1Z1GtaahXwVSJGQRn7tqkxLb288zwSYzELMEG5ZhXSBYSxsTsz1m9y8F'}, node1); - - var now = Math.round(new Date().getTime()/1000); - - return [ - // Self certifications - cat.selfCert(now), - tac.selfCert(now), - tic.selfCert(now), - tic.selfCert(now + 2), - tic.selfCert(now + 2), - tic.selfCert(now + 2), - tic.selfCert(now + 3), - toc.selfCert(now), - // Certifications - cat.cert(tac) - ]; -}; diff --git a/test/integration/tests.js b/test/integration/tests.js index 87a323ced2fe6181659927345d190194de6e083b..c86c2af4ca8df7fac003af94e4b62f78cfa8a1d4 100644 --- a/test/integration/tests.js +++ b/test/integration/tests.js @@ -1,9 +1,12 @@ "use strict"; +var co = require('co'); var _ = require('underscore'); var should = require('should'); var assert = require('assert'); +var constants = require('../../app/lib/constants'); var node = require('./tools/node'); +var user = require('./tools/user'); var jspckg = require('../../package'); var MEMORY_MODE = true; @@ -20,6 +23,11 @@ describe("Integration", function() { } }); + var cat = user('cat', { pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'}, node1); + var tac = user('tac', { pub: '2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc', sec: '2HuRLWgKgED1bVio1tdpeXrf7zuUszv1yPHDsDj7kcMC4rVSN9RC58ogjtKNfTbH1eFz7rn38U1PywNs3m6Q7UxE'}, node1); + var tic = user('tic', { pub: 'DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV', sec: '468Q1XtTq7h84NorZdWBZFJrGkB18CbmbHr9tkp9snt5GiERP7ySs3wM8myLccbAAGejgMRC9rqnXuW3iAfZACm7'}, node1); + var toc = user('toc', { pub: 'DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo', sec: '64EYRvdPpTfLGGmaX5nijLXRqWXaVz8r1Z1GtaahXwVSJGQRn7tqkxLb288zwSYzELMEG5ZhXSBYSxsTsz1m9y8F'}, node1); + before(function(done) { node1.startTesting() .then(function(){ @@ -59,8 +67,34 @@ describe("Integration", function() { describe("Lookup on", function(){ - before(function(done) { - node1.before(require('./scenarios/wot-lookup')(node1))(done); + before(function() { + return co(function *() { + let now = Math.round(new Date().getTime() / 1000); + + // Self certifications + yield cat.selfCert(now); + yield tac.selfCert(now); + yield tic.selfCert(now); + yield tic.selfCert(now + 2); + // We send again the same + try { + yield tic.selfCert(now + 2); + throw 'Should have thrown an error'; + } catch (e) { + JSON.parse(e).ucode.should.equal(constants.ERRORS.ALREADY_UP_TO_DATE.uerr.ucode); + } + // We send again the same, again! + try { + yield tic.selfCert(now + 2); + throw 'Should have thrown an error'; + } catch (e) { + JSON.parse(e).ucode.should.equal(constants.ERRORS.ALREADY_UP_TO_DATE.uerr.ucode); + } + yield tic.selfCert(now + 3); + yield toc.selfCert(now); + // Certifications + yield cat.cert(tac); + }); }); after(node1.after()); diff --git a/test/integration/tools/http.js b/test/integration/tools/http.js index 5ba9b5cb31214e63541212f3cd09e0f288c2ccbf..34546f64a993259d21cf083d503ebd92fd54a684 100644 --- a/test/integration/tools/http.js +++ b/test/integration/tools/http.js @@ -26,6 +26,27 @@ module.exports = { }); }, + expectError: function expectHttpCode(code, message, promise) { + if (arguments.length == 2) { + promise = arguments[1]; + message = undefined; + } + return promise + .then(function(){ + assert.equal(200, code); + }) + .catch(function(err){ + if (err.response) { + assert.equal(err.response.statusCode, code); + if (message) { + let errorObj = typeof err.error == "string" ? JSON.parse(err.error) : err.error; + assert.equal(errorObj.message, message); + } + } + else throw err; + }); + }, + expectJSON: function expectJSON(promise, json) { return promise .then(function(resJson){ diff --git a/test/integration/tools/user.js b/test/integration/tools/user.js index 1d0c6196fd9f6629c30b1cb35b1187d51093585f..17843d88ccbf5966920cc130f4d6a7d4f163f2b4 100644 --- a/test/integration/tools/user.js +++ b/test/integration/tools/user.js @@ -302,5 +302,5 @@ function User (uid, options, node) { this.certP = (user) => Q.nfcall(this.cert(user)); this.joinP = () => Q.nfcall(this.join()); this.leaveP = () => Q.nfcall(this.leave()); - this.sendP = () => Q.nfcall(this.send.apply(this, arguments)); + this.sendP = (amount, userid, comment) => Q.nfcall(this.send.apply(this, [amount, userid, comment])); } diff --git a/test/medium/peerserver.disabled b/test/medium/peerserver.disabled deleted file mode 100644 index e3b99d2057b470b4c549802af1c5722706550bf5..0000000000000000000000000000000000000000 --- a/test/medium/peerserver.disabled +++ /dev/null @@ -1,135 +0,0 @@ -var ucoin = require('./../..'); -var async = require('async'); -var should = require('should'); -var fs = require('fs'); -var unix2dos = require('../../app/lib/unix2dos'); -var parsers = require('../../app/lib/streams/parsers/doc'); -var logger = require('../../app/lib/logger')('[peerserver]'); - -var pubkeyCatRaw = unix2dos(fs.readFileSync(__dirname + '/../data/lolcat.pub', 'utf8')); -var pubkeySnowRaw = unix2dos(fs.readFileSync(__dirname + '/../data/snow.pub', 'utf8')); -var pubkeyUbot1Raw = unix2dos(fs.readFileSync(__dirname + '/../data/ubot1.pub', 'utf8')); -var privkeyUbot1Raw = unix2dos(fs.readFileSync(__dirname + '/../data/ubot1.priv', 'utf8')); - -var pubkeyCat, pubkeySnow, pubkeyUbot1; -var peerServer; - -before(function (done) { - async.parallel({ - cat: function(callback){ - parsers.parsePubkey().asyncWrite(pubkeyCatRaw, function (err, obj) { - pubkeyCat = obj; - callback(err); - }); - }, - snow: function(callback){ - parsers.parsePubkey().asyncWrite(pubkeySnowRaw, function (err, obj) { - pubkeySnow = obj; - callback(err); - }); - }, - ubot1: function(callback){ - parsers.parsePubkey().asyncWrite(pubkeyUbot1Raw, function (err, obj) { - pubkeyUbot1 = obj; - callback(err); - }); - }, - server: function (callback) { - peerServer = ucoin.createPeerServer({ name: 'hdc2', listenBMA: false, resetData: true }, { - pgpkey: privkeyUbot1Raw, - pgppasswd: 'ubot1', - currency: 'beta_brousouf', - ipv4: '127.0.0.1', - port: 8080, - remoteipv4: '127.0.0.1', - remoteport: 8080 - }); - peerServer.on('services', callback); - } - }, done); -}) - -describe('A server', function () { - - this.timeout(1000*5); - - beforeEach(function (done) { - peerServer.reset(done); - }) - - // afterEach(function (done) { - // peerServer.disconnect(done); - // }) - - it('Peer should emit error on wrong data type', function (done) { - peerServer.on('error', function (err) { - should.exist(err); - done(); - }); - peerServer.write({ some: 'data' }); - }); - - it('Peer should accept pubkeys', function (done) { - async.parallel({ - pubkey: until(peerServer, 'pubkey'), - }, done); - peerServer.write(pubkeyCat); - }); - - it('Peer should accept status', function (done) { - async.parallel({ - status: until(peerServer, 'status'), - }, done); - peerServer.write(pubkeyCat); - peerServer.write({ - "version": "1", - "currency": "beta_brousouf", - "fingerprint": "C73882B64B7E72237A2F460CE9CAB76D19A8651E", - "endpoints": [ - "BASIC_MERKLED_API 127.0.0.1 8080" - ], - "keyID": "E9CAB76D19A8651E", - "signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js VERSION\r\nComment: http://openpgpjs.org\r\n\r\nwsBcBAEBCAAQBQJTlsmOCRDpyrdtGahlHgAAGPoIANAv8Q6PtaLuCzD9aDH+\nue9G10QNsXBCOIErj7wocmct3Y9yeYBwyAfth+ia0K/YDgygOY+n1yKid6QD\nlEOaDSENcdONZlYO/zAHDu6vQR/zsAPyztRCp0TSOCxQcQV2xSFkSvUSF8g2\noNI8RETgpLIlbKE8sS3F4v5OcxSa6wkhgngqRL6ZmqYqTPzgsAXlguA/Tq48\nNwRUQZBeP/TnMvnhhaZeww5qgxMNKWAMIjv7RUvMoP+YMMwSpgIKD3QYOhFK\nZLfYnxhiS/1jtJ+GTVdPLr5MNjLnNAc195aBT7OGi2frIsr7Qhz6TdMQnh0b\n39ohs+qaacQFbPS8qyVbhsM=\r\n=0nGP\r\n-----END PGP SIGNATURE-----\r\n", - "pubkey": { fingerprint: "C73882B64B7E72237A2F460CE9CAB76D19A8651E" } - }); - peerServer.write({ - "version": "1", - "currency": "beta_brousouf", - "status": "UP", - "keyID": "E9CAB76D19A8651E", - "pubkey": { fingerprint: "C73882B64B7E72237A2F460CE9CAB76D19A8651E" } - }); - }); - - it('Peer should accept peerings', function (done) { - async.parallel({ - peer: until(peerServer, 'peer'), - }, done); - peerServer.write(pubkeyCat); - peerServer.write({ - "version": "1", - "currency": "beta_brousouf", - "fingerprint": "C73882B64B7E72237A2F460CE9CAB76D19A8651E", - "keyID": "E9CAB76D19A8651E", - "endpoints": [ - "BASIC_MERKLED_API 127.0.0.1 8090" - ], - "signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js VERSION\r\nComment: http://openpgpjs.org\r\n\r\nwsBcBAEBCAAQBQJTmIfQCRDpyrdtGahlHgAAPboIAIILjXEgODUmkq0shKi+\n+BsOnZNSQ6dzmUYtqjsN83VyqsDIDZSKqQz3khXgDqcAVBXQcaL3oSrZOv70\n53E7oCKh+dOnAuOGrmWUUc2T0lkxppiwINQ9o8JqzDp9qpH8fSlFQu0HWuq/\noYar5B64Tp+dRoUY6iP3qqOpKKRLVj3z8vKJFyRXULNbawQPtrWem5OLatP2\nJw16pK04+IuMdA1+6+t/aeiqIoS/KRT2XlqrJe9nV5YXPC7KlXI80kd0sCEy\nuc7h/WIxkOlTfeXLuSRnQD+JMUKagMvoL7DbjvIgRlPhHp1xk1VjWkqBzBof\ntsf7xfAms830g9nsYnkvy30=\r\n=1kYK\r\n-----END PGP SIGNATURE-----\r\n", - "pubkey": { fingerprint: "C73882B64B7E72237A2F460CE9CAB76D19A8651E" } - }); - }); -}) - -function until (server, eventName, count) { - var counted = 0; - var max = count == undefined ? 1 : count; - return function (callback) { - server.on(eventName, function (obj) { - logger.trace('event = %s', eventName); - should.exist(obj); - counted++; - if (counted == max) - callback(); - }); - } -} \ No newline at end of file diff --git a/test/medium/routing.disabled b/test/medium/routing.disabled deleted file mode 100644 index 2dcb1b2d67d5d28474bccb9aaabeb350dd5ab26a..0000000000000000000000000000000000000000 --- a/test/medium/routing.disabled +++ /dev/null @@ -1,172 +0,0 @@ -var ucoin = require('./../..'); -var should = require('should'); -var fs = require('fs'); -var async = require('async'); -var unix2dos = require('../../app/lib/unix2dos'); -var parsers = require('../../app/lib/streams/parsers/doc'); -var logger = require('../../app/lib/logger')('[routing]'); - -var pubkeyCatRaw = unix2dos(fs.readFileSync(__dirname + '/../data/lolcat.pub', 'utf8')); -var pubkeySnowRaw = unix2dos(fs.readFileSync(__dirname + '/../data/snow.pub', 'utf8')); -var pubkeyWhiteRaw = unix2dos(fs.readFileSync(__dirname + '/../data/white.pub', 'utf8')); -var pubkeyUbot1Raw = unix2dos(fs.readFileSync(__dirname + '/../data/ubot1.pub', 'utf8')); - -// Only show new data events -require('log4js').configure({}); -// require('log4js').configure({ -// "appenders": [ -// { category: "hdcA", type: "console" }, -// { category: "hdcB", type: "console" }, -// ] -// }); - -describe('In a unidirectional 2 servers network,', function () { - - this.timeout(5000); - - var serverA, serverB; - - beforeEach(function (done) { - serverA = ucoin.createWOTServer({ name: 'hdcA', resetData: true }); - serverB = ucoin.createWOTServer({ name: 'hdcB', resetData: true }); - serverA.pipe(serverB); - resetServers(serverA, serverB)(done); - }); - - it('writing a pubkey from A should reach B', function (done) { - async.parallel([ - until(serverA, 'pubkey', 2), - until(serverB, 'pubkey', 2), - ], done); - serverA.writeRawPubkey(pubkeyUbot1Raw); - serverA.writeRawPubkey(pubkeyCatRaw); - }); - - it('writing a pubkey from B should NOT reach A', function (done) { - async.parallel([ - until(serverA, 'pubkey', 1), - until(serverB, 'pubkey', 3), - ], done); - serverA.writeRawPubkey(pubkeyCatRaw); // A + B - serverB.writeRawPubkey(pubkeyUbot1Raw); // B - serverB.writeRawPubkey(pubkeySnowRaw); // B - }); -}); - -describe('In a bidirectionnal 2 servers network,', function () { - - this.timeout(5000); - - var serverA, serverB; - - beforeEach(function (done) { - serverA = ucoin.createWOTServer({ name: 'hdcC', resetData: true }); - serverB = ucoin.createWOTServer({ name: 'hdcD', resetData: true }); - serverA.pipe(serverB); - serverB.pipe(serverA); - resetServers(serverA, serverB)(done); - }); - - it('writing a pubkey from A should reach B', function (done) { - async.parallel([ - until(serverA, 'pubkey', 3), - until(serverB, 'pubkey', 3), - ], done); - serverA.writeRawPubkey(pubkeyUbot1Raw); - serverA.writeRawPubkey(pubkeyCatRaw); - serverA.writeRawPubkey(pubkeySnowRaw); - }); - - it('writing a pubkey from B should reach A', function (done) { - async.parallel([ - until(serverA, 'pubkey', 3), - until(serverB, 'pubkey', 3), - ], done); - serverB.writeRawPubkey(pubkeyUbot1Raw); - serverB.writeRawPubkey(pubkeyCatRaw); - serverB.writeRawPubkey(pubkeySnowRaw); - }); -}); - -describe('In an oriented 5 servers network,', function () { - - var serverA, serverB, serverC, serverD, serverE; - - this.timeout(5000); - - before(function (done) { - - serverA = ucoin.createWOTServer({ name: 'test_A', resetData: true }); - serverB = ucoin.createWOTServer({ name: 'test_B', resetData: true }); - serverC = ucoin.createWOTServer({ name: 'test_C', resetData: true }); - serverD = ucoin.createWOTServer({ name: 'test_D', resetData: true }); - serverE = ucoin.createWOTServer({ name: 'test_E', resetData: true }); - - serverA.pipe(serverB).pipe(serverA); // A ◀--▶ B - serverB.pipe(serverC).pipe(serverB); // B ◀--▶ C - serverB.pipe(serverD).pipe(serverB); // B ◀--▶ D - serverD.pipe(serverE); // D --▶ E - - // A ◀--▶ B ◀--▶ C - // ▲ - // | - // ▼ - // D --▶ E - - async.parallel([ - function(cb){ serverA.on('services', cb) }, - function(cb){ serverB.on('services', cb) }, - function(cb){ serverC.on('services', cb) }, - function(cb){ serverD.on('services', cb) }, - function(cb){ serverE.on('services', cb) }, - ], done); - }) - - it('writing a 4 pubkeys with 1 to receveing server should give 3 to every node, but 4 for receveing', function (done) { - async.parallel([ - until(serverA, 'pubkey', 3), - until(serverB, 'pubkey', 3), - until(serverC, 'pubkey', 3), - until(serverD, 'pubkey', 3), - until(serverE, 'pubkey', 4), - ], done); - serverA.writeRawPubkey(pubkeyCatRaw); - serverB.writeRawPubkey(pubkeySnowRaw); - serverD.writeRawPubkey(pubkeyWhiteRaw); - serverE.writeRawPubkey(pubkeyUbot1Raw); - }); -}); - -function resetServers () { - process.stdout.write(''); // Why this? Because otherwise, test is stuck - var nbArgs = arguments.length; - var resets = []; - for (var i = 0; i < (nbArgs || 0); i++) { - var server = arguments[i]; - resets.push(function (done) { - async.series([ - function (cb) { - server.on('services', cb); - }, - server.reset - ], done); - }); - } - return function (done) { - async.parallel(resets, done); - }; -} - -function until (server, eventName, count) { - var counted = 0; - var max = count == undefined ? 1 : count; - return function (callback) { - server.on(eventName, function (obj) { - logger.trace('event = %s', eventName); - should.exist(obj); - counted++; - if (counted == max) - callback(); - }); - } -} diff --git a/ucoin.sh b/ucoin.sh index 61c651673334380ae087b9dc0497539778409062..6cf02408bf769ce65caee7ca172b2b42469a5ac8 100755 --- a/ucoin.sh +++ b/ucoin.sh @@ -8,81 +8,43 @@ ucoind() { - local UCOIN_DATABASE - local UCOIN_LOG_FILE - local UCOIN_DATA_HOME local NODE - local PM2 - if [[ -d $UCOIN_DIR/node ]]; then - NODE=$UCOIN_DIR/node/bin/node - fi; - - VERSION=`$NODE -v` + if [ -z "$DEV_MODE" ]; then - if [[ $VERSION != v0.12* ]]; then - echo "$NODE v0.12 is required"; + ### Production mode + if [[ -d $UCOIN_DIR/node ]]; then + NODE=$UCOIN_DIR/node/bin/node + fi; else - # OK, execute command - PM2=$UCOIN_DIR/node_modules/pm2/bin/pm2 - UCOIN_DATA_HOME=$HOME/.config/ucoin + ### Cheating with DEV mode + UCOIN_DIR=`pwd` + NODE=node + fi - case "$1" in - - #--------------------------------- - # UCOIN DAEMON MANAGEMENT: START - #--------------------------------- + VERSION=`$NODE -v` - start) - local test - local UCOIN_LOG_FILE - local UCOIN_ERR_FILE - UCOIN_DATABASE=$2 - if [ -z $UCOIN_DATABASE ]; then - UCOIN_DATABASE="$UCOIN_DB" - fi - if [ -z $UCOIN_DATABASE ]; then - UCOIN_DATABASE="ucoin_default" - fi - UCOIN_LOG_FILE=$UCOIN_DATA_HOME/$UCOIN_DATABASE/ucoin.log - UCOIN_ERR_FILE=$UCOIN_DATA_HOME/$UCOIN_DATABASE/ucoin.err.log - test=`$NODE $PM2 list | grep "$UCOIN_DATABASE.*online"` - if [ -z "$test" ]; then - echo $UCOIN_LOG_FILE - $NODE $PM2 start -f "$UCOIN_DIR/bin/ucoind" --name "$UCOIN_DATABASE" --interpreter="$NODE" --node-args="--harmony" --log $UCOIN_LOG_FILE --error $UCOIN_ERR_FILE --merge-logs -- start --mdb "$UCOIN_DATABASE" --httplogs 2>/dev/null - echo "uCoin with DB '$UCOIN_DATABASE' started. Use 'ucoind logs' to see interactive logs." - else - echo 1>&2 "uCoin '$UCOIN_DATABASE' already started." - fi - ;; + if [[ $VERSION != v4* ]]; then + echo "$NODE v4+ is required"; + else + case "$1" in #--------------------------------- - # UCOIN DAEMON MANAGEMENT: STOP & OTHERS + # UCOIN DAEMON MANAGEMENT #--------------------------------- - list|info|logs|stop|restart|monit|delete) - UCOIN_DATABASE=$2 - if [ -z $UCOIN_DATABASE ]; then - UCOIN_DATABASE="$UCOIN_DB" - fi - if [ -z $UCOIN_DATABASE ]; then - UCOIN_DATABASE="ucoin_default" - fi - $NODE $PM2 $1 $UCOIN_DATABASE - ;; - - delete-all) - $NODE $PM2 delete all + start|stop|restart) + $NODE "$UCOIN_DIR/bin/daemon" $* ;; #--------------------------------- - # UCOIN NORMAL COMMANDS + # UCOIN CLI COMMANDS #--------------------------------- *) - $NODE --harmony "$UCOIN_DIR/bin/ucoind" --mdb "$UCOIN_DATABASE" $* + $NODE "$UCOIN_DIR/bin/ucoind" $* ;; esac