diff --git a/.eslintignore b/.eslintignore index 34c7cb27cf5ba69e475149455605b4d9c0da72b5..f8cb1376087dafdc61b8b96e46238d60be7149b1 100644 --- a/.eslintignore +++ b/.eslintignore @@ -9,5 +9,6 @@ app/lib/dal/drivers/*.js app/lib/dal/sqliteDAL/*.js app/lib/dal/sqliteDAL/index/*.js app/lib/dal/fileDALs/*.js +app/lib/dal/fileDAL.js test/*.js test/**/*.js \ No newline at end of file diff --git a/.gitignore b/.gitignore index c4cca1419073eb89a6e7b0017319bec510c3917e..f386f2a3996d00729a137caa3d5e656e7c5f9a44 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,6 @@ app/lib/indexer.js* app/lib/dal/drivers/*.js* app/lib/dal/sqliteDAL/*.js* app/lib/dal/sqliteDAL/index/*.js* -app/lib/dal/fileDALs/*.js* \ No newline at end of file +app/lib/dal/fileDALs/*.js* +app/lib/dal/fileDAL.js* +app/lib/wot.js* \ No newline at end of file diff --git a/app/lib/computation/BlockchainContext.ts b/app/lib/computation/BlockchainContext.ts index 5dc02dd817c747a1aac199da72efe7b059d5b262..88ab213cc681e1d5f4ce41d87d8b48f2c89b729d 100644 --- a/app/lib/computation/BlockchainContext.ts +++ b/app/lib/computation/BlockchainContext.ts @@ -89,7 +89,7 @@ export class BlockchainContext { */ async getIssuerPersonalizedDifficulty(issuer: string): Promise<any> { const local_vHEAD = await this.getvHeadCopy({ issuer }); - await indexer.preparePersonalizedPoW(local_vHEAD, this.vHEAD_1, this.dal.range, this.conf) + await indexer.preparePersonalizedPoW(local_vHEAD, this.vHEAD_1, (n:number, m:number, p = "") => this.dal.range(n,m,p), this.conf) return local_vHEAD.issuerDiff; } diff --git a/app/lib/dal/fileDAL.js b/app/lib/dal/fileDAL.js deleted file mode 100644 index 18aab97e1b787013d3092df6099d51cdf54a2855..0000000000000000000000000000000000000000 --- a/app/lib/dal/fileDAL.js +++ /dev/null @@ -1,780 +0,0 @@ -"use strict"; -const Q = require('q'); -const co = require('co'); -const _ = require('underscore'); -const common = require('duniter-common'); -const indexer = require('../indexer').Indexer -const logger = require('../logger')('filedal'); -const Configuration = require('../entity/configuration'); -const Merkle = require('../entity/merkle'); -const Transaction = require('../entity/transaction'); -const TransactionDTO = require('../dto/TransactionDTO').TransactionDTO -const constants = require('../constants'); -const ConfDAL = require('./fileDALs/ConfDAL').ConfDAL -const StatDAL = require('./fileDALs/StatDAL').StatDAL - -module.exports = (params) => { - return new FileDAL(params); -}; - -function FileDAL(params) { - - const rootPath = params.home; - const myFS = params.fs; - const sqliteDriver = params.dbf(); - const that = this; - - this.profile = 'DAL'; - this.wotb = params.wotb; - - // DALs - this.confDAL = new ConfDAL(rootPath, myFS) - this.metaDAL = new (require('./sqliteDAL/MetaDAL').MetaDAL)(sqliteDriver); - this.peerDAL = new (require('./sqliteDAL/PeerDAL').PeerDAL)(sqliteDriver); - this.blockDAL = new (require('./sqliteDAL/BlockDAL').BlockDAL)(sqliteDriver); - this.txsDAL = new (require('./sqliteDAL/TxsDAL').TxsDAL)(sqliteDriver); - this.statDAL = new StatDAL(rootPath, myFS) - this.idtyDAL = new (require('./sqliteDAL/IdentityDAL').IdentityDAL)(sqliteDriver); - this.certDAL = new (require('./sqliteDAL/CertDAL').CertDAL)(sqliteDriver); - this.msDAL = new (require('./sqliteDAL/MembershipDAL').MembershipDAL)(sqliteDriver); - this.walletDAL = new (require('./sqliteDAL/WalletDAL').WalletDAL)(sqliteDriver); - this.bindexDAL = new (require('./sqliteDAL/index/BIndexDAL').BIndexDAL)(sqliteDriver); - this.mindexDAL = new (require('./sqliteDAL/index/MIndexDAL').MIndexDAL)(sqliteDriver); - this.iindexDAL = new (require('./sqliteDAL/index/IIndexDAL').IIndexDAL)(sqliteDriver); - this.sindexDAL = new (require('./sqliteDAL/index/SIndexDAL').SIndexDAL)(sqliteDriver); - this.cindexDAL = new (require('./sqliteDAL/index/CIndexDAL').CIndexDAL)(sqliteDriver); - - this.newDals = { - 'metaDAL': that.metaDAL, - 'blockDAL': that.blockDAL, - 'certDAL': that.certDAL, - 'msDAL': that.msDAL, - 'idtyDAL': that.idtyDAL, - 'txsDAL': that.txsDAL, - 'peerDAL': that.peerDAL, - 'confDAL': that.confDAL, - 'statDAL': that.statDAL, - 'walletDAL': that.walletDAL, - 'bindexDAL': that.bindexDAL, - 'mindexDAL': that.mindexDAL, - 'iindexDAL': that.iindexDAL, - 'sindexDAL': that.sindexDAL, - 'cindexDAL': that.cindexDAL - }; - - this.init = (conf) => co(function *() { - const dalNames = _.keys(that.newDals); - for (const dalName of dalNames) { - const dal = that.newDals[dalName]; - yield dal.init(); - } - logger.debug("Upgrade database..."); - yield that.metaDAL.upgradeDatabase(conf); - const latestMember = yield that.iindexDAL.getLatestMember(); - if (latestMember && that.wotb.getWoTSize() > latestMember.wotb_id + 1) { - logger.warn('Maintenance: cleaning wotb...'); - while (that.wotb.getWoTSize() > latestMember.wotb_id + 1) { - that.wotb.removeNode(); - } - } - // Update the maximum certifications count a member can issue into the C++ addon - const currencyParams = yield that.getParameters(); - if (currencyParams && currencyParams.sigStock !== undefined && currencyParams.sigStock !== null) { - that.wotb.setMaxCert(currencyParams.sigStock); - } - }); - - this.getDBVersion = () => that.metaDAL.getVersion(); - - that.writeFileOfBlock = (block) => that.blockDAL.saveBlock(block); - - this.writeSideFileOfBlock = (block) => - that.blockDAL.saveSideBlock(block); - - this.listAllPeers = () => that.peerDAL.listAll(); - - this.getPeer = (pubkey) => co(function*() { - try { - return that.peerDAL.getPeer(pubkey) - } catch (err) { - throw Error('Unknown peer ' + pubkey); - } - }); - - this.getBlock = (number) => co(function*() { - const block = yield that.blockDAL.getBlock(number); - return block || null; - }); - - this.getAbsoluteBlockByNumberAndHash = (number, hash) => - that.blockDAL.getAbsoluteBlock(number, hash); - - this.getBlockByBlockstampOrNull = (blockstamp) => { - if (!blockstamp) throw "Blockstamp is required to find the block"; - const sp = blockstamp.split('-'); - const number = parseInt(sp[0]); - const hash = sp[1]; - return that.getBlockByNumberAndHashOrNull(number, hash); - }; - - this.getBlockByBlockstamp = (blockstamp) => { - if (!blockstamp) throw "Blockstamp is required to find the block"; - const sp = blockstamp.split('-'); - const number = parseInt(sp[0]); - const hash = sp[1]; - return that.getBlockByNumberAndHash(number, hash); - }; - - this.getBlockByNumberAndHash = (number, hash) => co(function*() { - try { - const block = yield that.getBlock(number); - if (!block || block.hash != hash) - throw "Not found"; - else - return block; - } catch (err) { - throw 'Block ' + [number, hash].join('-') + ' not found'; - } - }); - - this.getBlockByNumberAndHashOrNull = (number, hash) => co(function*() { - try { - return yield that.getBlockByNumberAndHash(number, hash); - } catch (e) { - return null; - } - }); - - this.existsNonChainableLink = (from, vHEAD_1, sigStock) => co(function *() { - // Cert period rule - const medianTime = vHEAD_1 ? vHEAD_1.medianTime : 0; - const linksFrom = yield that.cindexDAL.reducablesFrom(from); - const unchainables = _.filter(linksFrom, (link) => link.chainable_on > medianTime); - if (unchainables.length > 0) return true; - // Max stock rule - let activeLinks = _.filter(linksFrom, (link) => !link.expired_on); - return activeLinks.length >= sigStock; - }); - - - this.getCurrentBlockOrNull = () => co(function*() { - let current = null; - try { - current = yield that.getBlockCurrent(); - } catch (e) { - if (e != constants.ERROR.BLOCK.NO_CURRENT_BLOCK) { - throw e; - } - } - return current; - }); - - this.getPromoted = (number) => that.getBlock(number); - - // Block - this.lastUDBlock = () => that.blockDAL.lastBlockWithDividend(); - - this.getRootBlock = () => that.getBlock(0); - - this.lastBlockOfIssuer = function (issuer) { - return that.blockDAL.lastBlockOfIssuer(issuer); - }; - - this.getCountOfPoW = (issuer) => that.blockDAL.getCountOfBlocksIssuedBy(issuer); - - this.getBlocksBetween = (start, end) => Q(this.blockDAL.getBlocks(Math.max(0, start), end)); - - this.getForkBlocksFollowing = (current) => this.blockDAL.getNextForkBlocks(current.number, current.hash); - - this.getBlockCurrent = () => co(function*() { - const current = yield that.blockDAL.getCurrent(); - if (!current) - throw 'No current block'; - return current; - }); - - this.getValidLinksTo = (to) => that.cindexDAL.getValidLinksTo(to); - - this.getAvailableSourcesByPubkey = (pubkey) => this.sindexDAL.getAvailableForPubkey(pubkey); - - this.getIdentityByHashOrNull = (hash) => co(function*() { - const pending = yield that.idtyDAL.getByHash(hash); - if (!pending) { - return that.iindexDAL.getFromHash(hash); - } - return pending; - }); - - this.getMembers = () => that.iindexDAL.getMembers(); - - // TODO: this should definitely be reduced by removing fillInMembershipsOfIdentity - this.getWritten = (pubkey) => co(function*() { - try { - return yield that.fillInMembershipsOfIdentity(that.iindexDAL.getFromPubkey(pubkey)); - } catch (err) { - logger.error(err); - return null; - } - }); - - this.getWrittenIdtyByPubkey = (pubkey) => this.iindexDAL.getFromPubkey(pubkey); - this.getWrittenIdtyByUID = (uid) => this.iindexDAL.getFromUID(uid); - - this.fillInMembershipsOfIdentity = (queryPromise) => co(function*() { - try { - const idty = yield Q(queryPromise); - if (idty) { - const mss = yield that.msDAL.getMembershipsOfIssuer(idty.pubkey); - const mssFromMindex = yield that.mindexDAL.reducable(idty.pubkey); - idty.memberships = mss.concat(mssFromMindex.map((ms) => { - const sp = ms.created_on.split('-'); - return { - blockstamp: ms.created_on, - membership: ms.leaving ? 'OUT' : 'IN', - number: sp[0], - fpr: sp[1], - written_number: parseInt(ms.written_on) - } - })); - return idty; - } - } catch (err) { - logger.error(err); - } - return null; - }); - - this.findPeersWhoseHashIsIn = (hashes) => co(function*() { - const peers = yield that.peerDAL.listAll(); - return _.chain(peers).filter((p) => hashes.indexOf(p.hash) !== -1).value(); - }); - - this.getTxByHash = (hash) => that.txsDAL.getTX(hash); - - this.removeTxByHash = (hash) => that.txsDAL.removeTX(hash); - - this.getTransactionsPending = (versionMin) => that.txsDAL.getAllPending(versionMin); - - this.getNonWritten = (pubkey) => co(function*() { - const pending = yield that.idtyDAL.getPendingIdentities(); - return _.chain(pending).where({pubkey: pubkey}).value(); - }); - - this.getRevocatingMembers = () => co(function *() { - const revoking = yield that.idtyDAL.getToRevoke(); - const toRevoke = []; - for (const pending of revoking) { - const idty = yield that.getWrittenIdtyByPubkey(pending.pubkey); - if (!idty.revoked_on) { - toRevoke.push(pending); - } - } - return toRevoke; - }); - - this.getToBeKickedPubkeys = () => that.iindexDAL.getToBeKickedPubkeys(); - - this.searchJustIdentities = (search) => co(function*() { - const pendings = yield that.idtyDAL.searchThoseMatching(search); - const writtens = yield that.iindexDAL.searchThoseMatching(search); - const nonPendings = _.filter(writtens, (w) => { - return _.where(pendings, { pubkey: w.pub }).length == 0; - }); - const found = pendings.concat(nonPendings); - return yield found.map(f => co(function*() { - const ms = yield that.mindexDAL.getReducedMS(f.pub); - if (ms) { - f.revoked_on = ms.revoked_on ? parseInt(ms.revoked_on) : null; - f.revoked = !!f.revoked_on; - f.revocation_sig = ms.revocation || null; - } - return f; - })) - }); - - this.certsToTarget = (pub, hash) => co(function*() { - const certs = yield that.certDAL.getToTarget(hash); - const links = yield that.cindexDAL.getValidLinksTo(pub); - let matching = certs; - yield links.map((entry) => co(function*() { - entry.from = entry.issuer; - const wbt = entry.written_on.split('-'); - const blockNumber = parseInt(entry.created_on); // created_on field of `c_index` does not have the full blockstamp - const basedBlock = yield that.getBlock(blockNumber); - entry.block = blockNumber; - entry.block_number = blockNumber; - entry.block_hash = basedBlock ? basedBlock.hash : null; - entry.linked = true; - entry.written_block = parseInt(wbt[0]); - entry.written_hash = wbt[1]; - matching.push(entry); - })); - matching = _.sortBy(matching, (c) => -c.block); - matching.reverse(); - return matching; - }); - - this.certsFrom = (pubkey) => co(function*() { - const certs = yield that.certDAL.getFromPubkeyCerts(pubkey); - const links = yield that.cindexDAL.getValidLinksFrom(pubkey); - let matching = certs; - yield links.map((entry) => co(function*() { - const idty = yield that.getWrittenIdtyByPubkey(entry.receiver); - entry.from = entry.issuer; - entry.to = entry.receiver; - const cbt = entry.created_on.split('-'); - const wbt = entry.written_on.split('-'); - entry.block = parseInt(cbt[0]); - entry.block_number = parseInt(cbt[0]); - entry.block_hash = cbt[1]; - entry.target = idty.hash; - entry.linked = true; - entry.written_block = parseInt(wbt[0]); - entry.written_hash = wbt[1]; - matching.push(entry); - })); - matching = _.sortBy(matching, (c) => -c.block); - matching.reverse(); - return matching; - }); - - this.isSentry = (pubkey, conf) => co(function*() { - const current = yield that.getCurrentBlockOrNull(); - if (current) { - const dSen = Math.ceil(Math.pow(current.membersCount, 1 / conf.stepMax)); - const linksFrom = yield that.cindexDAL.getValidLinksFrom(pubkey); - const linksTo = yield that.cindexDAL.getValidLinksTo(pubkey); - return linksFrom.length >= dSen && linksTo.length >= dSen; - } - return false; - }); - - this.certsFindNew = () => co(function*() { - const certs = yield that.certDAL.getNotLinked(); - return _.chain(certs).where({linked: false}).sortBy((c) => -c.block).value(); - }); - - this.certsNotLinkedToTarget = (hash) => co(function*() { - const certs = yield that.certDAL.getNotLinkedToTarget(hash); - return _.chain(certs).sortBy((c) => -c.block).value(); - }); - - this.getMostRecentMembershipNumberForIssuer = (issuer) => co(function*() { - const mss = yield that.msDAL.getMembershipsOfIssuer(issuer); - const reduced = yield that.mindexDAL.getReducedMS(issuer); - let max = reduced ? parseInt(reduced.created_on) : -1; - for (const ms of mss) { - max = Math.max(ms.number, max); - } - return max; - }); - - this.lastJoinOfIdentity = (target) => co(function *() { - let pending = yield that.msDAL.getPendingINOfTarget(target); - return _(pending).sortBy((ms) => -ms.number)[0]; - }); - - this.findNewcomers = (blockMedianTime) => co(function*() { - const pending = yield that.msDAL.getPendingIN() - const mss = yield pending.map(p => co(function*() { - const reduced = yield that.mindexDAL.getReducedMS(p.issuer) - if (!reduced || !reduced.chainable_on || blockMedianTime >= reduced.chainable_on || blockMedianTime < constants.TIME_TO_TURN_ON_BRG_107) { - return p - } - return null - })) - return _.chain(mss) - .filter(ms => ms) - .sortBy((ms) => -ms.sigDate) - .value() - }); - - this.findLeavers = () => co(function*() { - const mss = yield that.msDAL.getPendingOUT(); - return _.chain(mss).sortBy((ms) => -ms.sigDate).value(); - }); - - this.existsNonReplayableLink = (from, to) => this.cindexDAL.existsNonReplayableLink(from, to); - - this.getSource = (identifier, pos) => that.sindexDAL.getSource(identifier, pos); - - this.isMember = (pubkey) => co(function*() { - try { - const idty = yield that.iindexDAL.getFromPubkey(pubkey); - return idty.member; - } catch (err) { - return false; - } - }); - - this.isMemberAndNonLeaver = (pubkey) => co(function*() { - try { - const idty = yield that.iindexDAL.getFromPubkey(pubkey); - if (idty && idty.member) { - return !(yield that.isLeaving(pubkey)); - } - return false; - } catch (err) { - return false; - } - }); - - this.isLeaving = (pubkey) => co(function*() { - const ms = yield that.mindexDAL.getReducedMS(pubkey); - return (ms && ms.leaving) || false; - }); - - this.existsCert = (cert) => co(function*() { - const existing = yield that.certDAL.existsGivenCert(cert); - if (existing) return existing; - const existsLink = yield that.cindexDAL.existsNonReplayableLink(cert.from, cert.to); - return !!existsLink; - }); - - this.deleteCert = (cert) => that.certDAL.deleteCert(cert); - - this.deleteMS = (ms) => that.msDAL.deleteMS(ms); - - this.setRevoked = (pubkey) => co(function*() { - const idty = yield that.getWrittenIdtyByPubkey(pubkey); - idty.revoked = true; - return yield that.idtyDAL.saveIdentity(idty); - }); - - this.setRevocating = (existing, revocation_sig) => co(function *() { - existing.revocation_sig = revocation_sig; - existing.revoked = false; - return that.idtyDAL.saveIdentity(existing); - }); - - this.getPeerOrNull = (pubkey) => co(function*() { - let peer = null; - try { - peer = yield that.getPeer(pubkey); - } catch (e) { - if (e != constants.ERROR.BLOCK.NO_CURRENT_BLOCK) { - throw e; - } - } - return peer; - }); - - this.findAllPeersNEWUPBut = (pubkeys) => co(function*() { - const peers = yield that.listAllPeers(); - return peers.filter((peer) => pubkeys.indexOf(peer.pubkey) == -1 - && ['UP'].indexOf(peer.status) !== -1); - }); - - this.listAllPeersWithStatusNewUP = () => co(function*() { - const peers = yield that.peerDAL.listAll(); - return _.chain(peers) - .filter((p) => ['UP'] - .indexOf(p.status) !== -1).value(); - }); - - this.listAllPeersWithStatusNewUPWithtout = (pub) => co(function *() { - const peers = yield that.peerDAL.listAll(); - return _.chain(peers).filter((p) => p.status == 'UP').filter((p) => p.pubkey !== pub).value(); - }); - - this.findPeers = (pubkey) => co(function*() { - try { - const peer = yield that.getPeer(pubkey); - return [peer]; - } catch (err) { - return []; - } - }); - - this.getRandomlyUPsWithout = (pubkeys) => co(function*() { - const peers = yield that.listAllPeersWithStatusNewUP(); - return peers.filter((peer) => pubkeys.indexOf(peer.pubkey) == -1); - }); - - this.setPeerUP = (pubkey) => co(function *() { - try { - const p = yield that.getPeer(pubkey); - p.status = 'UP'; - p.first_down = null; - p.last_try = null; - return that.peerDAL.savePeer(p); - } catch (err) { - return null; - } - }); - - this.setPeerDown = (pubkey) => co(function *() { - try { - // We do not set mirror peers as down (ex. of mirror: 'M1_HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk') - if (!pubkey.match(/_/)) { - const p = yield that.getPeer(pubkey); - if (p) { - const now = (new Date()).getTime(); - p.status = 'DOWN'; - if (!p.first_down) { - p.first_down = now; - } - p.last_try = now; - yield that.peerDAL.savePeer(p); - } - } - } catch (err) { - throw err; - } - }); - - this.saveBlock = (block) => co(function*() { - block.wrong = false; - yield [ - that.saveBlockInFile(block), - that.saveTxsInFiles(block.transactions, block.number, block.medianTime) - ]; - }); - - this.generateIndexes = (block, conf, index, HEAD) => co(function*() { - // We need to recompute the indexes for block#0 - if (!index || !HEAD || HEAD.number == 0) { - index = indexer.localIndex(block, conf) - HEAD = yield indexer.completeGlobalScope(block, conf, index, that) - } - let mindex = indexer.mindex(index); - let iindex = indexer.iindex(index); - let sindex = indexer.sindex(index); - let cindex = indexer.cindex(index); - sindex = sindex.concat(yield indexer.ruleIndexGenDividend(HEAD, that)); - sindex = sindex.concat(yield indexer.ruleIndexGarbageSmallAccounts(HEAD, sindex, that)); - cindex = cindex.concat(yield indexer.ruleIndexGenCertificationExpiry(HEAD, that)); - mindex = mindex.concat(yield indexer.ruleIndexGenMembershipExpiry(HEAD, that)); - iindex = iindex.concat(yield indexer.ruleIndexGenExclusionByMembership(HEAD, mindex, that)); - iindex = iindex.concat(yield indexer.ruleIndexGenExclusionByCertificatons(HEAD, cindex, iindex, conf, that)); - mindex = mindex.concat(yield indexer.ruleIndexGenImplicitRevocation(HEAD, that)); - yield indexer.ruleIndexCorrectMembershipExpiryDate(HEAD, mindex, that); - yield indexer.ruleIndexCorrectCertificationExpiryDate(HEAD, cindex, that); - return { HEAD, mindex, iindex, sindex, cindex }; - }); - - this.updateWotbLinks = (cindex) => co(function*() { - for (const entry of cindex) { - const from = yield that.getWrittenIdtyByPubkey(entry.issuer); - const to = yield that.getWrittenIdtyByPubkey(entry.receiver); - if (entry.op == common.constants.IDX_CREATE) { - that.wotb.addLink(from.wotb_id, to.wotb_id); - } else { - // Update = removal - that.wotb.removeLink(from.wotb_id, to.wotb_id); - } - } - }); - - this.trimIndexes = (maxNumber) => co(function*() { - yield that.bindexDAL.trimBlocks(maxNumber); - yield that.iindexDAL.trimRecords(maxNumber); - yield that.mindexDAL.trimRecords(maxNumber); - yield that.cindexDAL.trimExpiredCerts(maxNumber); - yield that.sindexDAL.trimConsumedSource(maxNumber); - return true; - }); - - this.trimSandboxes = (block) => co(function*() { - yield that.certDAL.trimExpiredCerts(block.medianTime); - yield that.msDAL.trimExpiredMemberships(block.medianTime); - yield that.idtyDAL.trimExpiredIdentities(block.medianTime); - yield that.txsDAL.trimExpiredNonWrittenTxs(block.medianTime - common.constants.TX_WINDOW); - return true; - }); - - this.savePendingMembership = (ms) => that.msDAL.savePendingMembership(ms); - - this.saveBlockInFile = (block) => co(function *() { - yield that.writeFileOfBlock(block); - }); - - this.saveSideBlockInFile = (block) => that.writeSideFileOfBlock(block); - - this.saveTxsInFiles = (txs, block_number, medianTime) => { - return Q.all(txs.map((tx) => co(function*() { - const sp = tx.blockstamp.split('-'); - tx.blockstampTime = (yield that.getBlockByNumberAndHash(sp[0], sp[1])).medianTime; - const txEntity = new Transaction(tx); - txEntity.computeAllHashes(); - return that.txsDAL.addLinked(TransactionDTO.fromJSONObject(txEntity), block_number, medianTime); - }))); - }; - - this.merkleForPeers = () => co(function *() { - let peers = yield that.listAllPeersWithStatusNewUP(); - const leaves = peers.map((peer) => peer.hash); - const merkle = new Merkle(); - merkle.initialize(leaves); - return merkle; - }); - - this.removeAllSourcesOfBlock = (blockstamp) => that.sindexDAL.removeBlock(blockstamp); - - this.updateTransactions = (txs) => that.txsDAL.insertBatchOfTxs(txs); - - this.savePendingIdentity = (idty) => that.idtyDAL.saveIdentity(idty); - - this.revokeIdentity = (pubkey) => that.idtyDAL.revokeIdentity(pubkey); - - this.removeUnWrittenWithPubkey = (pubkey) => co(function*() { - return yield that.idtyDAL.removeUnWrittenWithPubkey(pubkey) - }); - - this.removeUnWrittenWithUID = (pubkey) => co(function*() { - return yield that.idtyDAL.removeUnWrittenWithUID(pubkey); - }); - - this.registerNewCertification = (cert) => that.certDAL.saveNewCertification(cert); - - this.saveTransaction = (tx) => that.txsDAL.addPending(TransactionDTO.fromJSONObject(tx)) - - this.getTransactionsHistory = (pubkey) => co(function*() { - const history = { - sent: [], - received: [], - sending: [], - receiving: [] - }; - const res = yield [ - that.txsDAL.getLinkedWithIssuer(pubkey), - that.txsDAL.getLinkedWithRecipient(pubkey), - that.txsDAL.getPendingWithIssuer(pubkey), - that.txsDAL.getPendingWithRecipient(pubkey) - ]; - history.sent = res[0] || []; - history.received = res[1] || []; - history.sending = res[2] || []; - history.pending = res[3] || []; - return history; - }); - - this.getUDHistory = (pubkey) => co(function *() { - const sources = yield that.sindexDAL.getUDSources(pubkey); - return { - history: sources.map((src) => _.extend({ - block_number: src.pos, - time: src.written_time - }, src)) - }; - }); - - this.savePeer = (peer) => that.peerDAL.savePeer(peer); - - this.getUniqueIssuersBetween = (start, end) => co(function *() { - const current = yield that.blockDAL.getCurrent(); - const firstBlock = Math.max(0, start); - const lastBlock = Math.max(0, Math.min(current.number, end)); - const blocks = yield that.blockDAL.getBlocks(firstBlock, lastBlock); - return _.chain(blocks).pluck('issuer').uniq().value(); - }); - - /** - * Gets a range of entries for the last `start`th to the last `end`th HEAD entry. - * @param start The starting entry number (min. 1) - * @param end The ending entry (max. BINDEX length) - * @param property If provided, transforms the range of entries into an array of the asked property. - */ - this.range = (start, end, property) => co(function*() { - const range = yield that.bindexDAL.range(start, end); - if (property) { - // Filter on a particular property - return range.map((b) => b[property]); - } else { - return range; - } - }); - - /** - * Get the last `n`th entry from the BINDEX. - * @param n The entry number (min. 1). - */ - this.head = (n) => this.bindexDAL.head(n); - - /*********************** - * CONFIGURATION - **********************/ - - this.getParameters = () => that.confDAL.getParameters(); - - this.loadConf = (overrideConf, defaultConf) => co(function *() { - let conf = Configuration.statics.complete(overrideConf || {}); - if (!defaultConf) { - const savedConf = yield that.confDAL.loadConf(); - conf = _(savedConf).extend(overrideConf || {}); - } - if (that.loadConfHook) { - yield that.loadConfHook(conf); - } - return conf; - }); - - this.saveConf = (confToSave) => { - return co(function*() { - // Save the conf in file - let theConf = confToSave; - if (that.saveConfHook) { - theConf = yield that.saveConfHook(theConf); - } - return that.confDAL.saveConf(theConf); - }); - }; - - /*********************** - * WALLETS - **********************/ - - this.getWallet = (conditions) => co(function*() { - let wallet = yield that.walletDAL.getWallet(conditions) - if (!wallet) { - wallet = { conditions, balance: 0 } - } - return wallet - }) - - this.saveWallet = (wallet) => this.walletDAL.saveWallet(wallet) - - /*********************** - * STATISTICS - **********************/ - - this.loadStats = () => that.statDAL.loadStats(); - this.getStat = (name) => that.statDAL.getStat(name); - this.pushStats = (stats) => that.statDAL.pushStats(stats); - - this.cleanCaches = () => co(function *() { - yield _.values(that.newDals).map((dal) => dal.cleanCache && dal.cleanCache()); - }); - - this.close = () => co(function *() { - yield _.values(that.newDals).map((dal) => dal.cleanCache && dal.cleanCache()); - return sqliteDriver.closeConnection(); - }); - - this.resetPeers = () => co(function *() { - that.peerDAL.removeAll(); - return yield that.close(); - }); - - this.getLogContent = (linesQuantity) => new Promise((resolve, reject) => { - try { - let lines = [], i = 0; - const logPath = require('path').join(rootPath, 'duniter.log'); - const readStream = require('fs').createReadStream(logPath); - readStream.on('error', (err) => reject(err)); - const lineReader = require('readline').createInterface({ - input: readStream - }); - lineReader.on('line', (line) => { - line = "\n" + line; - lines.push(line); - i++; - if (i >= linesQuantity) lines.shift(); - }); - lineReader.on('close', () => resolve(lines)); - lineReader.on('error', (err) => reject(err)); - } catch (e) { - reject(e); - } - }); -} diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts new file mode 100644 index 0000000000000000000000000000000000000000..3258a8a89878bd521e7788d908c20d45aaae85b4 --- /dev/null +++ b/app/lib/dal/fileDAL.ts @@ -0,0 +1,902 @@ +import {SQLiteDriver} from "./drivers/SQLiteDriver"; +import {ConfDAL} from "./fileDALs/ConfDAL"; +import {StatDAL} from "./fileDALs/StatDAL"; +import {ConfDTO} from "../dto/ConfDTO"; +import {BlockDTO} from "../dto/BlockDTO"; +import {DBHead} from "../db/DBHead"; +import {DBIdentity} from "./sqliteDAL/IdentityDAL"; +import {CindexEntry, IindexEntry, IndexEntry, MindexEntry, SindexEntry} from "../indexer"; +import {DBPeer} from "./sqliteDAL/PeerDAL"; +import {TransactionDTO} from "../dto/TransactionDTO"; +import {DBCert} from "./sqliteDAL/CertDAL"; +import {DBWallet} from "./sqliteDAL/WalletDAL"; +import {DBTx} from "./sqliteDAL/TxsDAL"; +import {DBBlock} from "../db/DBBlock"; + +const Q = require('q'); +const fs = require('fs') +const path = require('path') +const readline = require('readline') +const _ = require('underscore'); +const common = require('duniter-common'); +const indexer = require('../indexer').Indexer +const logger = require('../logger')('filedal'); +const Configuration = require('../entity/configuration'); +const Merkle = require('../entity/merkle'); +const Transaction = require('../entity/transaction'); +const constants = require('../constants'); + +export interface FileDALParams { + home:string + fs:any + dbf:() => SQLiteDriver + wotb:any +} + +export class FileDAL { + + rootPath:string + myFS:any + sqliteDriver:SQLiteDriver + wotb:any + profile:string + + confDAL:any + metaDAL:any + peerDAL:any + blockDAL:any + txsDAL:any + statDAL:any + idtyDAL:any + certDAL:any + msDAL:any + walletDAL:any + bindexDAL:any + mindexDAL:any + iindexDAL:any + sindexDAL:any + cindexDAL:any + newDals:any + + loadConfHook: (conf:ConfDTO) => Promise<void> + saveConfHook: (conf:ConfDTO) => Promise<ConfDTO> + + constructor(params:FileDALParams) { + this.rootPath = params.home + this.myFS = params.fs + this.sqliteDriver = params.dbf() + this.wotb = params.wotb + this.profile = 'DAL' + + // DALs + this.confDAL = new ConfDAL(this.rootPath, this.myFS) + this.metaDAL = new (require('./sqliteDAL/MetaDAL').MetaDAL)(this.sqliteDriver); + this.peerDAL = new (require('./sqliteDAL/PeerDAL').PeerDAL)(this.sqliteDriver); + this.blockDAL = new (require('./sqliteDAL/BlockDAL').BlockDAL)(this.sqliteDriver); + this.txsDAL = new (require('./sqliteDAL/TxsDAL').TxsDAL)(this.sqliteDriver); + this.statDAL = new StatDAL(this.rootPath, this.myFS) + this.idtyDAL = new (require('./sqliteDAL/IdentityDAL').IdentityDAL)(this.sqliteDriver); + this.certDAL = new (require('./sqliteDAL/CertDAL').CertDAL)(this.sqliteDriver); + this.msDAL = new (require('./sqliteDAL/MembershipDAL').MembershipDAL)(this.sqliteDriver); + this.walletDAL = new (require('./sqliteDAL/WalletDAL').WalletDAL)(this.sqliteDriver); + this.bindexDAL = new (require('./sqliteDAL/index/BIndexDAL').BIndexDAL)(this.sqliteDriver); + this.mindexDAL = new (require('./sqliteDAL/index/MIndexDAL').MIndexDAL)(this.sqliteDriver); + this.iindexDAL = new (require('./sqliteDAL/index/IIndexDAL').IIndexDAL)(this.sqliteDriver); + this.sindexDAL = new (require('./sqliteDAL/index/SIndexDAL').SIndexDAL)(this.sqliteDriver); + this.cindexDAL = new (require('./sqliteDAL/index/CIndexDAL').CIndexDAL)(this.sqliteDriver); + + this.newDals = { + 'metaDAL': this.metaDAL, + 'blockDAL': this.blockDAL, + 'certDAL': this.certDAL, + 'msDAL': this.msDAL, + 'idtyDAL': this.idtyDAL, + 'txsDAL': this.txsDAL, + 'peerDAL': this.peerDAL, + 'confDAL': this.confDAL, + 'statDAL': this.statDAL, + 'walletDAL': this.walletDAL, + 'bindexDAL': this.bindexDAL, + 'mindexDAL': this.mindexDAL, + 'iindexDAL': this.iindexDAL, + 'sindexDAL': this.sindexDAL, + 'cindexDAL': this.cindexDAL + } + } + + async init(conf:ConfDTO) { + const dalNames = _.keys(this.newDals); + for (const dalName of dalNames) { + const dal = this.newDals[dalName]; + await dal.init(); + } + logger.debug("Upgrade database..."); + await this.metaDAL.upgradeDatabase(conf); + const latestMember = await this.iindexDAL.getLatestMember(); + if (latestMember && this.wotb.getWoTSize() > latestMember.wotb_id + 1) { + logger.warn('Maintenance: cleaning wotb...'); + while (this.wotb.getWoTSize() > latestMember.wotb_id + 1) { + this.wotb.removeNode(); + } + } + // Update the maximum certifications count a member can issue into the C++ addon + const currencyParams = await this.getParameters(); + if (currencyParams && currencyParams.sigStock !== undefined && currencyParams.sigStock !== null) { + this.wotb.setMaxCert(currencyParams.sigStock); + } + } + + getDBVersion() { + return this.metaDAL.getVersion() + } + + writeFileOfBlock(block:DBBlock) { + return this.blockDAL.saveBlock(block) + } + + writeSideFileOfBlock(block:DBBlock) { + return this.blockDAL.saveSideBlock(block) + } + + listAllPeers() { + return this.peerDAL.listAll() + } + + async getPeer(pubkey:string) { + try { + return this.peerDAL.getPeer(pubkey) + } catch (err) { + throw Error('Unknown peer ' + pubkey); + } + } + + async getBlock(number:number) { + const block = await this.blockDAL.getBlock(number) + return block || null; + } + + getAbsoluteBlockByNumberAndHash(number:number, hash:string) { + return this.blockDAL.getAbsoluteBlock(number, hash) + } + + getBlockByBlockstampOrNull(blockstamp:string) { + if (!blockstamp) throw "Blockstamp is required to find the block"; + const sp = blockstamp.split('-'); + const number = parseInt(sp[0]); + const hash = sp[1]; + return this.getBlockByNumberAndHashOrNull(number, hash); + } + + getBlockByBlockstamp(blockstamp:string) { + if (!blockstamp) throw "Blockstamp is required to find the block"; + const sp = blockstamp.split('-'); + const number = parseInt(sp[0]); + const hash = sp[1]; + return this.getBlockByNumberAndHash(number, hash); + } + + async getBlockByNumberAndHash(number:number, hash:string) { + try { + const block = await this.getBlock(number); + if (!block || block.hash != hash) + throw "Not found"; + else + return block; + } catch (err) { + throw 'Block ' + [number, hash].join('-') + ' not found'; + } + } + + async getBlockByNumberAndHashOrNull(number:number, hash:string) { + try { + return await this.getBlockByNumberAndHash(number, hash) + } catch (e) { + return null; + } + } + + async existsNonChainableLink(from:string, vHEAD_1:DBHead, sigStock:number) { + // Cert period rule + const medianTime = vHEAD_1 ? vHEAD_1.medianTime : 0; + const linksFrom = await this.cindexDAL.reducablesFrom(from) + const unchainables = _.filter(linksFrom, (link:CindexEntry) => link.chainable_on > medianTime); + if (unchainables.length > 0) return true; + // Max stock rule + let activeLinks = _.filter(linksFrom, (link:CindexEntry) => !link.expired_on); + return activeLinks.length >= sigStock; + } + + + async getCurrentBlockOrNull() { + let current = null; + try { + current = await this.getBlockCurrent() + } catch (e) { + if (e != constants.ERROR.BLOCK.NO_CURRENT_BLOCK) { + throw e; + } + } + return current; + } + + getPromoted(number:number) { + return this.getBlock(number) + } + + // Block + lastUDBlock() { + return this.blockDAL.lastBlockWithDividend() + } + + getRootBlock() { + return this.getBlock(0) + } + + lastBlockOfIssuer(issuer:string) { + return this.blockDAL.lastBlockOfIssuer(issuer); + } + + getCountOfPoW(issuer:string) { + return this.blockDAL.getCountOfBlocksIssuedBy(issuer) + } + + getBlocksBetween (start:number, end:number) { + return Q(this.blockDAL.getBlocks(Math.max(0, start), end)) + } + + getForkBlocksFollowing(current:DBBlock) { + return this.blockDAL.getNextForkBlocks(current.number, current.hash) + } + + async getBlockCurrent() { + const current = await this.blockDAL.getCurrent(); + if (!current) + throw 'No current block'; + return current; + } + + getValidLinksTo(to:string) { + return this.cindexDAL.getValidLinksTo(to) + } + + getAvailableSourcesByPubkey(pubkey:string) { + return this.sindexDAL.getAvailableForPubkey(pubkey) + } + + async getIdentityByHashOrNull(hash:string) { + const pending = await this.idtyDAL.getByHash(hash); + if (!pending) { + return this.iindexDAL.getFromHash(hash); + } + return pending; + } + + getMembers() { + return this.iindexDAL.getMembers() + } + + // TODO: this should definitely be reduced by removing fillInMembershipsOfIdentity + async getWritten(pubkey:string) { + try { + return await this.fillInMembershipsOfIdentity(this.iindexDAL.getFromPubkey(pubkey)); + } catch (err) { + logger.error(err); + return null; + } + } + + getWrittenIdtyByPubkey(pubkey:string) { + return this.iindexDAL.getFromPubkey(pubkey) + } + + getWrittenIdtyByUID(uid:string) { + return this.iindexDAL.getFromUID(uid) + } + + async fillInMembershipsOfIdentity(queryPromise:Promise<DBIdentity>) { + try { + const idty = await Q(queryPromise); + if (idty) { + const mss = await this.msDAL.getMembershipsOfIssuer(idty.pubkey); + const mssFromMindex = await this.mindexDAL.reducable(idty.pubkey); + idty.memberships = mss.concat(mssFromMindex.map((ms:MindexEntry) => { + const sp = ms.created_on.split('-'); + return { + blockstamp: ms.created_on, + membership: ms.leaving ? 'OUT' : 'IN', + number: sp[0], + fpr: sp[1], + written_number: parseInt(ms.written_on) + } + })); + return idty; + } + } catch (err) { + logger.error(err); + } + return null; + } + + async findPeersWhoseHashIsIn(hashes:string[]) { + const peers = await this.peerDAL.listAll(); + return _.chain(peers).filter((p:DBPeer) => hashes.indexOf(p.hash) !== -1).value(); + } + + getTxByHash(hash:string) { + return this.txsDAL.getTX(hash) + } + + removeTxByHash(hash:string) { + return this.txsDAL.removeTX(hash) + } + + getTransactionsPending(versionMin:number) { + return this.txsDAL.getAllPending(versionMin) + } + + async getNonWritten(pubkey:string) { + const pending = await this.idtyDAL.getPendingIdentities(); + return _.chain(pending).where({pubkey: pubkey}).value(); + } + + async getRevocatingMembers() { + const revoking = await this.idtyDAL.getToRevoke(); + const toRevoke = []; + for (const pending of revoking) { + const idty = await this.getWrittenIdtyByPubkey(pending.pubkey); + if (!idty.revoked_on) { + toRevoke.push(pending); + } + } + return toRevoke; + } + + getToBeKickedPubkeys() { + return this.iindexDAL.getToBeKickedPubkeys() + } + + async searchJustIdentities(search:string) { + const pendings = await this.idtyDAL.searchThoseMatching(search); + const writtens = await this.iindexDAL.searchThoseMatching(search); + const nonPendings = _.filter(writtens, (w:IindexEntry) => { + return _.where(pendings, { pubkey: w.pub }).length == 0; + }); + const found = pendings.concat(nonPendings); + return await Promise.all(found.map(async (f:any) => { + const ms = await this.mindexDAL.getReducedMS(f.pub); + if (ms) { + f.revoked_on = ms.revoked_on ? parseInt(ms.revoked_on) : null; + f.revoked = !!f.revoked_on; + f.revocation_sig = ms.revocation || null; + } + return f; + })) + } + + async certsToTarget(pub:string, hash:string) { + const certs = await this.certDAL.getToTarget(hash); + const links = await this.cindexDAL.getValidLinksTo(pub); + let matching = certs; + await Promise.all(links.map(async (entry:any) => { + entry.from = entry.issuer; + const wbt = entry.written_on.split('-'); + const blockNumber = parseInt(entry.created_on); // created_on field of `c_index` does not have the full blockstamp + const basedBlock = await this.getBlock(blockNumber); + entry.block = blockNumber; + entry.block_number = blockNumber; + entry.block_hash = basedBlock ? basedBlock.hash : null; + entry.linked = true; + entry.written_block = parseInt(wbt[0]); + entry.written_hash = wbt[1]; + matching.push(entry); + })) + matching = _.sortBy(matching, (c:DBCert) => -c.block); + matching.reverse(); + return matching; + } + + async certsFrom(pubkey:string) { + const certs = await this.certDAL.getFromPubkeyCerts(pubkey); + const links = await this.cindexDAL.getValidLinksFrom(pubkey); + let matching = certs; + await Promise.all(links.map(async (entry:any) => { + const idty = await this.getWrittenIdtyByPubkey(entry.receiver); + entry.from = entry.issuer; + entry.to = entry.receiver; + const cbt = entry.created_on.split('-'); + const wbt = entry.written_on.split('-'); + entry.block = parseInt(cbt[0]); + entry.block_number = parseInt(cbt[0]); + entry.block_hash = cbt[1]; + entry.target = idty.hash; + entry.linked = true; + entry.written_block = parseInt(wbt[0]); + entry.written_hash = wbt[1]; + matching.push(entry); + })) + matching = _.sortBy(matching, (c:DBCert) => -c.block); + matching.reverse(); + return matching; + } + + async isSentry(pubkey:string, conf:ConfDTO) { + const current = await this.getCurrentBlockOrNull(); + if (current) { + const dSen = Math.ceil(Math.pow(current.membersCount, 1 / conf.stepMax)); + const linksFrom = await this.cindexDAL.getValidLinksFrom(pubkey); + const linksTo = await this.cindexDAL.getValidLinksTo(pubkey); + return linksFrom.length >= dSen && linksTo.length >= dSen; + } + return false; + } + + async certsFindNew() { + const certs = await this.certDAL.getNotLinked(); + return _.chain(certs).where({linked: false}).sortBy((c:DBCert) => -c.block).value(); + } + + async certsNotLinkedToTarget(hash:string) { + const certs = await this.certDAL.getNotLinkedToTarget(hash); + return _.chain(certs).sortBy((c:any) => -c.block).value(); + } + + async getMostRecentMembershipNumberForIssuer(issuer:string) { + const mss = await this.msDAL.getMembershipsOfIssuer(issuer); + const reduced = await this.mindexDAL.getReducedMS(issuer); + let max = reduced ? parseInt(reduced.created_on) : -1; + for (const ms of mss) { + max = Math.max(ms.number, max); + } + return max; + } + + async lastJoinOfIdentity(target:string) { + let pending = await this.msDAL.getPendingINOfTarget(target); + return _(pending).sortBy((ms:any) => -ms.number)[0]; + } + + async findNewcomers(blockMedianTime:number) { + const pending = await this.msDAL.getPendingIN() + const mss = await Promise.all(pending.map(async (p:any) => { + const reduced = await this.mindexDAL.getReducedMS(p.issuer) + if (!reduced || !reduced.chainable_on || blockMedianTime >= reduced.chainable_on || blockMedianTime < constants.TIME_TO_TURN_ON_BRG_107) { + return p + } + return null + })) + return _.chain(mss) + .filter((ms:any) => ms) + .sortBy((ms:any) => -ms.sigDate) + .value() + } + + async findLeavers() { + const mss = await this.msDAL.getPendingOUT(); + return _.chain(mss).sortBy((ms:any) => -ms.sigDate).value(); + } + + existsNonReplayableLink(from:string, to:string) { + return this.cindexDAL.existsNonReplayableLink(from, to) + } + + getSource(identifier:string, pos:number) { + return this.sindexDAL.getSource(identifier, pos) + } + + async isMember(pubkey:string) { + try { + const idty = await this.iindexDAL.getFromPubkey(pubkey); + return idty.member; + } catch (err) { + return false; + } + } + + async isMemberAndNonLeaver(pubkey:string) { + try { + const idty = await this.iindexDAL.getFromPubkey(pubkey); + if (idty && idty.member) { + return !(await this.isLeaving(pubkey)); + } + return false; + } catch (err) { + return false; + } + } + + async isLeaving(pubkey:string) { + const ms = await this.mindexDAL.getReducedMS(pubkey); + return (ms && ms.leaving) || false; + } + + async existsCert(cert:any) { + const existing = await this.certDAL.existsGivenCert(cert); + if (existing) return existing; + const existsLink = await this.cindexDAL.existsNonReplayableLink(cert.from, cert.to); + return !!existsLink; + } + + deleteCert(cert:any) { + return this.certDAL.deleteCert(cert) + } + + deleteMS(ms:any) { + return this.msDAL.deleteMS(ms) + } + + async setRevoked(pubkey:string) { + const idty = await this.getWrittenIdtyByPubkey(pubkey); + idty.revoked = true; + return await this.idtyDAL.saveIdentity(idty); + } + + setRevocating = (existing:DBIdentity, revocation_sig:string) => { + existing.revocation_sig = revocation_sig; + existing.revoked = false; + return this.idtyDAL.saveIdentity(existing); + } + + async getPeerOrNull(pubkey:string) { + let peer = null; + try { + peer = await this.getPeer(pubkey); + } catch (e) { + if (e != constants.ERROR.BLOCK.NO_CURRENT_BLOCK) { + throw e; + } + } + return peer; + } + + async findAllPeersNEWUPBut(pubkeys:string[]) { + const peers = await this.listAllPeers(); + return peers.filter((peer:DBPeer) => pubkeys.indexOf(peer.pubkey) == -1 + && ['UP'].indexOf(peer.status) !== -1); + } + + async listAllPeersWithStatusNewUP() { + const peers = await this.peerDAL.listAll(); + return _.chain(peers) + .filter((p:DBPeer) => ['UP'] + .indexOf(p.status) !== -1).value(); + } + + async listAllPeersWithStatusNewUPWithtout(pub:string) { + const peers = await this.peerDAL.listAll(); + return _.chain(peers).filter((p:DBPeer) => p.status == 'UP').filter((p:DBPeer) => p.pubkey !== pub).value(); + } + + async findPeers(pubkey:string) { + try { + const peer = await this.getPeer(pubkey); + return [peer]; + } catch (err) { + return []; + } + } + + async getRandomlyUPsWithout(pubkeys:string[]) { + const peers = await this.listAllPeersWithStatusNewUP(); + return peers.filter((peer:DBPeer) => pubkeys.indexOf(peer.pubkey) == -1); + } + + async setPeerUP(pubkey:string) { + try { + const p = await this.getPeer(pubkey) + p.status = 'UP'; + p.first_down = null; + p.last_try = null; + return this.peerDAL.savePeer(p); + } catch (err) { + return null; + } + } + + async setPeerDown(pubkey:string) { + try { + // We do not set mirror peers as down (ex. of mirror: 'M1_HnFcSms8jzwngtVomTTnzudZx7SHUQY8sVE1y8yBmULk') + if (!pubkey.match(/_/)) { + const p = await this.getPeer(pubkey) + if (p) { + const now = (new Date()).getTime(); + p.status = 'DOWN'; + if (!p.first_down) { + p.first_down = now; + } + p.last_try = now; + await this.peerDAL.savePeer(p) + } + } + } catch (err) { + throw err; + } + } + + async saveBlock(block:BlockDTO) { + const dbb = DBBlock.fromBlockDTO(block) + dbb.wrong = false; + await Promise.all([ + this.saveBlockInFile(dbb), + this.saveTxsInFiles(block.transactions, block.number, block.medianTime) + ]) + } + + async generateIndexes(block:DBBlock, conf:ConfDTO, index:IndexEntry[], HEAD:DBHead) { + // We need to recompute the indexes for block#0 + if (!index || !HEAD || HEAD.number == 0) { + index = indexer.localIndex(block, conf) + HEAD = await indexer.completeGlobalScope(block, conf, index, this) + } + let mindex = indexer.mindex(index); + let iindex = indexer.iindex(index); + let sindex = indexer.sindex(index); + let cindex = indexer.cindex(index); + sindex = sindex.concat(await indexer.ruleIndexGenDividend(HEAD, this)); + sindex = sindex.concat(await indexer.ruleIndexGarbageSmallAccounts(HEAD, sindex, this)); + cindex = cindex.concat(await indexer.ruleIndexGenCertificationExpiry(HEAD, this)); + mindex = mindex.concat(await indexer.ruleIndexGenMembershipExpiry(HEAD, this)); + iindex = iindex.concat(await indexer.ruleIndexGenExclusionByMembership(HEAD, mindex, this)); + iindex = iindex.concat(await indexer.ruleIndexGenExclusionByCertificatons(HEAD, cindex, iindex, conf, this)); + mindex = mindex.concat(await indexer.ruleIndexGenImplicitRevocation(HEAD, this)); + await indexer.ruleIndexCorrectMembershipExpiryDate(HEAD, mindex, this); + await indexer.ruleIndexCorrectCertificationExpiryDate(HEAD, cindex, this); + return { HEAD, mindex, iindex, sindex, cindex }; + } + + async updateWotbLinks(cindex:CindexEntry[]) { + for (const entry of cindex) { + const from = await this.getWrittenIdtyByPubkey(entry.issuer); + const to = await this.getWrittenIdtyByPubkey(entry.receiver); + if (entry.op == common.constants.IDX_CREATE) { + this.wotb.addLink(from.wotb_id, to.wotb_id); + } else { + // Update = removal + this.wotb.removeLink(from.wotb_id, to.wotb_id); + } + } + } + + async trimIndexes(maxNumber:number) { + await this.bindexDAL.trimBlocks(maxNumber); + await this.iindexDAL.trimRecords(maxNumber); + await this.mindexDAL.trimRecords(maxNumber); + await this.cindexDAL.trimExpiredCerts(maxNumber); + await this.sindexDAL.trimConsumedSource(maxNumber); + return true; + } + + async trimSandboxes(block:DBBlock) { + await this.certDAL.trimExpiredCerts(block.medianTime); + await this.msDAL.trimExpiredMemberships(block.medianTime); + await this.idtyDAL.trimExpiredIdentities(block.medianTime); + await this.txsDAL.trimExpiredNonWrittenTxs(block.medianTime - common.constants.TX_WINDOW) + return true; + } + + savePendingMembership(ms:any) { + return this.msDAL.savePendingMembership(ms) + } + + async saveBlockInFile(block:DBBlock) { + await this.writeFileOfBlock(block) + } + + saveSideBlockInFile(block:DBBlock) { + return this.writeSideFileOfBlock(block) + } + + async saveTxsInFiles(txs:TransactionDTO[], block_number:number, medianTime:number) { + return Promise.all(txs.map(async (tx) => { + const sp = tx.blockstamp.split('-'); + tx.blockstampTime = (await this.getBlockByNumberAndHash(parseInt(sp[0]), sp[1])).medianTime; + const txEntity = new Transaction(tx); + txEntity.computeAllHashes(); + return this.txsDAL.addLinked(TransactionDTO.fromJSONObject(txEntity), block_number, medianTime); + })) + } + + async merkleForPeers() { + let peers = await this.listAllPeersWithStatusNewUP(); + const leaves = peers.map((peer:DBPeer) => peer.hash); + const merkle = new Merkle(); + merkle.initialize(leaves); + return merkle; + } + + removeAllSourcesOfBlock(blockstamp:string) { + return this.sindexDAL.removeBlock(blockstamp) + } + + updateTransactions(txs:DBTx[]) { + return this.txsDAL.insertBatchOfTxs(txs) + } + + savePendingIdentity(idty:DBIdentity) { + return this.idtyDAL.saveIdentity(idty) + } + + revokeIdentity(pubkey:string) { + return this.idtyDAL.revokeIdentity(pubkey) + } + + async removeUnWrittenWithPubkey(pubkey:string) { + return await this.idtyDAL.removeUnWrittenWithPubkey(pubkey) + } + + async removeUnWrittenWithUID(pubkey:string) { + return await this.idtyDAL.removeUnWrittenWithUID(pubkey); + } + + registerNewCertification(cert:DBCert) { + return this.certDAL.saveNewCertification(cert) + } + + saveTransaction(tx:DBTx) { + return this.txsDAL.addPending(TransactionDTO.fromJSONObject(tx)) + } + + async getTransactionsHistory(pubkey:string) { + const history = { + sent: [], + received: [], + sending: [], + receiving: [], + pending: [] + }; + const res = await Promise.all([ + this.txsDAL.getLinkedWithIssuer(pubkey), + this.txsDAL.getLinkedWithRecipient(pubkey), + this.txsDAL.getPendingWithIssuer(pubkey), + this.txsDAL.getPendingWithRecipient(pubkey) + ]) + history.sent = res[0] || []; + history.received = res[1] || []; + history.sending = res[2] || []; + history.pending = res[3] || []; + return history; + } + + async getUDHistory(pubkey:string) { + const sources = await this.sindexDAL.getUDSources(pubkey) + return { + history: sources.map((src:SindexEntry) => _.extend({ + block_number: src.pos, + time: src.written_time + }, src)) + } + } + + savePeer(peer:DBPeer) { + return this.peerDAL.savePeer(peer) + } + + async getUniqueIssuersBetween(start:number, end:number) { + const current = await this.blockDAL.getCurrent(); + const firstBlock = Math.max(0, start); + const lastBlock = Math.max(0, Math.min(current.number, end)); + const blocks = await this.blockDAL.getBlocks(firstBlock, lastBlock); + return _.chain(blocks).pluck('issuer').uniq().value(); + } + + /** + * Gets a range of entries for the last `start`th to the last `end`th HEAD entry. + * @param start The starting entry number (min. 1) + * @param end The ending entry (max. BINDEX length) + * @param property If provided, transforms the range of entries into an array of the asked property. + */ + async range(start:number, end:number, property:string) { + const range = await this.bindexDAL.range(start, end); + if (property) { + // Filter on a particular property + return range.map((b:any) => b[property]); + } else { + return range; + } + } + + /** + * Get the last `n`th entry from the BINDEX. + * @param n The entry number (min. 1). + */ + head(n:number) { + return this.bindexDAL.head(n) + } + + /*********************** + * CONFIGURATION + **********************/ + + getParameters() { + return this.confDAL.getParameters() + } + + async loadConf(overrideConf:ConfDTO, defaultConf = false) { + let conf = Configuration.statics.complete(overrideConf || {}); + if (!defaultConf) { + const savedConf = await this.confDAL.loadConf(); + conf = _(savedConf).extend(overrideConf || {}); + } + if (this.loadConfHook) { + await this.loadConfHook(conf) + } + return conf; + } + + async saveConf(confToSave:ConfDTO) { + // Save the conf in file + let theConf = confToSave; + if (this.saveConfHook) { + theConf = await this.saveConfHook(theConf) + } + return this.confDAL.saveConf(theConf); + } + + /*********************** + * WALLETS + **********************/ + + async getWallet(conditions:string) { + let wallet = await this.walletDAL.getWallet(conditions) + if (!wallet) { + wallet = { conditions, balance: 0 } + } + return wallet + } + + saveWallet(wallet:DBWallet) { + return this.walletDAL.saveWallet(wallet) + } + + /*********************** + * STATISTICS + **********************/ + + loadStats() { + return this.statDAL.loadStats() + } + + getStat(name:string) { + return this.statDAL.getStat(name) + } + pushStats(stats:any) { + return this.statDAL.pushStats(stats) + } + + async cleanCaches() { + await _.values(this.newDals).map((dal:any) => dal.cleanCache && dal.cleanCache()) + } + + async close() { + await _.values(this.newDals).map((dal:any) => dal.cleanCache && dal.cleanCache()) + return this.sqliteDriver.closeConnection(); + } + + async resetPeers() { + this.peerDAL.removeAll(); + return await this.close() + } + + getLogContent(linesQuantity:number) { + return new Promise((resolve, reject) => { + try { + let lines:string[] = [], i = 0; + const logPath = path.join(this.rootPath, 'duniter.log'); + const readStream = fs.createReadStream(logPath); + readStream.on('error', (err:any) => reject(err)); + const lineReader = readline.createInterface({ + input: readStream + }); + lineReader.on('line', (line:string) => { + line = "\n" + line; + lines.push(line); + i++; + if (i >= linesQuantity) lines.shift(); + }); + lineReader.on('close', () => resolve(lines)); + lineReader.on('error', (err:any) => reject(err)); + } catch (e) { + reject(e); + } + }) + } +} diff --git a/app/lib/dal/sqliteDAL/BlockDAL.ts b/app/lib/dal/sqliteDAL/BlockDAL.ts index 35b38d24107d0645d3086680d560ba04d80ce83b..4c046714c4d647645192d68c69c50a3cc72ef60f 100644 --- a/app/lib/dal/sqliteDAL/BlockDAL.ts +++ b/app/lib/dal/sqliteDAL/BlockDAL.ts @@ -1,44 +1,12 @@ import {AbstractSQLite} from "./AbstractSQLite"; import {SQLiteDriver} from "../drivers/SQLiteDriver"; +import {DBBlock} from "../../db/DBBlock"; const Q = require('q'); const constants = require('../../constants'); const IS_FORK = true; const IS_NOT_FORK = false; -export interface DBBlock { - fork: boolean - hash: string - inner_hash: string - signature: string - currency: string - issuer: string - parameters: string - previousHash: string - previousIssuer: string - version: string - membersCount: string - monetaryMass: string - UDTime: string - medianTime: string - dividend: string - unitbase: string - time: string - powMin: string - number: string - nonce: string - transactions: string - certifications: string - identities: string - joiners: string - actives: string - leavers: string - revoked: string - excluded: string - created: string - updated: string -} - export class BlockDAL extends AbstractSQLite<DBBlock> { private current: any diff --git a/app/lib/db/DBBlock.ts b/app/lib/db/DBBlock.ts index ad283135a58c0f0f93528d848061125ab3bf8d75..0a754d09b363c085de80f7aa328e16ae89e06b6c 100644 --- a/app/lib/db/DBBlock.ts +++ b/app/lib/db/DBBlock.ts @@ -33,6 +33,8 @@ export class DBBlock { parameters: string monetaryMass: number dividend: number | null + UDTime: number + wrong = false constructor( ) { @@ -69,6 +71,8 @@ export class DBBlock { dbb.inner_hash = b.inner_hash dbb.signature = b.signature dbb.nonce = b.nonce + dbb.UDTime = b.UDTime + dbb.monetaryMass = b.monetaryMass return dbb } } \ No newline at end of file diff --git a/app/lib/dto/BlockDTO.ts b/app/lib/dto/BlockDTO.ts index d966371fe052faac6ab6c1327d2d1ccde89f75fb..40fb747a9dcf7afb5d77672265dfc5ef8db0a70b 100644 --- a/app/lib/dto/BlockDTO.ts +++ b/app/lib/dto/BlockDTO.ts @@ -30,6 +30,8 @@ export class BlockDTO { fork: boolean parameters: string signature: string + monetaryMass: number + UDTime: number constructor( ) {} diff --git a/app/lib/indexer.ts b/app/lib/indexer.ts index 785ef39a40845817b91d22e54c3276c93f3c8d67..ba3423f48d2d54bc725041087c0ba1e81bda5d76 100644 --- a/app/lib/indexer.ts +++ b/app/lib/indexer.ts @@ -509,8 +509,8 @@ export class Indexer { const cindex = Indexer.cindex(index); const sindex = Indexer.sindex(index); - const range = dal.range; - const head = dal.head; + const range = (n:number,m:number,p = "") => dal.range(n, m, p) + const head = (n:number) => dal.head(n) const HEAD = new DBHead() diff --git a/app/lib/system/directory.js b/app/lib/system/directory.js index 7a968fcf1ec6b96830246dd53942f4ec8b6493fd..b5da75697f7240f6cc3fe880b572e32953a501b5 100644 --- a/app/lib/system/directory.js +++ b/app/lib/system/directory.js @@ -51,7 +51,7 @@ const dir = module.exports = { yield someDelayFix(); if (isMemory) { params.dbf = () => new SQLiteDriver(':memory:'); - params.wotb = require('../wot').memoryInstance(); + params.wotb = require('../wot').WoTBObject.memoryInstance(); } else { const sqlitePath = path.join(home, dir.DUNITER_DB_NAME + '.db'); params.dbf = () => new SQLiteDriver(sqlitePath); @@ -60,7 +60,7 @@ const dir = module.exports = { if (!existsFile) { fs.closeSync(fs.openSync(wotbFilePath, 'w')); } - params.wotb = require('../wot').fileInstance(wotbFilePath); + params.wotb = require('../wot').WoTBObject.fileInstance(wotbFilePath); } return params; }), diff --git a/app/lib/wot.js b/app/lib/wot.js deleted file mode 100644 index f18ab977defb7aae3ba4ff9349003b8c4e2e0ef5..0000000000000000000000000000000000000000 --- a/app/lib/wot.js +++ /dev/null @@ -1,10 +0,0 @@ -"use strict"; - -const wotb = require('wotb'); - -module.exports = { - - fileInstance: (filepath) => wotb.newFileInstance(filepath), - memoryInstance: () => wotb.newMemoryInstance(), - setVerbose: wotb.setVerbose -}; diff --git a/app/lib/wot.ts b/app/lib/wot.ts new file mode 100644 index 0000000000000000000000000000000000000000..bcfafa8fd1932298aaa9e4b27169669aeedcb093 --- /dev/null +++ b/app/lib/wot.ts @@ -0,0 +1,14 @@ +const wotb = require('wotb'); + +export interface WoTBInterface { + fileInstance: (filepath:string) => any + memoryInstance: (filepath:string) => any + setVerbose: (verbose:boolean) => void +} + +export const WoTBObject:WoTBInterface = { + + fileInstance: (filepath:string) => wotb.newFileInstance(filepath), + memoryInstance: () => wotb.newMemoryInstance(), + setVerbose: wotb.setVerbose +} diff --git a/doc/contribute-french.md b/doc/contribute-french.md index d7cc4839e1a3e158187eaf220755a73fde3526b7..a19ba9b29d596b35bb158d26a98b827c07ca609e 100644 --- a/doc/contribute-french.md +++ b/doc/contribute-french.md @@ -1094,7 +1094,7 @@ Réessayez avec une valeur autre que `abc` pour voir la valeur changer au niveau ### Voir le nombre d’identités trouvées -Regardez la ligne 27 du fichier wot.js que nous débogons actuellement : +Regardez la ligne 27 du fichier wot.ts que nous débogons actuellement : ```js const identities = yield IdentityService.searchIdentities(search); diff --git a/server.js b/server.js index fbedf48c9e1953226fc7aae457c8921c495fbeb4..3d8a77bb8a45bc338a3fdb343a1109f989763bae 100644 --- a/server.js +++ b/server.js @@ -11,7 +11,7 @@ const fs = require('fs'); const daemonize = require("daemonize2") const parsers = require('duniter-common').parsers; const constants = require('./app/lib/constants'); -const fileDAL = require('./app/lib/dal/fileDAL'); +const FileDAL = require('./app/lib/dal/fileDAL').FileDAL const jsonpckg = require('./package.json'); const keyring = require('duniter-common').keyring; const directory = require('./app/lib/system/directory'); @@ -70,7 +70,7 @@ function Server (home, memoryOnly, overrideConf) { this.plugFileSystem = () => co(function *() { logger.debug('Plugging file system...'); const params = yield paramsP; - that.dal = fileDAL(params); + that.dal = new FileDAL(params); yield that.onPluggedFSHook() }); diff --git a/test/dal/dal.js b/test/dal/dal.js index 6c7507a107f8a2a2533eabfcf436c31fc1f8942a..07f85bef3eb4b186a3eb0f4b8e3b0f24df092a1c 100644 --- a/test/dal/dal.js +++ b/test/dal/dal.js @@ -3,7 +3,7 @@ var co = require('co'); var _ = require('underscore'); var should = require('should'); var assert = require('assert'); -var dal = require('../../app/lib/dal/fileDAL'); +var dal = require('../../app/lib/dal/fileDAL').FileDAL var dir = require('../../app/lib/system/directory'); var constants = require('../../app/lib/constants'); var Peer = require('../../app/lib/entity/peer'); @@ -95,7 +95,7 @@ describe("DAL", function(){ before(() => co(function *() { let params = yield dir.getHomeParams(true, 'db0'); - fileDAL = dal(params); + fileDAL = new dal(params); yield fileDAL.init(); return fileDAL.saveConf({ currency: "meta_brouzouf" }); })); @@ -114,7 +114,7 @@ describe("DAL", function(){ it('should have 1 peer if 1 is created', function(){ return fileDAL.savePeer(new Peer(mocks.peer1)) - .then(fileDAL.listAllPeers) + .then(() => fileDAL.listAllPeers()) .then(function(peers){ peers.should.have.length(1); peers[0].should.have.property('pubkey').equal('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd'); diff --git a/test/dal/source_dal.js b/test/dal/source_dal.js index 1d98a1a994e72a63a9789c22abecf3b7e0d97b08..f117c41aaf535a39c17c068361cdb4524a7b7c30 100644 --- a/test/dal/source_dal.js +++ b/test/dal/source_dal.js @@ -1,7 +1,7 @@ "use strict"; const co = require('co'); const should = require('should'); -const FileDAL = require('../../app/lib/dal/fileDAL'); +const FileDAL = require('../../app/lib/dal/fileDAL').FileDAL const dir = require('../../app/lib/system/directory'); const indexer = require('../../app/lib/indexer').Indexer const toolbox = require('../integration/tools/toolbox'); @@ -11,7 +11,7 @@ let dal; describe("Source DAL", function(){ before(() => co(function *() { - dal = FileDAL(yield dir.getHomeParams(true, 'db0')); + dal = new FileDAL(yield dir.getHomeParams(true, 'db0')); yield dal.init(); })); diff --git a/test/dal/triming.js b/test/dal/triming.js index 5d61ef39338f56943cf8e8f1edfbbab4c2b2c195..000173abb5a0ade31a0d74ecd435514ce5f70bda 100644 --- a/test/dal/triming.js +++ b/test/dal/triming.js @@ -1,7 +1,7 @@ "use strict"; const co = require('co'); const should = require('should'); -const FileDAL = require('../../app/lib/dal/fileDAL'); +const FileDAL = require('../../app/lib/dal/fileDAL').FileDAL const dir = require('../../app/lib/system/directory'); const indexer = require('../../app/lib/indexer').Indexer const toolbox = require('../integration/tools/toolbox'); @@ -11,7 +11,7 @@ let dal; describe("Triming", function(){ before(() => co(function *() { - dal = FileDAL(yield dir.getHomeParams(true, 'db0')); + dal = new FileDAL(yield dir.getHomeParams(true, 'db0')); yield dal.init(); }));