Commit 05e89d84 authored by Cédric Moreau's avatar Cédric Moreau
Browse files

[enh] #1022 Reintroduce duniter-prover into Duniter core

parent 15eb7659
......@@ -54,7 +54,6 @@ If you wish to participate/debate on Duniter, you can:
Duniter is using modules on different git repositories:
- [Common](https://github.com/duniter/duniter-common): commons tools for Duniter core and modules.
- [Crawler](https://github.com/duniter/duniter-crawler): network crawler.
- [Prover](https://github.com/duniter/duniter-prover): handle Proof-of-Work.
- [BMA API](https://github.com/duniter/duniter-bma): Basic Merkled API.
- [Keypair](https://github.com/duniter/duniter-keypair): provide the cryptographic keypair.
- [WotB](https://github.com/duniter/wotb): compute Web of Trust.
......
"use strict";
const co = require('co');
const async = require('async');
const contacter = require('duniter-crawler').duniter.methods.contacter;
const common = require('duniter-common');
const constants = require('./lib/constants');
const Prover = require('./lib/prover');
const blockGenerator = require('./lib/blockGenerator');
const blockProver = require('./lib/blockProver');
const Peer = common.document.Peer
module.exports = {
duniter: {
/*********** Permanent prover **************/
config: {
onLoading: (conf) => co(function*() {
if (conf.cpu === null || conf.cpu === undefined) {
conf.cpu = constants.DEFAULT_CPU;
}
conf.powSecurityRetryDelay = constants.POW_SECURITY_RETRY_DELAY;
conf.powMaxHandicap = constants.POW_MAXIMUM_ACCEPTABLE_HANDICAP;
}),
beforeSave: (conf) => co(function*() {
delete conf.powSecurityRetryDelay;
delete conf.powMaxHandicap;
})
},
service: {
output: (server, conf, logger) => {
const generator = blockGenerator(server);
server.generatorGetJoinData = generator.getSinglePreJoinData.bind(generator)
server.generatorComputeNewCerts = generator.computeNewCerts.bind(generator)
server.generatorNewCertsToLinks = generator.newCertsToLinks.bind(generator)
return new Prover(server, conf, logger)
}
},
methods: {
hookServer: (server) => {
const generator = blockGenerator(server);
server.generatorGetJoinData = generator.getSinglePreJoinData.bind(generator)
server.generatorComputeNewCerts = generator.computeNewCerts.bind(generator)
server.generatorNewCertsToLinks = generator.newCertsToLinks.bind(generator)
},
blockProver: blockProver,
prover: (server, conf, logger) => new Prover(server, conf, logger),
blockGenerator: (server, prover) => blockGenerator(server, prover),
generateTheNextBlock: (server, manualValues) => co(function*() {
const prover = blockProver(server);
const generator = blockGenerator(server, prover);
return generator.nextBlock(manualValues);
}),
generateAndProveTheNext: (server, block, trial, manualValues) => co(function*() {
const prover = blockProver(server);
const generator = blockGenerator(server, prover);
let res = yield generator.makeNextBlock(block, trial, manualValues);
return res
})
},
/*********** CLI gen-next + gen-root **************/
cliOptions: [
{value: '--show', desc: 'With gen-next or gen-root commands, displays the generated block.'},
{value: '--check', desc: 'With gen-next: just check validity of generated block.'},
{value: '--at <medianTime>', desc: 'With gen-next --show --check: allows to try in a future time.', parser: parseInt }
],
cli: [{
name: 'gen-next [host] [port] [difficulty]',
desc: 'Tries to generate the next block of the blockchain.',
onDatabaseExecute: (server, conf, program, params) => co(function*() {
const host = params[0];
const port = params[1];
const difficulty = params[2];
const generator = blockGenerator(server, null);
return generateAndSend(program, host, port, difficulty, server, () => generator.nextBlock);
})
}, {
name: 'gen-root [host] [port] [difficulty]',
desc: 'Tries to generate the next block of the blockchain.',
preventIfRunning: true,
onDatabaseExecute: (server, conf, program, params) => co(function*() {
const host = params[0];
const port = params[1];
const difficulty = params[2];
const generator = blockGenerator(server, null);
let toDelete, catched = true;
do {
try {
yield generateAndSend(program, host, port, difficulty, server, () => generator.nextBlock);
catched = false;
} catch (e) {
toDelete = yield server.dal.idtyDAL.query('SELECT * FROM idty i WHERE 5 > (SELECT count(*) from cert c where c.`to` = i.pubkey)');
console.log('Deleting', toDelete.map(i => i.pubkey));
yield server.dal.idtyDAL.exec('DELETE FROM idty WHERE pubkey IN (' + toDelete.map(i => "'" + i.pubkey + "'").join(',') + ')');
yield server.dal.idtyDAL.exec('DELETE FROM cert WHERE `to` IN (' + toDelete.map(i => "'" + i.pubkey + "'").join(',') + ')');
yield server.dal.idtyDAL.exec('DELETE FROM cert WHERE `from` IN (' + toDelete.map(i => "'" + i.pubkey + "'").join(',') + ')');
}
} while (catched && toDelete.length);
console.log('Done');
})
}, {
name: 'gen-root-choose [host] [port] [difficulty]',
desc: 'Tries to generate root block, with choice of root members.',
preventIfRunning: true,
onDatabaseExecute: (server, conf, program, params, startServices, stopServices) => co(function*() {
const host = params[0];
const port = params[1];
const difficulty = params[2];
if (!host) {
throw 'Host is required.';
}
if (!port) {
throw 'Port is required.';
}
if (!difficulty) {
throw 'Difficulty is required.';
}
const generator = blockGenerator(server, null);
return generateAndSend(program, host, port, difficulty, server, () => generator.manualRoot);
})
}]
}
}
function generateAndSend(program, host, port, difficulty, server, getGenerationMethod) {
const logger = server.logger;
return new Promise((resolve, reject) => {
async.waterfall([
function (next) {
const method = getGenerationMethod(server);
co(function*(){
const simulationValues = {}
if (program.show && program.check) {
if (program.at && !isNaN(program.at)) {
simulationValues.medianTime = program.at
}
}
const block = yield method(null, simulationValues);
next(null, block);
});
},
function (block, next) {
if (program.check) {
block.time = block.medianTime;
program.show && console.log(block.getRawSigned());
co(function*(){
try {
const parsed = common.parsers.parseBlock.syncWrite(block.getRawSigned());
yield server.BlockchainService.checkBlock(parsed, false);
logger.info('Acceptable block');
next();
} catch (e) {
next(e);
}
});
}
else {
logger.debug('Block to be sent: %s', block.getRawInnerPart());
async.waterfall([
function (subNext) {
proveAndSend(program, server, block, server.conf.pair.pub, parseInt(difficulty), host, parseInt(port), subNext);
}
], next);
}
}
], (err, data) => {
err && reject(err);
!err && resolve(data);
});
});
}
function proveAndSend(program, server, block, issuer, difficulty, host, port, done) {
const logger = server.logger;
async.waterfall([
function (next) {
block.issuer = issuer;
program.show && console.log(block.getRawSigned());
co(function*(){
try {
const prover = blockProver(server);
const proven = yield prover.prove(block, difficulty);
const peer = Peer.fromJSON({
endpoints: [['BASIC_MERKLED_API', host, port].join(' ')]
});
program.show && console.log(proven.getRawSigned());
logger.info('Posted block ' + proven.getRawSigned());
const p = Peer.fromJSON(peer);
const contact = contacter(p.getHostPreferDNS(), p.getPort());
yield contact.postBlock(proven.getRawSigned());
} catch(e) {
next(e);
}
});
}
], done);
}
This diff is collapsed.
"use strict";
const co = require('co');
const engine = require('./engine');
const querablep = require('querablep');
const common = require('duniter-common');
const constants = require('./constants');
const Block = common.document.Block
const POW_FOUND = true;
const POW_NOT_FOUND_YET = false;
module.exports = (server) => new BlockProver(server);
function BlockProver(server) {
let conf = server.conf;
let pair = conf.pair;
let logger = server.logger;
let waitResolve;
let workerFarmPromise;
function getWorker() {
return (workerFarmPromise || (workerFarmPromise = co(function*() {
return new WorkerFarm();
})));
}
const debug = process.execArgv.toString().indexOf('--debug') !== -1;
if(debug) {
//Set an unused port number.
process.execArgv = [];
}
this.cancel = (gottenBlock) => co(function*() {
// If no farm was instanciated, there is nothing to do yet
if (workerFarmPromise) {
let farm = yield getWorker();
if (farm.isComputing() && !farm.isStopping()) {
yield farm.stopPoW(gottenBlock);
}
if (waitResolve) {
waitResolve();
waitResolve = null;
}
}
});
this.prove = function (block, difficulty, forcedTime) {
if (waitResolve) {
waitResolve();
waitResolve = null;
}
const remainder = difficulty % 16;
const nbZeros = (difficulty - remainder) / 16;
const highMark = common.constants.PROOF_OF_WORK.UPPER_BOUND[remainder];
return co(function*() {
let powFarm = yield getWorker();
if (block.number == 0) {
// On initial block, difficulty is the one given manually
block.powMin = difficulty;
}
// Start
powFarm.setOnAlmostPoW(function(pow, matches, aBlock, found) {
powEvent(found, pow);
if (matches && matches[1].length >= constants.MINIMAL_ZEROS_TO_SHOW_IN_LOGS) {
logger.info('Matched %s zeros %s with Nonce = %s for block#%s by %s', matches[1].length, pow, aBlock.nonce, aBlock.number, aBlock.issuer.slice(0,6));
}
});
block.nonce = 0;
logger.info('Generating proof-of-work with %s leading zeros followed by [0-' + highMark + ']... (CPU usage set to %s%) for block#%s', nbZeros, (conf.cpu * 100).toFixed(0), block.number, block.issuer.slice(0,6));
const start = Date.now();
let result = yield powFarm.askNewProof({
newPoW: { conf: conf, block: block, zeros: nbZeros, highMark: highMark, forcedTime: forcedTime, pair }
});
if (!result) {
logger.info('GIVEN proof-of-work for block#%s with %s leading zeros followed by [0-' + highMark + ']! stop PoW for %s', block.number, nbZeros, pair.pub.slice(0,6));
throw 'Proof-of-work computation canceled because block received';
} else {
const proof = result.block;
const testsCount = result.testsCount;
const duration = (Date.now() - start);
const testsPerSecond = (testsCount / (duration / 1000)).toFixed(2);
logger.info('Done: #%s, %s in %ss (%s tests, ~%s tests/s)', block.number, proof.hash, (duration / 1000).toFixed(2), testsCount, testsPerSecond);
logger.info('FOUND proof-of-work with %s leading zeros followed by [0-' + highMark + ']!', nbZeros);
return Block.fromJSON(proof);
}
});
};
this.changeCPU = (cpu) => co(function*() {
conf.cpu = cpu;
const farm = yield getWorker();
return farm.changeCPU(cpu);
});
this.changePoWPrefix = (prefix) => co(function*() {
const farm = yield getWorker();
return farm.changePoWPrefix(prefix);
});
function powEvent(found, hash) {
server && server.push({ pow: { found, hash } });
}
function WorkerFarm() {
// Create
const theEngine = engine(server.conf, server.logger)
let onAlmostPoW
// An utility method to filter the pow notifications
const checkPoWandNotify = (hash, block, found) => {
const matches = hash.match(/^(0{2,})[^0]/);
if (matches && onAlmostPoW) {
onAlmostPoW(hash, matches, block, found);
}
}
// Keep track of PoW advancement
theEngine.setOnInfoMessage((message) => {
if (message.error) {
logger.error('Error in engine#%s:', theEngine.id, message.error)
} else if (message.pow) {
// A message about the PoW
const msg = message.pow
checkPoWandNotify(msg.pow, msg.block, POW_NOT_FOUND_YET)
}
})
// We use as much cores as available, but not more than CORES_MAXIMUM_USE_IN_PARALLEL
let powPromise = null
let stopPromise = null
this.changeCPU = (cpu) => theEngine.setConf({ cpu })
this.changePoWPrefix = (prefix) => theEngine.setConf({ prefix })
this.isComputing = () => powPromise !== null && !powPromise.isResolved()
this.isStopping = () => stopPromise !== null && !stopPromise.isResolved()
/**
* Eventually stops the engine PoW if one was computing
*/
this.stopPoW = (gottenBlock) => {
stopPromise = querablep(theEngine.cancel(gottenBlock))
return stopPromise;
};
/**
* Starts a new computation of PoW
* @param stuff The necessary data for computing the PoW
*/
this.askNewProof = (stuff) => co(function*() {
// Starts the PoW
powPromise = querablep(theEngine.prove(stuff))
const res = yield powPromise
if (res) {
checkPoWandNotify(res.pow.pow, res.pow.block, POW_FOUND);
}
return res && res.pow
})
this.setOnAlmostPoW = (onPoW) => onAlmostPoW = onPoW
}
}
"use strict";
module.exports = {
PULLING_MAX_DURATION: 10 * 1000, // 10 seconds
CORES_MAXIMUM_USE_IN_PARALLEL: 8,
MINIMAL_ZEROS_TO_SHOW_IN_LOGS: 3,
POW_MINIMAL_TO_SHOW: 2,
DEFAULT_CPU: 0.6,
NONCE_RANGE: 1000 * 1000 * 1000 * 100,
POW_MAXIMUM_ACCEPTABLE_HANDICAP: 64,
// When to trigger the PoW process again if no PoW is triggered for a while. In milliseconds.
POW_SECURITY_RETRY_DELAY: 10 * 60 * 1000
};
"use strict";
const os = require('os')
const co = require('co')
const querablep = require('querablep')
const powCluster = require('./powCluster')
const constants = require('./constants')
module.exports = function (conf, logger) {
return new PowEngine(conf, logger);
};
function PowEngine(conf, logger) {
// Super important for Node.js debugging
const debug = process.execArgv.toString().indexOf('--debug') !== -1;
if(debug) {
//Set an unused port number.
process.execArgv = [];
}
const nbWorkers = require('os').cpus().slice(0, conf && conf.nbCores || constants.CORES_MAXIMUM_USE_IN_PARALLEL).length
const cluster = powCluster(nbWorkers, logger)
this.forceInit = () => cluster.initCluster()
this.id = cluster.clusterId
this.prove = (stuff) => co(function*() {
if (cluster.hasProofPending) {
yield cluster.cancelWork()
}
if (os.arch().match(/arm/)) {
stuff.conf.cpu /= 2; // Don't know exactly why is ARM so much saturated by PoW, so let's divide by 2
}
let res = yield cluster.proveByWorkers(stuff)
return res
})
this.cancel = () => cluster.cancelWork()
this.setConf = (value) => cluster.changeConf(value)
this.setOnInfoMessage = (callback) => cluster.onInfoMessage = callback
}
"use strict";
const co = require('co');
const querablep = require('querablep');
const common = require('duniter-common');
const constants = require('./constants');
const blockProver = require('./blockProver');
const blockGenerator = require('./blockGenerator');
module.exports = (server) => new PermanentProver(server);
function PermanentProver(server) {
const dos2unix = common.dos2unix;
const parsers = common.parsers;
const logger = server.logger;
const conf = server.conf;
const prover = this.prover = blockProver(server);
const generator = blockGenerator(server, prover);
const that = this;
let blockchainChangedResolver = null,
promiseOfWaitingBetween2BlocksOfOurs = null,
lastComputedBlock = null;
// Promises triggering the prooving lopp
let resolveContinuePromise = null;
let continuePromise = new Promise((resolve) => resolveContinuePromise = resolve);
let pullingResolveCallback = null;
let timeoutPullingCallback = null, timeoutPulling;
let pullingFinishedPromise = querablep(Promise.resolve());
this.allowedToStart = () => {
resolveContinuePromise(true);
};
// When we detected a pulling, we stop the PoW loop
this.pullingDetected = () => {
if (pullingFinishedPromise.isResolved()) {
pullingFinishedPromise = querablep(Promise.race([
// We wait for end of pulling signal
new Promise((res) => pullingResolveCallback = res),
// Security: if the end of pulling signal is not emitted after some, we automatically trigger it
new Promise((res) => timeoutPullingCallback = () => {
logger.warn('Pulling not finished after %s ms, continue PoW', constants.PULLING_MAX_DURATION);
res();
})
]));
}
// Delay the triggering of pulling timeout
if (timeoutPulling) {
clearTimeout(timeoutPulling);
}
timeoutPulling = setTimeout(timeoutPullingCallback, constants.PULLING_MAX_DURATION);
};
this.pullingFinished = () => pullingResolveCallback && pullingResolveCallback();
this.loops = 0;
/******************
* Main proof loop
*****************/
co(function*() {
while (yield continuePromise) {
try {
const waitingRaces = [];
// By default, we do not make a new proof
let doProof = false;
try {
const selfPubkey = server.keyPair.publicKey;
const dal = server.dal;
const theConf = server.conf;
if (!selfPubkey) {
throw 'No self pubkey found.';
}
let current;
const isMember = yield dal.isMember(selfPubkey);
if (!isMember) {
throw 'Local node is not a member. Waiting to be a member before computing a block.';
}
current = yield dal.getCurrentBlockOrNull();
if (!current) {
throw 'Waiting for a root block before computing new blocks';
}
const trial = yield server.getBcContext().getIssuerPersonalizedDifficulty(selfPubkey);
checkTrialIsNotTooHigh(trial, current, selfPubkey);
const lastIssuedByUs = current.issuer == selfPubkey;
if (pullingFinishedPromise && !pullingFinishedPromise.isFulfilled()) {
logger.warn('Waiting for the end of pulling...');
yield pullingFinishedPromise;
logger.warn('Pulling done. Continue proof-of-work loop.');
}
if (lastIssuedByUs && !promiseOfWaitingBetween2BlocksOfOurs) {
promiseOfWaitingBetween2BlocksOfOurs = new Promise((resolve) => setTimeout(resolve, theConf.powDelay));
logger.warn('Waiting ' + theConf.powDelay + 'ms before starting to compute next block...');
} else {
// We have waited enough
promiseOfWaitingBetween2BlocksOfOurs = null;
// But under some conditions, we can make one
doProof = true;
}
} catch (e) {
logger.warn(e);
}
if (doProof) {
/*******************
* COMPUTING A BLOCK
******************/
yield Promise.race([
// We still listen at eventual blockchain change
co(function*() {
// If the blockchain changes
yield new Promise((resolve) => blockchainChangedResolver = resolve);
// Then cancel the generation