dev to reduce_cpu · Pascal Engélibert / duniter

app/cli.ts

+2
−0

Original line number
Diff line number
Diff line

@@ -41,6 +41,8 @@ export const ExecuteCommand = () => {

        .option('--addep <endpoint>', 'With `config` command, add given endpoint to the list of endpoints of this node')

        .option('--remep <endpoint>', 'With `config` command, remove given endpoint to the list of endpoints of this node')

        .option('--eco-mode', 'reduce CPU usage for proof-of-work computation')

        .option('--no-eco-mode', 'Do not reduce CPU usage for proof-of-work computation')

        .option('--cpu <percent>', 'Percent of CPU usage for proof-of-work computation', parsePercent)

        .option('--nb-cores <number>', 'Number of cores uses for proof-of-work computation', parseInt)

        .option('--prefix <nodeId>', 'Prefix node id for the first character of nonce', parseInt)

app/lib/computation/BlockchainContext.ts

+10
−0

Original line number
Diff line number
Diff line

@@ -93,6 +93,16 @@ export class BlockchainContext {

    return local_vHEAD.issuerDiff;

  }

  /**

   * Utility method: gives the number of block in the current frame for a given issuer.

   * @param issuer The issuer we want.

   */

  async getIssuerNbBlockInFrame(issuer: string): Promise<number> {

    const local_vHEAD = await this.getvHeadCopy({ issuer })

    let issuerNbBlockInFrame = await indexer.preparePersonalizedPoW(local_vHEAD, this.vHEAD_1, (n:number, m:number, p = "") => this.dal.range(n,m,p), this.conf)

    return issuerNbBlockInFrame

  }

  setConfDAL(newConf: any, newDAL: any, theBlockchain: DuniterBlockchain, theQuickSynchronizer: QuickSynchronizer): void {

    this.dal = newDAL;

    this.conf = newConf;

app/lib/dto/ConfDTO.ts

+2
−1

Original line number
Diff line number
Diff line

@@ -91,6 +91,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,

    public rmEndpoints: string[],

    public rootoffset: number,

    public upInterval: number,

    public ecoMode: boolean|undefined,

    public cpu: number,

    public nbCores: number,

    public prefix: number,

@@ -162,7 +163,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,

) {}

  static mock() {

    return new ConfDTO("", "", [], [], 0, 3600 * 1000, constants.PROOF_OF_WORK.DEFAULT.CPU, 1, constants.PROOF_OF_WORK.DEFAULT.PREFIX, 0, 0, constants.CONTRACT.DEFAULT.C, constants.CONTRACT.DEFAULT.DT, constants.CONTRACT.DEFAULT.DT_REEVAL, 0, constants.CONTRACT.DEFAULT.UD0, 0, 0, constants.CONTRACT.DEFAULT.STEPMAX, constants.CONTRACT.DEFAULT.SIGPERIOD, 0, constants.CONTRACT.DEFAULT.SIGVALIDITY, constants.CONTRACT.DEFAULT.MSVALIDITY, constants.CONTRACT.DEFAULT.SIGQTY, constants.CONTRACT.DEFAULT.SIGSTOCK, constants.CONTRACT.DEFAULT.X_PERCENT, constants.CONTRACT.DEFAULT.PERCENTROT, constants.CONTRACT.DEFAULT.POWDELAY, constants.CONTRACT.DEFAULT.AVGGENTIME, constants.CONTRACT.DEFAULT.MEDIANTIMEBLOCKS, false, 3000, false, constants.BRANCHES.DEFAULT_WINDOW_SIZE, constants.CONTRACT.DEFAULT.IDTYWINDOW, constants.CONTRACT.DEFAULT.MSWINDOW, constants.CONTRACT.DEFAULT.SIGWINDOW, 0, { pub:'', sec:'' }, null, "", "", 0, "", "", "", "", 0, "", "", null, false, "", true, true, false, new ProxiesConf(), undefined)

    return new ConfDTO("", "", [], [], 0, 3600 * 1000, true, constants.PROOF_OF_WORK.DEFAULT.CPU, 1, constants.PROOF_OF_WORK.DEFAULT.PREFIX, 0, 0, constants.CONTRACT.DEFAULT.C, constants.CONTRACT.DEFAULT.DT, constants.CONTRACT.DEFAULT.DT_REEVAL, 0, constants.CONTRACT.DEFAULT.UD0, 0, 0, constants.CONTRACT.DEFAULT.STEPMAX, constants.CONTRACT.DEFAULT.SIGPERIOD, 0, constants.CONTRACT.DEFAULT.SIGVALIDITY, constants.CONTRACT.DEFAULT.MSVALIDITY, constants.CONTRACT.DEFAULT.SIGQTY, constants.CONTRACT.DEFAULT.SIGSTOCK, constants.CONTRACT.DEFAULT.X_PERCENT, constants.CONTRACT.DEFAULT.PERCENTROT, constants.CONTRACT.DEFAULT.POWDELAY, constants.CONTRACT.DEFAULT.AVGGENTIME, constants.CONTRACT.DEFAULT.MEDIANTIMEBLOCKS, false, 3000, false, constants.BRANCHES.DEFAULT_WINDOW_SIZE, constants.CONTRACT.DEFAULT.IDTYWINDOW, constants.CONTRACT.DEFAULT.MSWINDOW, constants.CONTRACT.DEFAULT.SIGWINDOW, 0, { pub:'', sec:'' }, null, "", "", 0, "", "", "", "", 0, "", "", null, false, "", true, true, false, new ProxiesConf(), undefined)

  }

  static defaultConf() {

app/lib/indexer.ts

+2
−0

Original line number
Diff line number
Diff line

@@ -1092,6 +1092,8 @@ export class Indexer {

    HEAD.powRemainder = HEAD.issuerDiff  % 16;

    HEAD.powZeros = (HEAD.issuerDiff - HEAD.powRemainder) / 16;

    return nbPersonalBlocksInFrame;

  }

  // BR_G19

app/modules/prover/index.ts

+3
−0

Original line number
Diff line number
Diff line

@@ -18,6 +18,9 @@ export const ProverDependency = {

    /*********** Permanent prover **************/

    config: {

      onLoading: async (conf:ConfDTO) => {

        if (conf.ecoMode === null || conf.ecoMode === undefined) {

          conf.ecoMode = ProverConstants.DEFAULT_ECO_MODE;

         }

        if (conf.cpu === null || conf.cpu === undefined) {

          conf.cpu = ProverConstants.DEFAULT_CPU;

        }

app/modules/prover/lib/blockProver.ts

+49
−1

Original line number
Diff line number
Diff line

@@ -48,6 +48,14 @@ export class WorkerFarm {

    return this.theEngine.getNbWorkers()

  }

  reduceNbCores() {

    return this.theEngine.reduceNbCores()

  }

  boostCPU(){

    return this.theEngine.boostCPU()

  }

  changeCPU(cpu:any) {

    return this.theEngine.setConf({ cpu })

  }

@@ -179,6 +187,7 @@ export class BlockProver {

      let result = await powFarm.askNewProof({

        newPoW: {

          conf: {

            nbCores: this.conf.nbCores,

            cpu: this.conf.cpu,

            prefix: this.conf.prefix,

            avgGenTime: this.conf.avgGenTime,

@@ -193,6 +202,14 @@ export class BlockProver {

      });

      if (!result) {

        this.logger.info('GIVEN proof-of-work for block#%s with %s leading zeros followed by [0-' + highMark + ']! stop PoW for %s', block.number, nbZeros, this.pair && this.pair.pub.slice(0,6));

        if(this.conf.ecoMode === true) {

          let selfNbBlockInFrame = await this.server.getBcContext().getIssuerNbBlockInFrame(this.server.PeeringService.selfPubkey)

          if(selfNbBlockInFrame < 2) {

            this.boostCPU()

            this.conf.nbCores = powFarm.nbWorkers

            this.logger.info("Boost number of CPU cores "+powFarm.nbWorkers+" with only "+selfNbBlockInFrame+" block member in frame")

          }

        }

        throw 'Proof-of-work computation canceled because block received';

      } else {

        const proof = result.block;

@@ -201,13 +218,44 @@ export class BlockProver {

        const testsPerSecond = testsCount / (duration / 1000)

        this.logger.info('Done: #%s, %s in %ss (~%s tests, ~%s tests/s, using %s cores, CPU %s%)', block.number, proof.hash, (duration / 1000).toFixed(2), testsCount, testsPerSecond.toFixed(2), powFarm.nbWorkers, Math.floor(100*this.conf.cpu))

        this.logger.info('FOUND proof-of-work with %s leading zeros followed by [0-' + highMark + ']!', nbZeros);

        if(this.conf.ecoMode === true) {

          let selfNbBlockInFrame = await this.server.getBcContext().getIssuerNbBlockInFrame(this.server.PeeringService.selfPubkey)

          if(selfNbBlockInFrame < 2) {

            this.boostCPU()

            this.conf.nbCores = powFarm.nbWorkers

            this.logger.info("Boost number of CPU cores "+powFarm.nbWorkers+" with only "+selfNbBlockInFrame+" block member in frame")

          }

          else if(testsPerSecond > ProverConstants.ECO_MODE_MINIMAL_TESTS_PER_SECONDS) {

            if(powFarm.nbWorkers > 1) {

              this.logger.info("Reducing number of CPU cores "+powFarm.nbWorkers)

              this.reduceNbCores()

              this.conf.nbCores = powFarm.nbWorkers

            }

            else if(this.conf.cpu > ProverConstants.ECO_MODE_MINIMAL_CPU){

              let cpu:number = this.conf.cpu - 0.1

              this.logger.info("Slowing down the CPU to "+cpu)

              this.changeCPU(cpu)

            }

          }

        }

        return BlockDTO.fromJSONObject(proof)

      }

    })()

  };

  async reduceNbCores() {

    const farm = await this.getWorker()

    return farm.reduceNbCores()

  }

  async boostCPU() {

    this.conf.cpu = 1.0

    const farm = await this.getWorker()

    return farm.boostCPU()

  }

  async changeCPU(cpu:number) {

    this.conf.cpu = cpu;

    this.conf.cpu = Math.max(0.01, Math.min(1.0, cpu));

    const farm = await this.getWorker()

    return farm.changeCPU(cpu)

  }

app/modules/prover/lib/constants.ts

+4
−0

Original line number
Diff line number
Diff line

@@ -5,11 +5,15 @@ export const ProverConstants = {

  MINIMAL_ZEROS_TO_SHOW_IN_LOGS: 3,

  POW_MINIMAL_TO_SHOW: 2,

  DEFAULT_ECO_MODE: true,

  DEFAULT_CPU: 0.6,

  DEFAULT_PEER_ID: 1,

  MIN_PEER_ID: 1,

  MAX_PEER_ID: 899, // Due to MAX_SAFE_INTEGER = 9007199254740991 (16 digits, and we use 11 digits for the nonce + 2 digits for core number => 3 digits for the peer, must be below 900)

  ECO_MODE_MINIMAL_TESTS_PER_SECONDS: 300,

  ECO_MODE_MINIMAL_CPU: 0.19,

  NONCE_RANGE: 1000 * 1000 * 1000 * 100,

  POW_MAXIMUM_ACCEPTABLE_HANDICAP: 64,

app/modules/prover/lib/engine.ts

+9
−4

Original line number
Diff line number
Diff line

@@ -12,15 +12,12 @@ if(debug) {

export class PowEngine {

  private nbWorkers:number

  private cluster:PowCluster

  readonly id:number

  constructor(private conf:ConfDTO, logger:any) {

    // We use as much cores as available, but not more than CORES_MAXIMUM_USE_IN_PARALLEL

    this.nbWorkers = conf.nbCores

    this.cluster = new PowCluster(this.nbWorkers, logger)

    this.cluster = new PowCluster(conf.nbCores, logger)

    this.id = this.cluster.clusterId

  }

@@ -41,6 +38,14 @@ export class PowEngine {

    return this.cluster.cancelWork()

  }

  reduceNbCores() {

    return this.cluster.removeSlave()

  }

  boostCPU() {

    return this.cluster.boostCPU()

  }

  setConf(value:any) {

    return this.cluster.changeConf(value)

  }

app/modules/prover/lib/powCluster.ts

+52
−30

Original line number
Diff line number
Diff line

@@ -35,6 +35,7 @@ export class Master {

  logger:any

  onInfoCallback:any

  workersOnline:Promise<any>[]

  maxNbCores:number = Math.min(ProverConstants.CORES_MAXIMUM_USE_IN_PARALLEL, require('os').cpus().length)

  constructor(private nbCores:number, logger:any) {

    this.clusterId = clusterId++

@@ -65,18 +66,7 @@ export class Master {

    // this.logger.debug(`ENGINE c#${this.clusterId}#${this.slavesMap[worker.id].index}:`, message)

  }

  /*****************

   * CLUSTER METHODS

   ****************/

  initCluster() {

    // Setup master

    cluster.setupMaster({

      exec: __filename,

      execArgv: [] // Do not try to debug forks

    })

    this.slaves = Array.from({ length: this.nbCores }).map((value, index) => {

  createSlave(index:number) {

    const nodejsWorker = cluster.fork()

    const worker = new PowWorker(nodejsWorker, message => {

      this.onWorkerMessage(index, message)

@@ -107,6 +97,38 @@ export class Master {

    }

    this.slavesMap[nodejsWorker.id] = slave

    return slave

  }

  boostCPU() {

    if(this.nbWorkers < this.maxNbCores) {

      while(this.nbWorkers < this.maxNbCores) {

        this.slaves.push(this.createSlave(this.nbWorkers))

      }

    }

    let conf:any = {cpu: 1}

    this.changeConf(conf)

  }

  async removeSlave() {

    let nb_workers = this.nbWorkers

    await this.slaves[nb_workers-1].worker.kill()

    this.slaves.pop()

    this.logger.info('Remove slave number '+ (nb_workers-1))

  }

  /*****************

   * CLUSTER METHODS

   ****************/

  initCluster() {

    // Setup master

    cluster.setupMaster({

      exec: __filename,

      execArgv: [] // Do not try to debug forks

    })

    this.slaves = Array.from({ length: this.nbCores }).map((value, index) => {

      return this.createSlave(index)

    })

    this.workersOnline = this.slaves.map((s) => s.online)

index.ts

+4
−0

Original line number
Diff line number
Diff line

@@ -457,6 +457,8 @@ function commandLineConf(program:any, conf:any = {}) {

  conf = conf || {};

  const cli = {

    currency: program.currency,

    ecoMode: program.ecoMode,

    noEcoMode: program.noEcoMode,

    cpu: program.cpu,

    nbCores: program.nbCores,

    prefix: program.prefix,

@@ -499,6 +501,8 @@ function commandLineConf(program:any, conf:any = {}) {

  // Update the rest of the conf

  if (cli.currency)                             conf.currency = cli.currency;

  if (cli.server.port)                          conf.port = cli.server.port;

  if (cli.ecoMode)                              conf.ecoMode = true

  if (cli.noEcoMode)                            conf.ecoMode = false

  if (cli.cpu)                                  conf.cpu = Math.max(0.01, Math.min(1.0, cli.cpu));

  if (cli.prefix)                               conf.prefix = Math.max(ProverConstants.MIN_PEER_ID, Math.min(ProverConstants.MAX_PEER_ID, cli.prefix));

  if (cli.logs.http)                            conf.httplogs = true;

test/integration/proof-of-work.js

+20
−2

Original line number
Diff line number
Diff line

@@ -16,13 +16,15 @@ keyring from Key

***/

const intermediateProofs = [];

const NB_CORES_FOR_COMPUTATION = 1 // For simple tests. Can be changed to test multiple cores.

const NB_CORES_FOR_COMPUTATION = 2 // For simple tests. Can be changed to test multiple cores.

const prover = new BlockProver({

  push: (data) => intermediateProofs.push(data),

  conf: {

    avgGenTime: 20,//1*60,

    ecoMode: true,

    nbCores: NB_CORES_FOR_COMPUTATION,

    cpu: 1.0, // 80%,

    cpu: 0.8, // 80%,

    pair: {

      pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd',

      sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'

@@ -34,6 +36,7 @@ const prover = new BlockProver({

const now = 1474382274 * 1000;

const MUST_START_WITH_A_ZERO = 16;

const MUST_START_WITH_TWO_ZEROS = 32;

const MUST_START_WITH_A_ZERO_AND_A_NUMBER = 22

describe("Proof-of-work", function() {

@@ -48,6 +51,21 @@ describe("Proof-of-work", function() {

    intermediateProofs[intermediateProofs.length - 1].pow.should.have.property('hash').equal(block.hash);

  }));

  it('should be reducing cpu when the PoW is too easy for the cpu', () => co(function*() {

    prover.conf.nbCores = 2

    prover.conf.cpu = 0.9

    prover.conf.nbCores.should.equal(2)

    prover.conf.cpu.should.equal(0.9)

    for(let i=0; i<8; ++i) {

      yield prover.prove({

        issuer: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd',

        number: i+2,

        now

      }, MUST_START_WITH_A_ZERO_AND_A_NUMBER, now);

    }

    prover.conf.nbCores.should.equal(1)

    prover.conf.cpu.should.be.below(0.9)

  }));

  // Too randomly successing test

  // it('should be able to cancel a proof-of-work on other PoW receival', () => co(function*() {

  //   const now = 1474464489;

Compare revisions

Source

Target

Commits on Source 10

Files

app/cli.ts

app/lib/computation/BlockchainContext.ts

app/lib/dto/ConfDTO.ts

app/lib/indexer.ts

app/modules/prover/index.ts

app/modules/prover/lib/blockProver.ts

app/modules/prover/lib/constants.ts

app/modules/prover/lib/engine.ts

app/modules/prover/lib/powCluster.ts

index.ts

test/integration/proof-of-work.js