Commit a86c24f9 authored by Cédric Moreau's avatar Cédric Moreau

[enh] #1037 Migrate indexer.js + blockchainContext.js

parent 61bcbbf8
app/lib/blockchain/*.js
app/lib/blockchain/interfaces/*.js
app/lib/computation/*.js
app/lib/db/*.js
app/lib/dto/*.js
app/lib/indexer.js
app/lib/common.js
test/blockchain/*.js
test/blockchain/lib/*.js
\ No newline at end of file
......@@ -35,4 +35,10 @@ test/blockchain/lib/*.js.map
app/lib/blockchain/*.js
app/lib/blockchain/*.js.map
app/lib/blockchain/interfaces/*.js
app/lib/blockchain/interfaces/*.js.map
\ No newline at end of file
app/lib/blockchain/interfaces/*.js.map
app/lib/computation/*.js
app/lib/computation/*.js.map
app/lib/common.js*
app/lib/db/*.js*
app/lib/dto/*.js*
app/lib/indexer.js*
\ No newline at end of file
......@@ -9,7 +9,7 @@ export class BasicBlockchain {
/**
* Adds a block at the end of the blockchain.
*/
pushBlock(b) {
pushBlock(b:any) {
return this.op.store(b)
}
......@@ -18,7 +18,7 @@ export class BasicBlockchain {
* @param number block ID.
* @returns {*} Promise<Block>
*/
getBlock(number) {
getBlock(number:number) {
return this.op.read(number)
}
......@@ -44,7 +44,7 @@ export class BasicBlockchain {
* @param n Quantity from top. E.g. `1` = [HEAD], `3` = [HEAD, HEAD~1, HEAD~2], etc.
* @returns {*} Promise<Block>
*/
headRange(n) {
headRange(n:number) {
return this.op.headRange(n)
}
......
This diff is collapsed.
......@@ -2,32 +2,33 @@
import {BasicBlockchain} from "./BasicBlockchain"
import {IndexOperator} from "./interfaces/IndexOperator"
import {BlockchainOperator} from "./interfaces/BlockchainOperator"
import * as _ from "underscore"
const _ = require('underscore')
export class IndexedBlockchain extends BasicBlockchain {
private initIndexer: Promise<void>
constructor(bcOperations: BlockchainOperator, private indexOperations: IndexOperator, private numberField, private pkFields: any) {
constructor(bcOperations: BlockchainOperator, private indexOperations: IndexOperator, private numberField: string, private pkFields: any) {
super(bcOperations)
this.initIndexer = indexOperations.initIndexer(pkFields)
}
async recordIndex(index) {
async recordIndex(index: { [index: string]: any }) {
// Wait indexer init
await this.initIndexer
return this.indexOperations.recordIndex(index)
}
async indexTrim(maxNumber) {
async indexTrim(maxNumber:number) {
// Wait indexer init
await this.initIndexer
const subIndexes = await this.indexOperations.getSubIndexes()
// Trim the subIndexes
const records = {}
const records: { [index: string]: any } = {}
for (const subIndex of subIndexes) {
records[subIndex] = []
const pks = typeof this.pkFields[subIndex].pk !== 'string' && this.pkFields[subIndex].pk.length ? Array.from(this.pkFields[subIndex].pk) : [this.pkFields[subIndex].pk]
......@@ -57,7 +58,7 @@ export class IndexedBlockchain extends BasicBlockchain {
return Promise.resolve()
}
async indexCount(indexName, criterias) {
async indexCount(indexName: string, criterias: { [index: string]: any }) {
// Wait indexer init
await this.initIndexer
......@@ -66,7 +67,7 @@ export class IndexedBlockchain extends BasicBlockchain {
return records.length
}
async indexReduce(indexName, criterias) {
async indexReduce(indexName: string, criterias: { [index: string]: any }) {
// Wait indexer init
await this.initIndexer
......@@ -75,7 +76,7 @@ export class IndexedBlockchain extends BasicBlockchain {
return reduce(records)
}
async indexReduceGroupBy(indexName, criterias, properties) {
async indexReduceGroupBy(indexName: string, criterias: { [index: string]: any }, properties: string[]) {
// Wait indexer init
await this.initIndexer
......@@ -84,17 +85,17 @@ export class IndexedBlockchain extends BasicBlockchain {
return reduceBy(records, properties)
}
async indexRevert(blockNumber) {
async indexRevert(blockNumber:number) {
const subIndexes = await this.indexOperations.getSubIndexes()
for (const subIndex of subIndexes) {
const removeCriterias = {}
const removeCriterias: { [index: string]: any } = {}
removeCriterias[this.numberField] = blockNumber
await this.indexOperations.removeWhere(subIndex, removeCriterias)
}
}
}
function reduce(records) {
function reduce(records: any[]) {
return records.reduce((obj, record) => {
const keys = Object.keys(record);
for (const k of keys) {
......@@ -106,18 +107,18 @@ function reduce(records) {
}, {});
}
function reduceBy(reducables, properties) {
function reduceBy(reducables: any[], properties: string[]) {
const reduced = reducables.reduce((map, entry) => {
const id = properties.map((prop) => entry[prop]).join('-');
map[id] = map[id] || [];
map[id].push(entry);
return map;
}, {});
return _.values(reduced).map((value) => reduce(value))
return _.values(reduced).map((rows: any[]) => reduce(rows))
}
function criteriasFromPks(pks, values) {
const criterias = {}
function criteriasFromPks(pks: string[], values: any): { [index: string]: any } {
const criterias: { [index: string]: any } = {}
for (const key of pks) {
criterias[key] = values[key]
}
......
"use strict"
import {IndexedBlockchain} from "./IndexedBlockchain"
import {SQLIndex} from "./SqlIndex"
import {BlockchainOperator} from "./interfaces/BlockchainOperator"
export class MiscIndexedBlockchain extends IndexedBlockchain {
constructor(blockchainStorage, mindexDAL, iindexDAL, sindexDAL, cindexDAL) {
constructor(blockchainStorage: BlockchainOperator, mindexDAL:any, iindexDAL:any, sindexDAL:any, cindexDAL:any) {
super(blockchainStorage, new SQLIndex(null, {
m_index: { handler: mindexDAL },
i_index: { handler: iindexDAL },
s_index: {
handler: sindexDAL,
findTrimable: (maxNumber) => sindexDAL.query('SELECT * FROM s_index WHERE consumed AND writtenOn < ?', [maxNumber])
findTrimable: (maxNumber:number) => sindexDAL.query('SELECT * FROM s_index WHERE consumed AND writtenOn < ?', [maxNumber])
},
c_index: {
handler: cindexDAL,
findTrimable: (maxNumber) => cindexDAL.query('SELECT * FROM c_index WHERE expired_on > 0 AND writtenOn < ?', [maxNumber])
findTrimable: (maxNumber:number) => cindexDAL.query('SELECT * FROM c_index WHERE expired_on > 0 AND writtenOn < ?', [maxNumber])
}
}), 'writtenOn', {
m_index: {
......
"use strict"
import {BlockchainOperator} from "./interfaces/BlockchainOperator"
const indexer = require('../../lib/indexer')
const indexer = require('../../lib/indexer').Indexer
export class SQLBlockchain implements BlockchainOperator {
......
......@@ -6,7 +6,7 @@ export interface BlockchainOperator {
* Pushes a new block at the top of the blockchain.
* @param b Block.
*/
store(b):Promise<any>
store(b:any):Promise<any>
/**
* Reads the block at index `i`.
......
const common = require('duniter-common')
export function hashf(str:string) {
return common.hashf(str).toUpperCase()
}
"use strict";
import {BlockDTO} from "../dto/BlockDTO"
import {DuniterBlockchain} from "../blockchain/DuniterBlockchain"
import {QuickSynchronizer} from "./QuickSync"
import {DBHead} from "../db/DBHead"
const _ = require('underscore');
const co = require('co');
const indexer = require('../indexer');
const indexer = require('../indexer').Indexer
const constants = require('../constants');
const Block = require('../entity/block');
module.exports = () => { return new BlockchainContext() };
export class BlockchainContext {
function BlockchainContext() {
const that = this;
let conf, dal, logger, blockchain, quickSynchronizer
private conf:any
private dal:any
private logger:any
private blockchain:DuniterBlockchain
private quickSynchronizer:QuickSynchronizer
/**
* The virtual next HEAD. Computed each time a new block is added, because a lot of HEAD variables are deterministic
* and can be computed one, just after a block is added for later controls.
*/
let vHEAD;
private vHEAD:any
/**
* The currently written HEAD, aka. HEAD_1 relatively to incoming HEAD.
*/
let vHEAD_1;
private vHEAD_1:any
let HEADrefreshed = Promise.resolve();
private HEADrefreshed: Promise<any> | null = Promise.resolve();
/**
* Refresh the virtual HEAD value for determined variables of the next coming block, avoiding to recompute them
* each time a new block arrives to check if the values are correct. We can know and store them early on, in vHEAD.
*/
function refreshHead() {
HEADrefreshed = co(function*() {
vHEAD_1 = yield dal.head(1);
private refreshHead(): Promise<void> {
this.HEADrefreshed = (async (): Promise<void> => {
this.vHEAD_1 = await this.dal.head(1);
// We suppose next block will have same version #, and no particular data in the block (empty index)
let block;
// But if no HEAD_1 exist, we must initialize a block with default values
if (!vHEAD_1) {
if (!this.vHEAD_1) {
block = {
version: constants.BLOCK_GENERATED_VERSION,
time: Math.round(Date.now() / 1000),
powMin: conf.powMin || 0,
powMin: this.conf.powMin || 0,
powZeros: 0,
powRemainder: 0,
avgBlockSize: 0
};
} else {
block = { version: vHEAD_1.version };
block = { version: this.vHEAD_1.version };
}
vHEAD = yield indexer.completeGlobalScope(Block.statics.fromJSON(block), conf, [], dal);
});
return HEADrefreshed;
this.vHEAD = await indexer.completeGlobalScope(Block.statics.fromJSON(block), this.conf, [], this.dal);
})()
return this.HEADrefreshed;
}
/**
* Gets a copy of vHEAD, extended with some extra properties.
* @param props The extra properties to add.
*/
this.getvHeadCopy = (props) => co(function*() {
if (!vHEAD) {
yield refreshHead();
async getvHeadCopy(props: any): Promise<any> {
if (!this.vHEAD) {
await this.refreshHead();
}
const copy = {};
const keys = Object.keys(vHEAD);
const copy: any = {};
const keys = Object.keys(this.vHEAD);
for (const k of keys) {
copy[k] = vHEAD[k];
copy[k] = this.vHEAD[k];
}
_.extend(copy, props);
return copy;
});
}
/**
* Get currently written HEAD.
*/
this.getvHEAD_1 = () => co(function*() {
if (!vHEAD) {
yield refreshHead();
async getvHEAD_1(): Promise<any> {
if (!this.vHEAD) {
await this.refreshHead();
}
return vHEAD_1;
});
return this.vHEAD_1
}
/**
* Utility method: gives the personalized difficulty level of a given issuer for next block.
* @param issuer The issuer we want to get the difficulty level.
*/
this.getIssuerPersonalizedDifficulty = (issuer) => co(function *() {
const local_vHEAD = yield that.getvHeadCopy({ issuer });
yield indexer.preparePersonalizedPoW(local_vHEAD, vHEAD_1, dal.range, conf);
async getIssuerPersonalizedDifficulty(issuer: string): Promise<any> {
const local_vHEAD = await this.getvHeadCopy({ issuer });
await indexer.preparePersonalizedPoW(local_vHEAD, this.vHEAD_1, this.dal.range, this.conf)
return local_vHEAD.issuerDiff;
});
}
this.setConfDAL = (newConf, newDAL, theBlockchain, theQuickSynchronizer) => {
dal = newDAL;
conf = newConf;
blockchain = theBlockchain
quickSynchronizer = theQuickSynchronizer
logger = require('../logger')(dal.profile);
};
setConfDAL(newConf: any, newDAL: any, theBlockchain: DuniterBlockchain, theQuickSynchronizer: QuickSynchronizer): void {
this.dal = newDAL;
this.conf = newConf;
this.blockchain = theBlockchain
this.quickSynchronizer = theQuickSynchronizer
this.logger = require('../logger')(this.dal.profile);
}
this.checkBlock = (block, withPoWAndSignature) => blockchain.checkBlock(block, withPoWAndSignature, conf, dal)
checkBlock(block: BlockDTO, withPoWAndSignature = true): Promise<any> {
return this.blockchain.checkBlock(block, withPoWAndSignature, this.conf, this.dal)
}
this.addBlock = (obj, index, HEAD) => co(function*() {
const block = yield blockchain.pushTheBlock(obj, index, HEAD, conf, dal, logger)
vHEAD_1 = vHEAD = HEADrefreshed = null
async addBlock(obj: BlockDTO, index: any, HEAD: DBHead): Promise<any> {
const block = await this.blockchain.pushTheBlock(obj, index, HEAD, this.conf, this.dal, this.logger)
this.vHEAD_1 = this.vHEAD = this.HEADrefreshed = null
return block
})
}
this.addSideBlock = (obj) => blockchain.pushSideBlock(obj, dal, logger)
addSideBlock(obj:BlockDTO): Promise<any> {
return this.blockchain.pushSideBlock(obj, this.dal, this.logger)
}
this.revertCurrentBlock = () => co(function *() {
const head_1 = yield dal.bindexDAL.head(1);
logger.debug('Reverting block #%s...', head_1.number);
const res = yield blockchain.revertBlock(head_1.number, head_1.hash, dal)
logger.debug('Reverted block #%s', head_1.number);
async revertCurrentBlock(): Promise<any> {
const head_1 = await this.dal.bindexDAL.head(1);
this.logger.debug('Reverting block #%s...', head_1.number);
const res = await this.blockchain.revertBlock(head_1.number, head_1.hash, this.dal)
this.logger.debug('Reverted block #%s', head_1.number);
// Invalidates the head, since it has changed.
yield refreshHead();
await this.refreshHead();
return res;
});
}
this.applyNextAvailableFork = () => co(function *() {
const current = yield that.current();
logger.debug('Find next potential block #%s...', current.number + 1);
const forks = yield dal.getForkBlocksFollowing(current);
async applyNextAvailableFork(): Promise<any> {
const current = await this.current();
this.logger.debug('Find next potential block #%s...', current.number + 1);
const forks = await this.dal.getForkBlocksFollowing(current);
if (!forks.length) {
throw constants.ERRORS.NO_POTENTIAL_FORK_AS_NEXT;
}
const block = forks[0];
const { index, HEAD } = yield that.checkBlock(block, constants.WITH_SIGNATURES_AND_POW);
yield that.addBlock(block, index, HEAD);
logger.debug('Applied block #%s', block.number);
});
const { index, HEAD } = await this.checkBlock(block, constants.WITH_SIGNATURES_AND_POW);
await this.addBlock(block, index, HEAD);
this.logger.debug('Applied block #%s', block.number);
}
this.current = () => dal.getCurrentBlockOrNull();
current(): Promise<any> {
return this.dal.getCurrentBlockOrNull()
}
this.checkHaveEnoughLinks = (target, newLinks) => co(function*() {
const links = yield dal.getValidLinksTo(target);
async checkHaveEnoughLinks(target: string, newLinks: any): Promise<any> {
const links = await this.dal.getValidLinksTo(target);
let count = links.length;
if (newLinks[target] && newLinks[target].length)
if (newLinks[target] && newLinks[target].length) {
count += newLinks[target].length;
if (count < conf.sigQty)
throw 'Key ' + target + ' does not have enough links (' + count + '/' + conf.sigQty + ')';
});
}
if (count < this.conf.sigQty) {
throw 'Key ' + target + ' does not have enough links (' + count + '/' + this.conf.sigQty + ')';
}
}
this.quickApplyBlocks = (blocks, to) => quickSynchronizer.quickApplyBlocks(blocks, to)
quickApplyBlocks(blocks:BlockDTO[], to: number | null): Promise<any> {
return this.quickSynchronizer.quickApplyBlocks(blocks, to)
}
}
......@@ -3,7 +3,7 @@ const Q = require('q');
const co = require('co');
const _ = require('underscore');
const common = require('duniter-common');
const indexer = require('../indexer');
const indexer = require('../indexer').Indexer
const logger = require('../logger')('filedal');
const Configuration = require('../entity/configuration');
const Merkle = require('../entity/merkle');
......
......@@ -4,7 +4,7 @@
const _ = require('underscore');
const co = require('co');
const indexer = require('../../indexer');
const indexer = require('../../indexer').Indexer
module.exports = AbstractIndex;
......
......@@ -5,7 +5,7 @@
const co = require('co');
const constants = require('./../../../constants');
const common = require('duniter-common');
const indexer = require('../../../indexer');
const indexer = require('../../../indexer').Indexer
const AbstractSQLite = require('./../AbstractSQLite');
const AbstractIndex = require('./../AbstractIndex');
......
......@@ -4,7 +4,7 @@
const co = require('co');
const _ = require('underscore');
const indexer = require('../../../indexer');
const indexer = require('../../../indexer').Indexer
const AbstractSQLite = require('./../AbstractSQLite');
const AbstractIndex = require('./../AbstractIndex');
......
......@@ -3,7 +3,7 @@
*/
const co = require('co');
const indexer = require('../../../indexer');
const indexer = require('../../../indexer').Indexer
const AbstractSQLite = require('./../AbstractSQLite');
const AbstractIndex = require('./../AbstractIndex');
......
......@@ -5,7 +5,7 @@
const _ = require('underscore');
const co = require('co');
const common = require('duniter-common');
const indexer = require('../../../indexer');
const indexer = require('../../../indexer').Indexer
const constants = require('../../../constants');
const AbstractSQLite = require('./../AbstractSQLite');
const AbstractIndex = require('./../AbstractIndex');
......
import {BlockDTO} from "../dto/BlockDTO"
import {TransactionDTO} from "../dto/TransactionDTO"
export class DBBlock {
version: number
number: number
currency: string
hash: string
inner_hash: string
signature: string
previousHash: string
issuer: string
previousIssuer: string
time: number
powMin: number
unitbase: number
membersCount: number
issuersCount: number
issuersFrame: number
issuersFrameVar: number
identities: string[]
joiners: string[]
actives: string[]
leavers: string[]
revoked: string[]
excluded: string[]
certifications: string[]
transactions: TransactionDTO[]
medianTime: number
nonce: number
fork: boolean
parameters: string
monetaryMass: number
dividend: number | null
constructor(
) {
}
static fromBlockDTO(b:BlockDTO) {
const dbb = new DBBlock()
dbb.version = b.version
dbb.number = b.number
dbb.currency = b.currency
dbb.hash = b.hash
dbb.previousHash = b.previousHash
dbb.issuer = b.issuer
dbb.previousIssuer = b.previousIssuer
dbb.dividend = b.dividend
dbb.time = b.time
dbb.powMin = b.powMin
dbb.unitbase = b.unitbase
dbb.membersCount = b.membersCount
dbb.issuersCount = b.issuersCount
dbb.issuersFrame = b.issuersFrame
dbb.issuersFrameVar = b.issuersFrameVar
dbb.identities = b.identities
dbb.joiners = b.joiners
dbb.actives = b.actives
dbb.leavers = b.leavers
dbb.revoked = b.revoked
dbb.excluded = b.excluded
dbb.certifications = b.certifications
dbb.transactions = b.transactions
dbb.medianTime = b.medianTime
dbb.fork = b.fork
dbb.parameters = b.parameters
dbb.inner_hash = b.inner_hash
dbb.signature = b.signature
dbb.nonce = b.nonce
return dbb
}
}
\ No newline at end of file
export class DBHead {
// TODO: some properties are not registered in the DB, we should create another class
version: number
currency: string | null
bsize: number
avgBlockSize: number
udTime: number
udReevalTime: number
massReeval: number