Skip to content
Snippets Groups Projects
Commit b7f9f4df authored by Cédric Moreau's avatar Cédric Moreau
Browse files

[enh] #1037 Migrate new INDEX code to TypeScript

parent 593aae77
No related branches found
No related tags found
No related merge requests found
Showing
with 908 additions and 875 deletions
app/lib/blockchain/*.js
app/lib/blockchain/interfaces/*.js
test/blockchain/*.js
test/blockchain/lib/*.js
\ No newline at end of file
......@@ -23,3 +23,16 @@ vagrant/duniter
*.tar.gz
*.log
*.exe
# vscode
.vscode
# TS migration
test/blockchain/*.js
test/blockchain/*.js.map
test/blockchain/lib/*.js
test/blockchain/lib/*.js.map
app/lib/blockchain/*.js
app/lib/blockchain/*.js.map
app/lib/blockchain/interfaces/*.js
app/lib/blockchain/interfaces/*.js.map
\ No newline at end of file
"use strict"
import {BlockchainOperator} from "./interfaces/BlockchainOperator"
module.exports = class BasicBlockchain {
export class BasicBlockchain {
constructor(operations) {
this.operations = operations
constructor(private op:BlockchainOperator) {
}
/**
* Adds a block at the end of the blockchain.
* @param b Block.
* @returns {*} Promise<Boolean>
*/
pushBlock(b) {
return this.operations.store(b)
return this.op.store(b)
}
/**
......@@ -21,7 +19,7 @@ module.exports = class BasicBlockchain {
* @returns {*} Promise<Block>
*/
getBlock(number) {
return this.operations.read(number)
return this.op.read(number)
}
/**
......@@ -29,8 +27,8 @@ module.exports = class BasicBlockchain {
* @param index Index from top. Defaults to `0`. E.g. `0` = HEAD, `1` = HEAD~1, etc.
* @returns {*} Promise<Block>
*/
head(index) {
return this.operations.head(index)
head(index = 0) {
return this.op.head(index)
}
/**
......@@ -38,7 +36,7 @@ module.exports = class BasicBlockchain {
* @returns {*} Size.
*/
height() {
return this.operations.height()
return this.op.height()
}
/**
......@@ -47,7 +45,7 @@ module.exports = class BasicBlockchain {
* @returns {*} Promise<Block>
*/
headRange(n) {
return this.operations.headRange(n)
return this.op.headRange(n)
}
/**
......@@ -55,6 +53,6 @@ module.exports = class BasicBlockchain {
* @returns {*} Promise<Block> The reverted block.
*/
revertHead() {
return this.operations.revert()
return this.op.revertHead()
}
}
This diff is collapsed.
"use strict"
import {BasicBlockchain} from "./BasicBlockchain"
import {IndexOperator} from "./interfaces/IndexOperator"
import {BlockchainOperator} from "./interfaces/BlockchainOperator"
import * as _ from "underscore"
export class IndexedBlockchain extends BasicBlockchain {
private initIndexer: Promise<void>
constructor(bcOperations: BlockchainOperator, private indexOperations: IndexOperator, private numberField, private pkFields: any) {
super(bcOperations)
this.initIndexer = indexOperations.initIndexer(pkFields)
}
async recordIndex(index) {
// Wait indexer init
await this.initIndexer
return this.indexOperations.recordIndex(index)
}
async indexTrim(maxNumber) {
// Wait indexer init
await this.initIndexer
const subIndexes = await this.indexOperations.getSubIndexes()
// Trim the subIndexes
const records = {}
for (const subIndex of subIndexes) {
records[subIndex] = []
const pks = typeof this.pkFields[subIndex].pk !== 'string' && this.pkFields[subIndex].pk.length ? Array.from(this.pkFields[subIndex].pk) : [this.pkFields[subIndex].pk]
const rm = this.pkFields[subIndex].remove
let potentialRecords = await this.indexOperations.findTrimable(subIndex, this.numberField, maxNumber)
potentialRecords = reduceBy(potentialRecords, pks)
for (const potential of potentialRecords) {
const subCriteriasRowsToDelete = criteriasFromPks(pks, potential)
subCriteriasRowsToDelete[this.numberField] = { $lt: maxNumber }
const rowsToReduce = await this.indexOperations.findWhere(subIndex, subCriteriasRowsToDelete)
// No reduction if 1 line to delete
if (rowsToReduce.length > 1) {
const reduced = reduce(rowsToReduce)
const subCriteriasRowsToKeep = criteriasFromPks(pks, potential)
subCriteriasRowsToKeep[this.numberField] = { $gte: maxNumber }
const toKeep = await this.indexOperations.findWhere(subIndex, subCriteriasRowsToKeep)
const subCriteriasAllRowsOfObject = criteriasFromPks(pks, potential)
await this.indexOperations.removeWhere(subIndex, subCriteriasAllRowsOfObject)
// Add the reduced row + rows to keep
if (!rm || !reduced[rm]) {
records[subIndex] = records[subIndex].concat([reduced]).concat(toKeep)
}
}
}
}
await this.recordIndex(records)
return Promise.resolve()
}
async indexCount(indexName, criterias) {
// Wait indexer init
await this.initIndexer
const records = await this.indexOperations.findWhere(indexName, criterias)
return records.length
}
async indexReduce(indexName, criterias) {
// Wait indexer init
await this.initIndexer
const records = await this.indexOperations.findWhere(indexName, criterias)
return reduce(records)
}
async indexReduceGroupBy(indexName, criterias, properties) {
// Wait indexer init
await this.initIndexer
const records = await this.indexOperations.findWhere(indexName, criterias)
return reduceBy(records, properties)
}
async indexRevert(blockNumber) {
const subIndexes = await this.indexOperations.getSubIndexes()
for (const subIndex of subIndexes) {
const removeCriterias = {}
removeCriterias[this.numberField] = blockNumber
await this.indexOperations.removeWhere(subIndex, removeCriterias)
}
}
}
function reduce(records) {
return records.reduce((obj, record) => {
const keys = Object.keys(record);
for (const k of keys) {
if (record[k] !== undefined && record[k] !== null) {
obj[k] = record[k];
}
}
return obj;
}, {});
}
function reduceBy(reducables, properties) {
const reduced = reducables.reduce((map, entry) => {
const id = properties.map((prop) => entry[prop]).join('-');
map[id] = map[id] || [];
map[id].push(entry);
return map;
}, {});
return _.values(reduced).map((value) => reduce(value))
}
function criteriasFromPks(pks, values) {
const criterias = {}
for (const key of pks) {
criterias[key] = values[key]
}
return criterias
}
"use strict"
import {IndexedBlockchain} from "./IndexedBlockchain"
import {SQLIndex} from "./SqlIndex"
const co = require('co')
const IndexedBlockchain = require('./indexedBlockchain')
const SQLIndex = require('./sqlIndex')
module.exports = class MiscIndexedBlockchain extends IndexedBlockchain {
export class MiscIndexedBlockchain extends IndexedBlockchain {
constructor(blockchainStorage, mindexDAL, iindexDAL, sindexDAL, cindexDAL) {
super(blockchainStorage, SQLIndex(null, {
super(blockchainStorage, new SQLIndex(null, {
m_index: { handler: mindexDAL },
i_index: { handler: iindexDAL },
s_index: {
......
"use strict"
import {BlockchainOperator} from "./interfaces/BlockchainOperator"
const indexer = require('../../lib/indexer')
export class SQLBlockchain implements BlockchainOperator {
constructor(private dal: { bindexDAL:any }) {
}
store(b: any): Promise<any> {
return this.dal.bindexDAL.saveEntity(b)
}
read(i: number): Promise<any> {
return this.dal.bindexDAL.sqlFindOne({ number: i })
}
async head(n = 0): Promise<any> {
/**
* IMPORTANT NOTICE
* ----------------
*
* There is a difference between the protocol's HEAD (P_HEAD) and the database's HEAD (DB_HEAD). The relation is:
*
* DB_HEAD~<i> = P_HEAD~<i+1>
*
* In this class methods, we expose DB_HEAD logic. But the data is stored/retrieved by DAL objects using P_HEAD logic.
*
* So if we want DB_HEAD~0 for example, we must ask P_HEAD~(0+1). The DAL object will then retrieve P_HEAD~1, which
* is the latest stored block in the blockchain.
*
* Note: the DAL object cannot retrieve P_HEAD~0, this value does not exist and refers to the potential incoming block.
*
* Why this behavior?
* ------------------
*
* Because the DAL was wrongly coded. It will be easy to fix this problem once indexer.js will only use **this class'
* methods**. Then, we will be able to replace (n + 1) by just (n), and replace also the complementary behavior in
* the DAL (BIndexDAL).
*/
return this.dal.bindexDAL.head(n + 1)
}
async height(): Promise<number> {
const head = await this.dal.bindexDAL.head(1) // We do not use head(0). See the above notice.
if (head) {
return head.number + 1
} else {
return 0
}
}
headRange(m: number): Promise<any[]> {
return this.dal.bindexDAL.range(1, m) // We do not use range(0, m). See the above notice.
}
async revertHead(): Promise<any> {
const head = await this.dal.bindexDAL.head(1) // We do not use head(0). See the above notice.
await this.dal.bindexDAL.removeBlock(head.number)
return head
}
}
"use strict"
import {IndexOperator} from "./interfaces/IndexOperator"
import * as _ from "underscore"
const IndexDAL = require('../dal/sqliteDAL/IndexDAL')
export class SQLIndex implements IndexOperator {
private indexes: { [k:string]: any } = {}
constructor(private db, private definitions: any) {
}
async initIndexer(pkFields: any): Promise<void> {
const keys = _.keys(pkFields)
for (const k of keys) {
if (this.definitions[k].handler) {
// External table: managed by another object
this.indexes[k] = this.definitions[k].handler
} else {
// Internal table: managed here
const indexTable = new IndexDAL(this.db);
const pk = pkFields[k].pk
indexTable.table = k
indexTable.fields = this.definitions[k].fields
indexTable.booleans = this.definitions[k].booleans
this.indexes[k] = indexTable
indexTable.init = () => {
return indexTable.exec('BEGIN;' +
'CREATE TABLE IF NOT EXISTS ' + indexTable.table + ' (' +
this.definitions[k].sqlFields.join(',') +
');' +
'COMMIT;', [])
}
await indexTable.init()
}
}
}
getSubIndexes(): Promise<string[]> {
return Promise.resolve(_.keys(this.indexes))
}
findTrimable(subIndex: string, numberField: string, maxNumber: number): Promise<any[]> {
if (this.definitions[subIndex].findTrimable) {
return this.definitions[subIndex].findTrimable(maxNumber)
} else {
const criterias = {}
criterias[numberField] = { $lt: maxNumber }
return this.indexes[subIndex].sqlFind(criterias)
}
}
removeWhere(subIndex: string, criterias: {}): Promise<void> {
if (!this.indexes[subIndex]) {
return Promise.resolve()
}
return this.indexes[subIndex].sqlRemoveWhere(criterias)
}
async recordIndex(index: any): Promise<void> {
const subIndexes = _.keys(index)
// Feed the this.indexes
for (const subIndex of subIndexes) {
await this.indexes[subIndex].insertBatch(index[subIndex])
}
return Promise.resolve()
}
findWhere(subIndex: string, criterias: {}): Promise<any[]> {
if (!this.indexes[subIndex]) {
return Promise.resolve([])
}
return this.indexes[subIndex].sqlFind(criterias)
}
}
This diff is collapsed.
"use strict"
const co = require('co')
const _ = require('underscore')
const BasicBlockchain = require('./basicBlockchain')
module.exports = class IndexedBlockchain extends BasicBlockchain {
constructor(bcOperations, indexOperations, numberField, pkFields) {
super(bcOperations)
this.indexOperations = indexOperations
this.numberField = numberField
this.pkFields = pkFields
this.initIndexer = indexOperations.initIndexer(pkFields)
}
recordIndex(index) {
const that = this
return co(function*() {
// Wait indexer init
yield that.initIndexer
return that.indexOperations.recordIndex(index)
})
}
indexTrim(maxNumber) {
const that = this
return co(function*() {
// Wait indexer init
yield that.initIndexer
const subIndexes = yield that.indexOperations.getSubIndexes()
// Trim the subIndexes
const records = {}
for (const subIndex of subIndexes) {
records[subIndex] = []
const pks = typeof that.pkFields[subIndex].pk !== 'string' && that.pkFields[subIndex].pk.length ? Array.from(that.pkFields[subIndex].pk) : [that.pkFields[subIndex].pk]
const rm = that.pkFields[subIndex].remove
let potentialRecords = yield that.indexOperations.findTrimable(subIndex, that.numberField, maxNumber)
potentialRecords = reduceBy(potentialRecords, pks)
for (const potential of potentialRecords) {
const subCriteriasRowsToDelete = criteriasFromPks(pks, potential)
subCriteriasRowsToDelete[that.numberField] = { $lt: maxNumber }
const rowsToReduce = yield that.indexOperations.findWhere(subIndex, subCriteriasRowsToDelete)
// No reduction if 1 line to delete
if (rowsToReduce.length > 1) {
const reduced = reduce(rowsToReduce)
const subCriteriasRowsToKeep = criteriasFromPks(pks, potential)
subCriteriasRowsToKeep[that.numberField] = { $gte: maxNumber }
const toKeep = yield that.indexOperations.findWhere(subIndex, subCriteriasRowsToKeep)
const subCriteriasAllRowsOfObject = criteriasFromPks(pks, potential)
yield that.indexOperations.removeWhere(subIndex, subCriteriasAllRowsOfObject)
// Add the reduced row + rows to keep
if (!rm || !reduced[rm]) {
records[subIndex] = records[subIndex].concat([reduced]).concat(toKeep)
}
}
}
}
yield that.recordIndex(records)
return Promise.resolve()
})
}
indexCount(indexName, criterias) {
const that = this
return co(function*() {
// Wait indexer init
yield that.initIndexer
const records = yield that.indexOperations.findWhere(indexName, criterias)
return records.length
})
}
indexReduce(indexName, criterias) {
const that = this
return co(function*() {
// Wait indexer init
yield that.initIndexer
const records = yield that.indexOperations.findWhere(indexName, criterias)
return reduce(records)
})
}
indexReduceGroupBy(indexName, criterias, properties) {
const that = this
return co(function*() {
// Wait indexer init
yield that.initIndexer
const records = yield that.indexOperations.findWhere(indexName, criterias)
return reduceBy(records, properties)
})
}
indexRevert(blockNumber) {
const that = this
return co(function*() {
const subIndexes = yield that.indexOperations.getSubIndexes()
for (const subIndex of subIndexes) {
const removeCriterias = {}
removeCriterias[that.numberField] = blockNumber
yield that.indexOperations.removeWhere(subIndex, removeCriterias)
}
})
}
}
function reduce(records) {
return records.reduce((obj, record) => {
const keys = Object.keys(record);
for (const k of keys) {
if (record[k] !== undefined && record[k] !== null) {
obj[k] = record[k];
}
}
return obj;
}, {});
}
function reduceBy(reducables, properties) {
const reduced = reducables.reduce((map, entry) => {
const id = properties.map((prop) => entry[prop]).join('-');
map[id] = map[id] || [];
map[id].push(entry);
return map;
}, {});
return _.values(reduced).map((value) => reduce(value))
}
function criteriasFromPks(pks, values) {
const criterias = {}
for (const key of pks) {
criterias[key] = values[key]
}
return criterias
}
"use strict"
export interface BlockchainOperator {
/**
* Pushes a new block at the top of the blockchain.
* @param b Block.
*/
store(b):Promise<any>
/**
* Reads the block at index `i`.
* @param i Block number.
*/
read(i:number):Promise<any>
/**
* Reads the block at index `n` from the top of the blockchain.
* @param n Reverse index.
*/
head(n:number):Promise<any>
/**
* Gives the number of blocks in the blockchain.
*/
height():Promise<number>
/**
* Reads the blocks from head(0) to head(m)
* @param m Quantity.
*/
headRange(m:number):Promise<any[]>
/**
* Pops the top block.
*/
revertHead():Promise<any>
}
"use strict"
export interface IndexOperator {
initIndexer(pkFields: any): Promise<void>,
getSubIndexes(): Promise<string[]>,
findWhere(subIndex: string, criterias: {}): Promise<any[]>,
findTrimable(subIndex: string, numberField: string, maxNumber: number): Promise<any[]>,
removeWhere(subIndex: string, criterias: {}): Promise<void>,
recordIndex(index: any): Promise<void>
}
"use strict"
const co = require('co')
module.exports = function SQLBlockchain(dal) {
return {
store: (b) => {
return Promise.resolve(b)
},
read: (index) => {
return dal.bindexDAL.sqlFindOne({ number: index, fork: false })
},
head: (number) => {
return dal.bindexDAL.head(number)
},
headRange: (number) => {
return dal.bindexDAL.range(0, number)
},
height: () => {
return co(function*() {
const head = yield dal.bindexDAL.head()
return head.number + 1
})
},
revert: () => {
return co(function*() {
const head = yield dal.bindexDAL.head()
yield dal.bindexDAL.removeBlock(head.number)
})
}
}
}
"use strict"
const co = require('co')
const _ = require('underscore')
const AbstractSQLite = require('../../../app/lib/dal/sqliteDAL/AbstractSQLite')
const AbstractIndex = require('../../../app/lib/dal/sqliteDAL/AbstractIndex')
module.exports = function SQLIndex(db, definitions) {
const indexes = {
}
const findWhere = (subIndex, criterias) => {
if (!indexes[subIndex]) {
return Promise.resolve([])
}
return indexes[subIndex].sqlFind(criterias)
}
return {
initIndexer: (pkFields) => co(function*() {
const keys = _.keys(pkFields)
for (const k of keys) {
if (definitions[k].handler) {
// External table: managed by another object
indexes[k] = definitions[k].handler
} else {
// Internal table: managed here
const indexTable = new IndexDAL(db);
const pk = pkFields[k].pk
indexTable.table = k
indexTable.fields = definitions[k].fields
indexTable.booleans = definitions[k].booleans
indexes[k] = indexTable
indexTable.init = () => {
return indexTable.exec('BEGIN;' +
'CREATE TABLE IF NOT EXISTS ' + indexTable.table + ' (' +
definitions[k].sqlFields.join(',') +
');' +
'COMMIT;', [])
}
yield indexTable.init()
}
}
}),
getSubIndexes: () => Promise.resolve(_.keys(indexes)),
findWhere,
findTrimable: (subIndex, numberField, maxNumber) => {
if (definitions[subIndex].findTrimable) {
return definitions[subIndex].findTrimable(maxNumber)
} else {
const criterias = {}
criterias[numberField] = { $lt: maxNumber }
return indexes[subIndex].sqlFind(criterias)
}
},
removeWhere: (subIndex, criterias) => {
if (!indexes[subIndex]) {
return Promise.resolve([])
}
return indexes[subIndex].sqlRemoveWhere(criterias)
},
recordIndex: (index) => co(function*() {
const subIndexes = _.keys(index)
// Feed the indexes
for (const subIndex of subIndexes) {
yield indexes[subIndex].insertBatch(index[subIndex])
}
return Promise.resolve()
})
}
}
function IndexDAL(driver) {
AbstractSQLite.call(this, driver);
AbstractIndex.call(this);
}
......@@ -100,7 +100,7 @@ function BlockchainContext() {
this.checkBlock = (block, withPoWAndSignature) => blockchain.checkBlock(block, withPoWAndSignature, conf, dal)
this.addBlock = (obj, index, HEAD) => co(function*() {
const block = yield blockchain.pushBlock(obj, index, HEAD, conf, dal, logger)
const block = yield blockchain.pushTheBlock(obj, index, HEAD, conf, dal, logger)
vHEAD_1 = vHEAD = HEADrefreshed = null
return block
})
......
......@@ -7,7 +7,7 @@ const indexer = require('../indexer')
const constants = require('../constants')
const Block = require('../entity/block')
const Transaction = require('../entity/transaction')
const DuniterBlockchain = require('../blockchain/duniterBlockchain')
const DuniterBlockchain = require('../blockchain/DuniterBlockchain').DuniterBlockchain
module.exports = (blockchain, conf, dal, logger) => {
......@@ -239,7 +239,7 @@ module.exports = (blockchain, conf, dal, logger) => {
// Last block: cautious mode to trigger all the INDEX expiry mechanisms
const { index, HEAD } = yield blockchain.checkBlock(block, constants.WITH_SIGNATURES_AND_POW, conf, dal)
yield blockchain.pushBlock(block, index, HEAD, conf, dal, logger)
yield blockchain.pushTheBlock(block, index, HEAD, conf, dal, logger)
// Clean temporary variables
sync_bindex = [];
......
"use strict";
const AbstractSQLite = require('./AbstractSQLite')
const AbstractIndex = require('./AbstractIndex')
module.exports = function IndexDAL(driver) {
AbstractSQLite.call(this, driver);
AbstractIndex.call(this);
}
......@@ -82,8 +82,9 @@ function BIndexDAL(driver) {
* @param n Position
*/
this.head = (n) => co(function*() {
// Default to HEAD~1
n = n || 1;
if (!n) {
throw "Cannot read HEAD~0, which is the incoming block"
}
const headRecords = yield that.query('SELECT * FROM ' + that.table + ' ORDER BY number DESC LIMIT 1 OFFSET ?', [n - 1]);
return headRecords[0];
});
......
......@@ -22,9 +22,10 @@
"test": "test"
},
"scripts": {
"tsc": "tsc",
"test": "mocha --growl --timeout 20000 test test/fast test/integration test/",
"start": "node bin/duniter start",
"build": "cd \"node_modules/duniter-ui\" && npm install && npm run build",
"build": "tsc && cd \"node_modules/duniter-ui\" && npm install && npm run build",
"test-travis": "node ./node_modules/istanbul/lib/cli.js cover ./node_modules/mocha/bin/_mocha --report lcovonly -- -R spec --timeout 20000 test test/fast test/integration test/"
},
"repository": {
......@@ -76,6 +77,9 @@
"wotb": "0.5.x"
},
"devDependencies": {
"@types/mocha": "^2.2.41",
"@types/node": "^8.0.9",
"@types/should": "^8.3.0",
"coveralls": "2.11.4",
"duniter-bma": "1.3.x",
"duniter-crawler": "1.3.x",
......@@ -90,7 +94,8 @@
"sha1": "",
"should": "",
"supertest": "",
"tmp": "0.0.29"
"tmp": "0.0.29",
"typescript": "^2.4.1"
},
"peerDependencies": {
"duniter-bma": "1.3.x",
......
......@@ -16,8 +16,8 @@ const jsonpckg = require('./package.json');
const keyring = require('duniter-common').keyring;
const directory = require('./app/lib/system/directory');
const rawer = require('duniter-common').rawer;
const SQLBlockchain = require('./app/lib/blockchain/sqlBlockchain')
const DuniterBlockchain = require('./app/lib/blockchain/duniterBlockchain')
const SQLBlockchain = require('./app/lib/blockchain/SqlBlockchain').SQLBlockchain
const DuniterBlockchain = require('./app/lib/blockchain/DuniterBlockchain').DuniterBlockchain
function Server (home, memoryOnly, overrideConf) {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment