Commit b7f9f4df authored by Cédric Moreau's avatar Cédric Moreau

[enh] #1037 Migrate new INDEX code to TypeScript

parent 593aae77
app/lib/blockchain/*.js
app/lib/blockchain/interfaces/*.js
test/blockchain/*.js
test/blockchain/lib/*.js
\ No newline at end of file
......@@ -23,3 +23,16 @@ vagrant/duniter
*.tar.gz
*.log
*.exe
# vscode
.vscode
# TS migration
test/blockchain/*.js
test/blockchain/*.js.map
test/blockchain/lib/*.js
test/blockchain/lib/*.js.map
app/lib/blockchain/*.js
app/lib/blockchain/*.js.map
app/lib/blockchain/interfaces/*.js
app/lib/blockchain/interfaces/*.js.map
\ No newline at end of file
"use strict"
import {BlockchainOperator} from "./interfaces/BlockchainOperator"
module.exports = class BasicBlockchain {
export class BasicBlockchain {
constructor(operations) {
this.operations = operations
constructor(private op:BlockchainOperator) {
}
/**
* Adds a block at the end of the blockchain.
* @param b Block.
* @returns {*} Promise<Boolean>
*/
pushBlock(b) {
return this.operations.store(b)
return this.op.store(b)
}
/**
......@@ -21,7 +19,7 @@ module.exports = class BasicBlockchain {
* @returns {*} Promise<Block>
*/
getBlock(number) {
return this.operations.read(number)
return this.op.read(number)
}
/**
......@@ -29,8 +27,8 @@ module.exports = class BasicBlockchain {
* @param index Index from top. Defaults to `0`. E.g. `0` = HEAD, `1` = HEAD~1, etc.
* @returns {*} Promise<Block>
*/
head(index) {
return this.operations.head(index)
head(index = 0) {
return this.op.head(index)
}
/**
......@@ -38,7 +36,7 @@ module.exports = class BasicBlockchain {
* @returns {*} Size.
*/
height() {
return this.operations.height()
return this.op.height()
}
/**
......@@ -47,7 +45,7 @@ module.exports = class BasicBlockchain {
* @returns {*} Promise<Block>
*/
headRange(n) {
return this.operations.headRange(n)
return this.op.headRange(n)
}
/**
......@@ -55,6 +53,6 @@ module.exports = class BasicBlockchain {
* @returns {*} Promise<Block> The reverted block.
*/
revertHead() {
return this.operations.revert()
return this.op.revertHead()
}
}
This diff is collapsed.
"use strict"
import {BasicBlockchain} from "./BasicBlockchain"
import {IndexOperator} from "./interfaces/IndexOperator"
import {BlockchainOperator} from "./interfaces/BlockchainOperator"
import * as _ from "underscore"
export class IndexedBlockchain extends BasicBlockchain {
private initIndexer: Promise<void>
constructor(bcOperations: BlockchainOperator, private indexOperations: IndexOperator, private numberField, private pkFields: any) {
super(bcOperations)
this.initIndexer = indexOperations.initIndexer(pkFields)
}
async recordIndex(index) {
// Wait indexer init
await this.initIndexer
return this.indexOperations.recordIndex(index)
}
async indexTrim(maxNumber) {
// Wait indexer init
await this.initIndexer
const subIndexes = await this.indexOperations.getSubIndexes()
// Trim the subIndexes
const records = {}
for (const subIndex of subIndexes) {
records[subIndex] = []
const pks = typeof this.pkFields[subIndex].pk !== 'string' && this.pkFields[subIndex].pk.length ? Array.from(this.pkFields[subIndex].pk) : [this.pkFields[subIndex].pk]
const rm = this.pkFields[subIndex].remove
let potentialRecords = await this.indexOperations.findTrimable(subIndex, this.numberField, maxNumber)
potentialRecords = reduceBy(potentialRecords, pks)
for (const potential of potentialRecords) {
const subCriteriasRowsToDelete = criteriasFromPks(pks, potential)
subCriteriasRowsToDelete[this.numberField] = { $lt: maxNumber }
const rowsToReduce = await this.indexOperations.findWhere(subIndex, subCriteriasRowsToDelete)
// No reduction if 1 line to delete
if (rowsToReduce.length > 1) {
const reduced = reduce(rowsToReduce)
const subCriteriasRowsToKeep = criteriasFromPks(pks, potential)
subCriteriasRowsToKeep[this.numberField] = { $gte: maxNumber }
const toKeep = await this.indexOperations.findWhere(subIndex, subCriteriasRowsToKeep)
const subCriteriasAllRowsOfObject = criteriasFromPks(pks, potential)
await this.indexOperations.removeWhere(subIndex, subCriteriasAllRowsOfObject)
// Add the reduced row + rows to keep
if (!rm || !reduced[rm]) {
records[subIndex] = records[subIndex].concat([reduced]).concat(toKeep)
}
}
}
}
await this.recordIndex(records)
return Promise.resolve()
}
async indexCount(indexName, criterias) {
// Wait indexer init
await this.initIndexer
const records = await this.indexOperations.findWhere(indexName, criterias)
return records.length
}
async indexReduce(indexName, criterias) {
// Wait indexer init
await this.initIndexer
const records = await this.indexOperations.findWhere(indexName, criterias)
return reduce(records)
}
async indexReduceGroupBy(indexName, criterias, properties) {
// Wait indexer init
await this.initIndexer
const records = await this.indexOperations.findWhere(indexName, criterias)
return reduceBy(records, properties)
}
async indexRevert(blockNumber) {
const subIndexes = await this.indexOperations.getSubIndexes()
for (const subIndex of subIndexes) {
const removeCriterias = {}
removeCriterias[this.numberField] = blockNumber
await this.indexOperations.removeWhere(subIndex, removeCriterias)
}
}
}
function reduce(records) {
return records.reduce((obj, record) => {
const keys = Object.keys(record);
for (const k of keys) {
if (record[k] !== undefined && record[k] !== null) {
obj[k] = record[k];
}
}
return obj;
}, {});
}
function reduceBy(reducables, properties) {
const reduced = reducables.reduce((map, entry) => {
const id = properties.map((prop) => entry[prop]).join('-');
map[id] = map[id] || [];
map[id].push(entry);
return map;
}, {});
return _.values(reduced).map((value) => reduce(value))
}
function criteriasFromPks(pks, values) {
const criterias = {}
for (const key of pks) {
criterias[key] = values[key]
}
return criterias
}
"use strict"
import {IndexedBlockchain} from "./IndexedBlockchain"
import {SQLIndex} from "./SqlIndex"
const co = require('co')
const IndexedBlockchain = require('./indexedBlockchain')
const SQLIndex = require('./sqlIndex')
module.exports = class MiscIndexedBlockchain extends IndexedBlockchain {
export class MiscIndexedBlockchain extends IndexedBlockchain {
constructor(blockchainStorage, mindexDAL, iindexDAL, sindexDAL, cindexDAL) {
super(blockchainStorage, SQLIndex(null, {
super(blockchainStorage, new SQLIndex(null, {
m_index: { handler: mindexDAL },
i_index: { handler: iindexDAL },
s_index: {
......
"use strict"
import {BlockchainOperator} from "./interfaces/BlockchainOperator"
const indexer = require('../../lib/indexer')
export class SQLBlockchain implements BlockchainOperator {
constructor(private dal: { bindexDAL:any }) {
}
store(b: any): Promise<any> {
return this.dal.bindexDAL.saveEntity(b)
}
read(i: number): Promise<any> {
return this.dal.bindexDAL.sqlFindOne({ number: i })
}
async head(n = 0): Promise<any> {
/**
* IMPORTANT NOTICE
* ----------------
*
* There is a difference between the protocol's HEAD (P_HEAD) and the database's HEAD (DB_HEAD). The relation is:
*
* DB_HEAD~<i> = P_HEAD~<i+1>
*
* In this class methods, we expose DB_HEAD logic. But the data is stored/retrieved by DAL objects using P_HEAD logic.
*
* So if we want DB_HEAD~0 for example, we must ask P_HEAD~(0+1). The DAL object will then retrieve P_HEAD~1, which
* is the latest stored block in the blockchain.
*
* Note: the DAL object cannot retrieve P_HEAD~0, this value does not exist and refers to the potential incoming block.
*
* Why this behavior?
* ------------------
*
* Because the DAL was wrongly coded. It will be easy to fix this problem once indexer.js will only use **this class'
* methods**. Then, we will be able to replace (n + 1) by just (n), and replace also the complementary behavior in
* the DAL (BIndexDAL).
*/
return this.dal.bindexDAL.head(n + 1)
}
async height(): Promise<number> {
const head = await this.dal.bindexDAL.head(1) // We do not use head(0). See the above notice.
if (head) {
return head.number + 1
} else {
return 0
}
}
headRange(m: number): Promise<any[]> {
return this.dal.bindexDAL.range(1, m) // We do not use range(0, m). See the above notice.
}
async revertHead(): Promise<any> {
const head = await this.dal.bindexDAL.head(1) // We do not use head(0). See the above notice.
await this.dal.bindexDAL.removeBlock(head.number)
return head
}
}
"use strict"
import {IndexOperator} from "./interfaces/IndexOperator"
import * as _ from "underscore"
const IndexDAL = require('../dal/sqliteDAL/IndexDAL')
export class SQLIndex implements IndexOperator {
private indexes: { [k:string]: any } = {}
constructor(private db, private definitions: any) {
}
async initIndexer(pkFields: any): Promise<void> {
const keys = _.keys(pkFields)
for (const k of keys) {
if (this.definitions[k].handler) {
// External table: managed by another object
this.indexes[k] = this.definitions[k].handler
} else {
// Internal table: managed here
const indexTable = new IndexDAL(this.db);
const pk = pkFields[k].pk
indexTable.table = k
indexTable.fields = this.definitions[k].fields
indexTable.booleans = this.definitions[k].booleans
this.indexes[k] = indexTable
indexTable.init = () => {
return indexTable.exec('BEGIN;' +
'CREATE TABLE IF NOT EXISTS ' + indexTable.table + ' (' +
this.definitions[k].sqlFields.join(',') +
');' +
'COMMIT;', [])
}
await indexTable.init()
}
}
}
getSubIndexes(): Promise<string[]> {
return Promise.resolve(_.keys(this.indexes))
}
findTrimable(subIndex: string, numberField: string, maxNumber: number): Promise<any[]> {
if (this.definitions[subIndex].findTrimable) {
return this.definitions[subIndex].findTrimable(maxNumber)
} else {
const criterias = {}
criterias[numberField] = { $lt: maxNumber }
return this.indexes[subIndex].sqlFind(criterias)
}
}
removeWhere(subIndex: string, criterias: {}): Promise<void> {
if (!this.indexes[subIndex]) {
return Promise.resolve()
}
return this.indexes[subIndex].sqlRemoveWhere(criterias)
}
async recordIndex(index: any): Promise<void> {
const subIndexes = _.keys(index)
// Feed the this.indexes
for (const subIndex of subIndexes) {
await this.indexes[subIndex].insertBatch(index[subIndex])
}
return Promise.resolve()
}
findWhere(subIndex: string, criterias: {}): Promise<any[]> {
if (!this.indexes[subIndex]) {
return Promise.resolve([])
}
return this.indexes[subIndex].sqlFind(criterias)
}
}
This diff is collapsed.
"use strict"
const co = require('co')
const _ = require('underscore')
const BasicBlockchain = require('./basicBlockchain')
module.exports = class IndexedBlockchain extends BasicBlockchain {
constructor(bcOperations, indexOperations, numberField, pkFields) {
super(bcOperations)
this.indexOperations = indexOperations
this.numberField = numberField
this.pkFields = pkFields
this.initIndexer = indexOperations.initIndexer(pkFields)
}
recordIndex(index) {
const that = this
return co(function*() {
// Wait indexer init
yield that.initIndexer
return that.indexOperations.recordIndex(index)
})
}
indexTrim(maxNumber) {
const that = this
return co(function*() {
// Wait indexer init
yield that.initIndexer
const subIndexes = yield that.indexOperations.getSubIndexes()
// Trim the subIndexes
const records = {}
for (const subIndex of subIndexes) {
records[subIndex] = []
const pks = typeof that.pkFields[subIndex].pk !== 'string' && that.pkFields[subIndex].pk.length ? Array.from(that.pkFields[subIndex].pk) : [that.pkFields[subIndex].pk]
const rm = that.pkFields[subIndex].remove
let potentialRecords = yield that.indexOperations.findTrimable(subIndex, that.numberField, maxNumber)
potentialRecords = reduceBy(potentialRecords, pks)
for (const potential of potentialRecords) {
const subCriteriasRowsToDelete = criteriasFromPks(pks, potential)
subCriteriasRowsToDelete[that.numberField] = { $lt: maxNumber }
const rowsToReduce = yield that.indexOperations.findWhere(subIndex, subCriteriasRowsToDelete)
// No reduction if 1 line to delete
if (rowsToReduce.length > 1) {
const reduced = reduce(rowsToReduce)
const subCriteriasRowsToKeep = criteriasFromPks(pks, potential)
subCriteriasRowsToKeep[that.numberField] = { $gte: maxNumber }
const toKeep = yield that.indexOperations.findWhere(subIndex, subCriteriasRowsToKeep)
const subCriteriasAllRowsOfObject = criteriasFromPks(pks, potential)
yield that.indexOperations.removeWhere(subIndex, subCriteriasAllRowsOfObject)
// Add the reduced row + rows to keep
if (!rm || !reduced[rm]) {
records[subIndex] = records[subIndex].concat([reduced]).concat(toKeep)
}
}
}
}
yield that.recordIndex(records)
return Promise.resolve()
})
}
indexCount(indexName, criterias) {
const that = this
return co(function*() {
// Wait indexer init
yield that.initIndexer
const records = yield that.indexOperations.findWhere(indexName, criterias)
return records.length
})
}
indexReduce(indexName, criterias) {
const that = this
return co(function*() {
// Wait indexer init
yield that.initIndexer
const records = yield that.indexOperations.findWhere(indexName, criterias)
return reduce(records)
})
}
indexReduceGroupBy(indexName, criterias, properties) {
const that = this
return co(function*() {
// Wait indexer init
yield that.initIndexer
const records = yield that.indexOperations.findWhere(indexName, criterias)
return reduceBy(records, properties)
})
}
indexRevert(blockNumber) {
const that = this
return co(function*() {
const subIndexes = yield that.indexOperations.getSubIndexes()
for (const subIndex of subIndexes) {
const removeCriterias = {}
removeCriterias[that.numberField] = blockNumber
yield that.indexOperations.removeWhere(subIndex, removeCriterias)
}
})
}
}
function reduce(records) {
return records.reduce((obj, record) => {
const keys = Object.keys(record);
for (const k of keys) {
if (record[k] !== undefined && record[k] !== null) {
obj[k] = record[k];
}
}
return obj;
}, {});
}
function reduceBy(reducables, properties) {
const reduced = reducables.reduce((map, entry) => {
const id = properties.map((prop) => entry[prop]).join('-');
map[id] = map[id] || [];
map[id].push(entry);
return map;
}, {});
return _.values(reduced).map((value) => reduce(value))
}
function criteriasFromPks(pks, values) {
const criterias = {}
for (const key of pks) {
criterias[key] = values[key]
}
return criterias
}
"use strict"
export interface BlockchainOperator {
/**
* Pushes a new block at the top of the blockchain.
* @param b Block.
*/
store(b):Promise<any>
/**
* Reads the block at index `i`.
* @param i Block number.
*/
read(i:number):Promise<any>
/**
* Reads the block at index `n` from the top of the blockchain.
* @param n Reverse index.
*/
head(n:number):Promise<any>
/**
* Gives the number of blocks in the blockchain.
*/
height():Promise<number>
/**
* Reads the blocks from head(0) to head(m)
* @param m Quantity.
*/
headRange(m:number):Promise<any[]>
/**
* Pops the top block.
*/
revertHead():Promise<any>
}
"use strict"
export interface IndexOperator {
initIndexer(pkFields: any): Promise<void>,
getSubIndexes(): Promise<string[]>,
findWhere(subIndex: string, criterias: {}): Promise<any[]>,
findTrimable(subIndex: string, numberField: string, maxNumber: number): Promise<any[]>,
removeWhere(subIndex: string, criterias: {}): Promise<void>,
recordIndex(index: any): Promise<void>
}
"use strict"
const co = require('co')
module.exports = function SQLBlockchain(dal) {
return {
store: (b) => {
return Promise.resolve(b)
},
read: (index) => {
return dal.bindexDAL.sqlFindOne({ number: index, fork: false })
},
head: (number) => {
return dal.bindexDAL.head(number)
},
headRange: (number) => {
return dal.bindexDAL.range(0, number)
},
height: () => {
return co(function*() {
const head = yield dal.bindexDAL.head()
return head.number + 1
})
},
revert: () => {
return co(function*() {
const head = yield dal.bindexDAL.head()
yield dal.bindexDAL