Commit e9b5436b authored by Cédric Moreau's avatar Cédric Moreau

Test Loki

parent d1e77251
......@@ -9,6 +9,9 @@ app/lib/dto/*.js
app/lib/indexer.js
app/lib/common.js
app/lib/dal/drivers/*.js
app/lib/dal/indexDAL/*.js
app/lib/dal/indexDAL/loki/*.js
app/lib/dal/indexDAL/abstract/*.js
app/lib/dal/sqliteDAL/*.js
app/lib/dal/sqliteDAL/index/*.js
app/lib/dal/fileDALs/*.js
......
......@@ -80,3 +80,5 @@ test/fast/prover/pow-1-cluster.js.map
test/fast/protocol-local-rule-chained-tx-depth.js
test/fast/protocol-local-rule-chained-tx-depth.js.map
test/fast/protocol-local-rule-chained-tx-depth.d.ts
test/fast/dal/*-loki.d.ts
test/fast/dal/*-loki.js*
......@@ -28,6 +28,7 @@ import {CommonConstants} from "../common-libs/constants"
import {FileDAL} from "../dal/fileDAL"
import {DBTx} from "../dal/sqliteDAL/TxsDAL"
import {DataErrors} from "../common-libs/errors"
import {NewLogger} from "../logger"
const _ = require('underscore')
......@@ -58,7 +59,9 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// BR_G98
if (Indexer.ruleCurrency(block, HEAD) === false) throw Error('ruleCurrency');
// BR_G51
if (Indexer.ruleNumber(block, HEAD) === false) throw Error('ruleNumber');
if (Indexer.ruleNumber(block, HEAD) === false) {
throw Error('ruleNumber')
}
// BR_G52
if (Indexer.rulePreviousHash(block, HEAD) === false) throw Error('rulePreviousHash');
// BR_G53
......@@ -106,7 +109,9 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// BR_G70
if (Indexer.ruleCertificationToLeaver(cindex) === false) throw Error('ruleCertificationToLeaver');
// BR_G71
if (Indexer.ruleCertificationReplay(cindex) === false) throw Error('ruleCertificationReplay');
if (Indexer.ruleCertificationReplay(cindex) === false) {
throw Error('ruleCertificationReplay')
}
// BR_G72
if (Indexer.ruleCertificationSignature(cindex) === false) throw Error('ruleCertificationSignature');
// BR_G73
......@@ -126,7 +131,9 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// BR_G80
if (Indexer.ruleMembershipLeaverIsMember(mindex) === false) throw Error('ruleMembershipLeaverIsMember');
// BR_G81
if (Indexer.ruleMembershipActiveIsMember(mindex) === false) throw Error('ruleMembershipActiveIsMember');
if (Indexer.ruleMembershipActiveIsMember(mindex) === false) {
throw Error('ruleMembershipActiveIsMember')
}
// BR_G82
if (Indexer.ruleMembershipRevokedIsMember(mindex) === false) throw Error('ruleMembershipRevokedIsMember');
// BR_G83
......@@ -215,7 +222,7 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
await this.createNewcomers(indexes.iindex, dal, logger);
// Save indexes
await dal.bindexDAL.saveEntity(indexes.HEAD);
await dal.bindexDAL.insert(indexes.HEAD);
await dal.mindexDAL.insertBatch(indexes.mindex);
await dal.iindexDAL.insertBatch(indexes.iindex);
await dal.sindexDAL.insertBatch(indexes.sindex);
......@@ -375,7 +382,7 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
const REVERSE_BALANCE = true
const sindexOfBlock = await dal.sindexDAL.getWrittenOn(blockstamp)
await dal.bindexDAL.removeBlock(number);
await dal.bindexDAL.removeBlock(blockstamp);
await dal.mindexDAL.removeBlock(blockstamp);
await dal.iindexDAL.removeBlock(blockstamp);
await dal.cindexDAL.removeBlock(blockstamp);
......@@ -411,6 +418,7 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// => equivalent to i_index.op = 'CREATE'
if (entry.op === CommonConstants.IDX_CREATE) {
// Does not matter which one it really was, we pop the last X identities
NewLogger().trace('removeNode')
dal.wotb.removeNode();
}
}
......
......@@ -15,7 +15,7 @@
import {DuniterBlockchain} from "../blockchain/DuniterBlockchain";
import {BlockDTO} from "../dto/BlockDTO";
import {DBTransaction} from "../db/DBTransaction";
import {AccountsGarbagingDAL, Indexer} from "../indexer";
import {AccountsGarbagingDAL, FullSindexEntry, Indexer} from "../indexer";
import {CurrencyConfDTO} from "../dto/ConfDTO";
import {FileDAL} from "../dal/fileDAL"
import {DBBlock} from "../db/DBBlock"
......@@ -131,11 +131,11 @@ export class QuickSynchronizer {
sync_mindex = sync_mindex.concat(local_mindex);
const HEAD = await Indexer.quickCompleteGlobalScope(block, sync_currConf, sync_bindex, sync_iindex, sync_mindex, sync_cindex, ({
getBlock: (number: number) => {
return Promise.resolve(sync_allBlocks[number]);
async getBlock(number: number) {
return sync_allBlocks[number]
},
getBlockByBlockstamp: (blockstamp: string) => {
return Promise.resolve(sync_allBlocks[parseInt(blockstamp)]);
async getBlockByBlockstamp(blockstamp: string) {
return sync_allBlocks[parseInt(blockstamp)]
}
}) as any);
sync_bindex.push(HEAD);
......@@ -148,8 +148,10 @@ export class QuickSynchronizer {
if (entry.revokes_on) {
sync_expires.push(entry.revokes_on)
}
if (entry.expires_on || entry.revokes_on) {
sync_expires = _.uniq(sync_expires)
}
}
sync_expires = _.uniq(sync_expires);
await this.blockchain.createNewcomers(local_iindex, this.dal, this.logger)
......@@ -177,7 +179,7 @@ export class QuickSynchronizer {
// Fills in correctly the SINDEX
await Promise.all(_.where(sync_sindex.concat(local_sindex), { op: 'UPDATE' }).map(async (entry: any) => {
if (!entry.conditions) {
const src = await this.dal.sindexDAL.getSource(entry.identifier, entry.pos);
const src = (await this.dal.sindexDAL.getSource(entry.identifier, entry.pos)) as FullSindexEntry
entry.conditions = src.conditions;
}
}))
......
......@@ -18,7 +18,10 @@ import {ConfDTO} from "../dto/ConfDTO"
import {BlockDTO} from "../dto/BlockDTO"
import {DBHead} from "../db/DBHead"
import {DBIdentity, IdentityDAL} from "./sqliteDAL/IdentityDAL"
import {CindexEntry, FullMindexEntry, IindexEntry, IndexEntry, SindexEntry} from "../indexer"
import {
CindexEntry, FullCindexEntry, FullMindexEntry, FullSindexEntry, IindexEntry, IndexEntry,
SindexEntry
} from "../indexer"
import {DBPeer, PeerDAL} from "./sqliteDAL/PeerDAL"
import {TransactionDTO} from "../dto/TransactionDTO"
import {CertDAL, DBCert} from "./sqliteDAL/CertDAL"
......@@ -31,18 +34,26 @@ import {CommonConstants} from "../common-libs/constants"
import {PowDAL} from "./fileDALs/PowDAL";
import {Initiable} from "./sqliteDAL/Initiable"
import {MetaDAL} from "./sqliteDAL/MetaDAL"
import {BIndexDAL} from "./sqliteDAL/index/BIndexDAL"
import {MIndexDAL} from "./sqliteDAL/index/MIndexDAL"
import {CIndexDAL} from "./sqliteDAL/index/CIndexDAL"
import {SIndexDAL} from "./sqliteDAL/index/SIndexDAL"
import {IIndexDAL} from "./sqliteDAL/index/IIndexDAL"
import {DataErrors} from "../common-libs/errors"
import {BasicRevocableIdentity, IdentityDTO} from "../dto/IdentityDTO"
import {BlockDAL} from "./sqliteDAL/BlockDAL"
import {FileSystem} from "../system/directory"
import {WoTBInstance} from "../wot"
import {IIndexDAO} from "./indexDAL/abstract/IIndexDAO"
import {LokiIIndex} from "./indexDAL/loki/LokiIIndex"
import {BIndexDAO} from "./indexDAL/abstract/BIndexDAO"
import {MIndexDAO} from "./indexDAL/abstract/MIndexDAO"
import {SIndexDAO} from "./indexDAL/abstract/SIndexDAO"
import {CIndexDAO} from "./indexDAL/abstract/CIndexDAO"
import {IdentityForRequirements} from "../../service/BlockchainService"
import {LokiSIndex} from "./indexDAL/loki/LokiSIndex"
import {LokiCIndex} from "./indexDAL/loki/LokiCIndex"
import {LokiMIndex} from "./indexDAL/loki/LokiMIndex";
import {LokiBIndex} from "./indexDAL/loki/LokiBIndex"
import {NewLogger} from "../logger"
const fs = require('fs')
const loki = require('lokijs')
const path = require('path')
const readline = require('readline')
const _ = require('underscore');
......@@ -64,6 +75,7 @@ export class FileDAL {
wotb:WoTBInstance
profile:string
loki:any
powDAL:PowDAL
confDAL:ConfDAL
metaDAL:MetaDAL
......@@ -75,11 +87,11 @@ export class FileDAL {
certDAL:CertDAL
msDAL:MembershipDAL
walletDAL:WalletDAL
bindexDAL:BIndexDAL
mindexDAL:MIndexDAL
iindexDAL:IIndexDAL
sindexDAL:SIndexDAL
cindexDAL:CIndexDAL
bindexDAL:BIndexDAO
mindexDAL:MIndexDAO
iindexDAL:IIndexDAO
sindexDAL:SIndexDAO
cindexDAL:CIndexDAO
newDals:{ [k:string]: Initiable }
loadConfHook: (conf:ConfDTO) => Promise<void>
......@@ -90,6 +102,7 @@ export class FileDAL {
this.sqliteDriver = params.dbf()
this.wotb = params.wotb
this.profile = 'DAL'
this.loki = new loki('index.db')
// DALs
this.powDAL = new PowDAL(this.rootPath, params.fs)
......@@ -103,11 +116,11 @@ export class FileDAL {
this.certDAL = new (require('./sqliteDAL/CertDAL').CertDAL)(this.sqliteDriver);
this.msDAL = new (require('./sqliteDAL/MembershipDAL').MembershipDAL)(this.sqliteDriver);
this.walletDAL = new (require('./sqliteDAL/WalletDAL').WalletDAL)(this.sqliteDriver);
this.bindexDAL = new (require('./sqliteDAL/index/BIndexDAL').BIndexDAL)(this.sqliteDriver);
this.mindexDAL = new (require('./sqliteDAL/index/MIndexDAL').MIndexDAL)(this.sqliteDriver);
this.iindexDAL = new (require('./sqliteDAL/index/IIndexDAL').IIndexDAL)(this.sqliteDriver);
this.sindexDAL = new (require('./sqliteDAL/index/SIndexDAL').SIndexDAL)(this.sqliteDriver);
this.cindexDAL = new (require('./sqliteDAL/index/CIndexDAL').CIndexDAL)(this.sqliteDriver);
this.bindexDAL = new LokiBIndex(this.loki)
this.mindexDAL = new LokiMIndex(this.loki)
this.iindexDAL = new LokiIIndex(this.loki)
this.sindexDAL = new LokiSIndex(this.loki)
this.cindexDAL = new LokiCIndex(this.loki)
this.newDals = {
'powDAL': this.powDAL,
......@@ -232,7 +245,7 @@ export class FileDAL {
async existsNonChainableLink(from:string, vHEAD_1:DBHead, sigStock:number) {
// Cert period rule
const medianTime = vHEAD_1 ? vHEAD_1.medianTime : 0;
const linksFrom = await this.cindexDAL.reducablesFrom(from)
const linksFrom:FullCindexEntry[] = await this.cindexDAL.reducablesFrom(from)
const unchainables = _.filter(linksFrom, (link:CindexEntry) => link.chainable_on > medianTime);
if (unchainables.length > 0) return true;
// Max stock rule
......@@ -601,17 +614,7 @@ export class FileDAL {
const links = await this.cindexDAL.getValidLinksTo(pub);
let matching = certs;
await Promise.all(links.map(async (entry:any) => {
entry.from = entry.issuer;
const wbt = entry.written_on.split('-');
const blockNumber = parseInt(entry.created_on); // created_on field of `c_index` does not have the full blockstamp
const basedBlock = await this.getBlock(blockNumber);
entry.block = blockNumber;
entry.block_number = blockNumber;
entry.block_hash = basedBlock ? basedBlock.hash : null;
entry.linked = true;
entry.written_block = parseInt(wbt[0]);
entry.written_hash = wbt[1];
matching.push(entry);
matching.push(await this.cindexEntry2DBCert(entry))
}))
matching = _.sortBy(matching, (c:DBCert) => -c.block);
matching.reverse();
......@@ -622,26 +625,36 @@ export class FileDAL {
const certs = await this.certDAL.getFromPubkeyCerts(pubkey);
const links = await this.cindexDAL.getValidLinksFrom(pubkey);
let matching = certs;
await Promise.all(links.map(async (entry:any) => {
const idty = await this.getWrittenIdtyByPubkeyForHash(entry.receiver)
entry.from = entry.issuer;
entry.to = entry.receiver;
const cbt = entry.created_on.split('-');
const wbt = entry.written_on.split('-');
entry.block = parseInt(cbt[0]);
entry.block_number = parseInt(cbt[0]);
entry.block_hash = cbt[1];
entry.target = idty.hash;
entry.linked = true;
entry.written_block = parseInt(wbt[0]);
entry.written_hash = wbt[1];
matching.push(entry);
await Promise.all(links.map(async (entry:CindexEntry) => {
matching.push(await this.cindexEntry2DBCert(entry))
}))
matching = _.sortBy(matching, (c:DBCert) => -c.block);
matching.reverse();
return matching;
}
async cindexEntry2DBCert(entry:CindexEntry): Promise<DBCert> {
const idty = await this.getWrittenIdtyByPubkeyForHash(entry.receiver)
const wbt = entry.written_on.split('-')
const block = (await this.blockDAL.getBlock(entry.created_on)) as DBBlock
return {
issuers: [entry.issuer],
linked: true,
written: true,
written_block: parseInt(wbt[0]),
written_hash: wbt[1],
sig: entry.sig,
block_number: block.number,
block_hash: block.hash,
target: idty.hash,
to: entry.receiver,
from: entry.issuer,
block: block.number,
expired: !!entry.expired_on,
expires_on: entry.expires_on,
}
}
async isSentry(pubkey:string, conf:ConfDTO) {
const current = await this.getCurrentBlockOrNull();
if (current) {
......@@ -712,7 +725,7 @@ export class FileDAL {
return this.cindexDAL.existsNonReplayableLink(from, to)
}
getSource(identifier:string, pos:number) {
getSource(identifier:string, pos:number): Promise<FullSindexEntry | null> {
return this.sindexDAL.getSource(identifier, pos)
}
......@@ -892,9 +905,11 @@ export class FileDAL {
const from = await this.getWrittenIdtyByPubkeyForWotbID(entry.issuer);
const to = await this.getWrittenIdtyByPubkeyForWotbID(entry.receiver);
if (entry.op == CommonConstants.IDX_CREATE) {
NewLogger().trace('addLink %s -> %s', from.wotb_id, to.wotb_id)
this.wotb.addLink(from.wotb_id, to.wotb_id);
} else {
// Update = removal
NewLogger().trace('removeLink %s -> %s', from.wotb_id, to.wotb_id)
this.wotb.removeLink(from.wotb_id, to.wotb_id);
}
}
......@@ -1154,4 +1169,25 @@ export class FileDAL {
}
})
}
async findReceiversAbove(minsig: number) {
const receiversAbove:string[] = await this.cindexDAL.getReceiversAbove(minsig)
const members:IdentityForRequirements[] = []
for (const r of receiversAbove) {
const i = await this.iindexDAL.getFullFromPubkey(r)
members.push({
hash: i.hash || "",
member: i.member || false,
wasMember: i.wasMember || false,
pubkey: i.pub,
uid: i.uid || "",
buid: i.created_on || "",
sig: i.sig || "",
revocation_sig: "",
revoked: false,
revoked_on: 0
})
}
return members
}
}
import {GenericDAO} from "./GenericDAO"
import {DBHead} from "../../../db/DBHead"
export interface BIndexDAO extends GenericDAO<DBHead> {
head(n:number): Promise<DBHead> // TODO: possibly null?
tail(): Promise<DBHead> // TODO: possibly null?
range(n:number, m:number): Promise<DBHead[]>
trimBlocks(maxnumber:number): Promise<void>
}
import {CindexEntry, FullCindexEntry} from "../../../indexer"
import {ReduceableDAO} from "./ReduceableDAO"
export interface CIndexDAO extends ReduceableDAO<CindexEntry> {
getValidLinksTo(receiver:string): Promise<CindexEntry[]>
getValidLinksFrom(issuer:string): Promise<CindexEntry[]>
findExpired(medianTime:number): Promise<CindexEntry[]>
findByIssuerAndReceiver(issuer: string, receiver: string): Promise<CindexEntry[]>
findByIssuerAndChainableOnGt(issuer: string, medianTime: number): Promise<CindexEntry[]>
findByReceiverAndExpiredOn(pub: string, expired_on: number): Promise<CindexEntry[]>
existsNonReplayableLink(issuer:string, receiver:string): Promise<boolean>
getReceiversAbove(minsig: number): Promise<string[]>
reducablesFrom(from:string): Promise<FullCindexEntry[]>
trimExpiredCerts(belowNumber:number): Promise<void>
}
import {Initiable} from "../../sqliteDAL/Initiable"
export interface GenericDAO<T> extends Initiable {
/**
* Make a generic find.
* @param criterion Criterion object, LokiJS's find object format.
* @returns {Promise<any>} A set of records.
*/
findRaw(criterion: any): Promise<any>
/**
* Make a single insert.
* @param record The record to insert.
*/
insert(record:T): Promise<void>
/**
* Make a batch insert.
* @param records The records to insert as a batch.
*/
insertBatch(records:T[]): Promise<void>
/**
* Get the set of records written on a particular blockstamp.
* @param {string} blockstamp The blockstamp we want the records written at.
* @returns {Promise<T[]>} The records (array).
*/
getWrittenOn(blockstamp:string): Promise<T[]>
/**
* Remove all entries written at given `blockstamp`, if these entries are still in the index.
* @param {string} blockstamp Blockstamp of the entries we want to remove.
* @returns {Promise<void>}
*/
removeBlock(blockstamp:string): Promise<void>
}
import {FullIindexEntry, IindexEntry} from "../../../indexer"
import {ReduceableDAO} from "./ReduceableDAO"
import {OldIindexEntry} from "../../sqliteDAL/index/IIndexDAL"
export interface IIndexDAO extends ReduceableDAO<IindexEntry> {
reducable(pub:string): Promise<IindexEntry[]>
findByPub(pub:string): Promise<IindexEntry[]>
findByUid(pub:string): Promise<IindexEntry[]>
getMembers(): Promise<{ pubkey:string, uid:string|null }[]>
getFromPubkey(pub:string): Promise<FullIindexEntry|null>
getFromUID(uid:string): Promise<FullIindexEntry|null>
getFromPubkeyOrUid(search:string): Promise<FullIindexEntry|null>
searchThoseMatching(search:string): Promise<OldIindexEntry[]>
getFullFromUID(uid:string): Promise<FullIindexEntry>
getFullFromPubkey(pub:string): Promise<FullIindexEntry>
getFullFromHash(hash:string): Promise<FullIindexEntry>
getMembersPubkeys(): Promise<{ pub:string }[]>
getToBeKickedPubkeys(): Promise<string[]>
}
import {FullMindexEntry, MindexEntry} from "../../../indexer"
import {ReduceableDAO} from "./ReduceableDAO"
export interface MIndexDAO extends ReduceableDAO<MindexEntry> {
reducable(pub:string): Promise<MindexEntry[]>
getRevokedPubkeys(): Promise<string[]>
findByPubAndChainableOnGt(pub:string, medianTime:number): Promise<MindexEntry[]>
findRevokesOnLteAndRevokedOnIsNull(medianTime:number): Promise<MindexEntry[]>
findExpiresOnLteAndRevokesOnGt(medianTime:number): Promise<MindexEntry[]>
getReducedMS(pub:string): Promise<FullMindexEntry|null>
}
import {GenericDAO} from "./GenericDAO"
export interface ReduceableDAO<T> extends GenericDAO<T> {
/**
* Reduce all records sharing a same reduction key that written before given block number.
* @param {number} belowNumber All records written strictly under `belowNumber` have to be reduced on the reduction key.
* @returns {Promise<void>}
*/
trimRecords(belowNumber:number): Promise<void>
}
import {FullSindexEntry, SindexEntry} from "../../../indexer"
import {ReduceableDAO} from "./ReduceableDAO"
export interface SIndexDAO extends ReduceableDAO<SindexEntry> {
findByIdentifierPosAmountBase(identifier: string, pos: number, amount: number, base: number): Promise<SindexEntry[]>
getSource(identifier:string, pos:number): Promise<FullSindexEntry|null>
getUDSources(pubkey:string): Promise<FullSindexEntry[]>
getAvailableForPubkey(pubkey:string): Promise<{ amount:number, base:number }[]>
getAvailableForConditions(conditionsStr:string): Promise<{ amount:number, base:number }[]>
trimConsumedSource(belowNumber:number): Promise<void>
//---------------------
//- TESTING FUNCTIONS -
//---------------------
}
import {LokiIndex} from "./LokiIndex"
import {DBHead} from "../../../db/DBHead"
import {BIndexDAO} from "../abstract/BIndexDAO"
import {NewLogger} from "../../../logger"
const logger = NewLogger()
export class LokiBIndex extends LokiIndex<DBHead> implements BIndexDAO {
private HEAD:DBHead|null = null
constructor(loki:any) {
super(loki, 'bindex', ['number', 'hash'])
}
async insert(record: DBHead): Promise<void> {
this.HEAD = record
return super.insert(record);
}
async removeBlock(blockstamp: string): Promise<void> {
this.HEAD = await this.head(2)
return super.removeBlock(blockstamp);
}
async head(n: number): Promise<DBHead> {
if (!n) {
throw "Cannot read HEAD~0, which is the incoming block"
}
if (n === 1 && this.HEAD) {
// Cached
return this.HEAD
} else if (this.HEAD) {
// Another than HEAD
return this.collection
.find({ number: this.HEAD.number - n + 1 })[0]
} else {
// Costly method, as a fallback
return this.collection
.chain()
.find({})
.simplesort('number', true)
.data()[n - 1]
}
}
async range(n: number, m: number): Promise<DBHead[]> {
if (!n) {
throw "Cannot read HEAD~0, which is the incoming block"
}
const HEAD = await this.head(1)
if (!HEAD) {
return []
}
return this.collection
.chain()
.find({
$and: [
{ number: { $lte: HEAD.number - n + 1 } },
{ number: { $gte: HEAD.number - m + 1 } },
]
})
.simplesort('number', true)
.data().slice(n - 1, m)
}
async tail(): Promise<DBHead> {
const HEAD = await this.head(1)
if (!HEAD) {
return HEAD
}
const nbHEADs = this.collection.length()
return this.collection
.find({ number: HEAD.number - nbHEADs + 1 })[0]
}
async trimBlocks(maxnumber: number): Promise<void> {
this.collection
.chain()
.find({ number: { $lt: maxnumber }})
.remove()
}
async getWrittenOn(blockstamp: string): Promise<DBHead[]> {
const now = Date.now()
const criterion:any = { number: parseInt(blockstamp) }
const res = this.collection.find(criterion)
logger.trace('[loki][%s][getWrittenOn] %sms', this.collectionName, (Date.now() - now), blockstamp)
return res
}
}