From 6c86081532069941f734ecd134dd85d89151aa44 Mon Sep 17 00:00:00 2001 From: Moul <moul@moul.re> Date: Tue, 9 Apr 2019 23:29:25 +0200 Subject: [PATCH] [mod] #1358: Remove Loki related code --- app/lib/dal/fileDAL.ts | 2 - app/lib/dal/indexDAL/loki/LokiBIndex.ts | 99 ------- app/lib/dal/indexDAL/loki/LokiBlockchain.ts | 270 ------------------ app/lib/dal/indexDAL/loki/LokiCollection.ts | 59 ---- .../indexDAL/loki/LokiCollectionManager.ts | 54 ---- app/lib/dal/indexDAL/loki/LokiDAO.ts | 10 - app/lib/dal/indexDAL/loki/LokiDividend.ts | 157 ---------- app/lib/dal/indexDAL/loki/LokiIIndex.ts | 158 ---------- app/lib/dal/indexDAL/loki/LokiIndex.ts | 39 --- app/lib/dal/indexDAL/loki/LokiMIndex.ts | 104 ------- .../dal/indexDAL/loki/LokiProtocolIndex.ts | 27 -- .../indexDAL/loki/LokiPubkeySharingIndex.ts | 39 --- app/lib/dal/indexDAL/loki/LokiSIndex.ts | 129 --------- app/lib/dal/indexDAL/loki/LokiTypes.ts | 47 --- app/lib/dal/indexDAL/loki/LokiWallet.ts | 48 ---- test/fast/dal/basic-loki.ts | 50 ---- test/fast/dal/iindex-loki.ts | 77 ----- 17 files changed, 1369 deletions(-) delete mode 100644 app/lib/dal/indexDAL/loki/LokiBIndex.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiBlockchain.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiCollection.ts delete mode 100755 app/lib/dal/indexDAL/loki/LokiCollectionManager.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiDAO.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiDividend.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiIIndex.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiIndex.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiMIndex.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiProtocolIndex.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiPubkeySharingIndex.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiSIndex.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiTypes.ts delete mode 100644 app/lib/dal/indexDAL/loki/LokiWallet.ts delete mode 100644 test/fast/dal/basic-loki.ts delete mode 100644 test/fast/dal/iindex-loki.ts diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts index bf6d449f2..0f701ccb1 100644 --- a/app/lib/dal/fileDAL.ts +++ b/app/lib/dal/fileDAL.ts @@ -69,7 +69,6 @@ import {cliprogram} from "../common-libs/programOptions" import {DividendDAO, UDSource} from "./indexDAL/abstract/DividendDAO" import {HttpSource, HttpUD} from "../../modules/bma/lib/dtos" import {GenericDAO} from "./indexDAL/abstract/GenericDAO" -import {LokiDAO} from "./indexDAL/loki/LokiDAO" import {MonitorExecutionTime} from "../debug/MonitorExecutionTime" import {LevelDBDividend} from "./indexDAL/leveldb/LevelDBDividend" import {LevelDBBindex} from "./indexDAL/leveldb/LevelDBBindex" @@ -136,7 +135,6 @@ export class FileDAL { dividendDAL:DividendDAO newDals:{ [k:string]: Initiable } private dals:(BlockchainArchiveDAO<any>|PeerDAO|WalletDAO|GenericDAO<any>)[] - private lokiDaos:LokiDAO[] = [] loadConfHook: (conf:ConfDTO) => Promise<void> saveConfHook: (conf:ConfDTO) => Promise<ConfDTO> diff --git a/app/lib/dal/indexDAL/loki/LokiBIndex.ts b/app/lib/dal/indexDAL/loki/LokiBIndex.ts deleted file mode 100644 index 8c38b2025..000000000 --- a/app/lib/dal/indexDAL/loki/LokiBIndex.ts +++ /dev/null @@ -1,99 +0,0 @@ -import {DBHead} from "../../../db/DBHead" -import {BIndexDAO} from "../abstract/BIndexDAO" -import {NewLogger} from "../../../logger" -import {MonitorLokiExecutionTime} from "../../../debug/MonitorLokiExecutionTime" -import {LokiProtocolIndex} from "./LokiProtocolIndex" -import {MonitorExecutionTime} from "../../../debug/MonitorExecutionTime" - -const logger = NewLogger() - -export class LokiBIndex extends LokiProtocolIndex<DBHead> implements BIndexDAO { - - private HEAD:DBHead|null = null - - constructor(loki:any) { - super(loki, 'bindex', ['number', 'hash']) - } - - @MonitorExecutionTime() - async insert(record: DBHead): Promise<void> { - this.HEAD = record - return super.insert(record); - } - - @MonitorExecutionTime() - async removeBlock(blockstamp: string): Promise<void> { - this.HEAD = await this.head(2) - return super.removeBlock(blockstamp); - } - - @MonitorExecutionTime() - async head(n: number): Promise<DBHead> { - if (!n) { - throw "Cannot read HEAD~0, which is the incoming block" - } - if (n === 1 && this.HEAD) { - // Cached - return this.HEAD - } else if (this.HEAD) { - // Another than HEAD - return this.collection - .find({ number: this.HEAD.number - n + 1 })[0] - } else { - // Costly method, as a fallback - return this.collection - .chain() - .find({}) - .simplesort('number', true) - .data()[n - 1] - } - } - - @MonitorExecutionTime() - async range(n: number, m: number): Promise<DBHead[]> { - if (!n) { - throw "Cannot read HEAD~0, which is the incoming block" - } - const HEAD = await this.head(1) - if (!HEAD) { - return [] - } - return this.collection - .chain() - .find({ - $and: [ - { number: { $lte: HEAD.number - n + 1 } }, - { number: { $gte: HEAD.number - m + 1 } }, - ] - }) - .simplesort('number', true) - .data().slice(n - 1, m) - } - - @MonitorExecutionTime() - async tail(): Promise<DBHead> { - const HEAD = await this.head(1) - if (!HEAD) { - return HEAD - } - const nbHEADs = this.collection.length() - return this.collection - .find({ number: HEAD.number - nbHEADs + 1 })[0] - } - - @MonitorExecutionTime() - @MonitorLokiExecutionTime(true) - async trimBlocks(maxnumber: number): Promise<void> { - this.collection - .chain() - .find({ number: { $lt: maxnumber }}) - .remove() - } - - @MonitorExecutionTime() - @MonitorLokiExecutionTime(true) - async getWrittenOn(blockstamp: string): Promise<DBHead[]> { - const criterion:any = { number: parseInt(blockstamp) } - return this.collection.find(criterion) - } -} diff --git a/app/lib/dal/indexDAL/loki/LokiBlockchain.ts b/app/lib/dal/indexDAL/loki/LokiBlockchain.ts deleted file mode 100644 index 8a208cd91..000000000 --- a/app/lib/dal/indexDAL/loki/LokiBlockchain.ts +++ /dev/null @@ -1,270 +0,0 @@ -import {BlockchainDAO} from "../abstract/BlockchainDAO" -import {DBBlock} from "../../../db/DBBlock" -import {MonitorLokiExecutionTime} from "../../../debug/MonitorLokiExecutionTime" -import {LokiProtocolIndex} from "./LokiProtocolIndex" -import {MonitorExecutionTime} from "../../../debug/MonitorExecutionTime" - -export class LokiBlockchain extends LokiProtocolIndex<DBBlock> implements BlockchainDAO { - - private current:DBBlock|null = null - - constructor(loki:any) { - super(loki, 'blockchain', ['number', 'hash', 'fork']) - } - - cleanCache(): void { - super.cleanCache() - this.current = null - } - - @MonitorExecutionTime() - async getCurrent() { - if (this.current) { - // Cached - return this.current - } else { - // Costly method, as a fallback - return this.collection - .chain() - .find({ - fork: false - }) - .simplesort('number', true) - .data()[0] - } - } - - @MonitorExecutionTime() - @MonitorLokiExecutionTime(true) - async getBlock(number:string | number) { - return this.collection - .chain() - .find({ - number: parseInt(String(number)), - fork: false - }) - .data()[0] - } - - @MonitorExecutionTime() - async getPotentialRoots() { - return this.collection - .chain() - .find({ number: 0, fork: true }) - .data() - } - - @MonitorExecutionTime() - async saveBunch(blocks:DBBlock[]) { - return this.insertBatch(blocks) - } - - @MonitorExecutionTime() - async insertBatch(records: DBBlock[]): Promise<void> { - const lastInBatch = records[records.length - 1] - if (!this.current || this.current.number < lastInBatch.number) { - this.current = lastInBatch - } - return super.insertBatch(records) - } - - @MonitorExecutionTime() - async removeBlock(blockstamp: string): Promise<void> { - // Never remove blocks - } - - @MonitorExecutionTime() - async removeForkBlock(number:number): Promise<void> { - await this.collection - .chain() - .find({ - fork: true, - number - }) - .remove() - } - - @MonitorExecutionTime() - async removeForkBlockAboveOrEqual(number:number): Promise<void> { - await this.collection - .chain() - .find({ - fork: true, - number: { $gte: number } - }) - .remove() - } - - @MonitorExecutionTime() - async trimBlocks(number:number): Promise<void> { - await this.collection - .chain() - .find({ - number: { $lte: number } - }) - .remove() - } - - @MonitorExecutionTime() - async getAbsoluteBlock(number: number, hash: string): Promise<DBBlock | null> { - return this.collection - .chain() - .find({ - number, - hash - }) - .data()[0] - } - - @MonitorExecutionTime() - async getBlocks(start: number, end: number): Promise<DBBlock[]> { - return this.collection - .chain() - .find({ - number: { $between: [start, end] }, - fork: false - }) - .simplesort('number') - .data() - } - - @MonitorExecutionTime() - async getCountOfBlocksIssuedBy(issuer: string): Promise<number> { - return this.collection - .chain() - .find({ - issuer, - fork: false - }) - .data() - .length - } - - @MonitorExecutionTime() - async getNextForkBlocks(number: number, hash: string): Promise<DBBlock[]> { - return this.collection - .chain() - .find({ - fork: true, - number: number + 1, - previousHash: hash - }) - .simplesort('number') - .data() - } - - @MonitorExecutionTime() - async getPotentialForkBlocks(numberStart: number, medianTimeStart: number, maxNumber: number): Promise<DBBlock[]> { - return this.collection - .chain() - .find({ - fork: true, - number: { $between: [numberStart, maxNumber] }, - medianTime: { $gte: medianTimeStart } - }) - .simplesort('number', true) - .data() - } - - @MonitorExecutionTime() - async lastBlockOfIssuer(issuer: string): Promise<DBBlock | null> { - return this.collection - .chain() - .find({ - fork: false, - issuer - }) - .simplesort('number', true) - .data()[0] - } - - @MonitorExecutionTime() - async lastBlockWithDividend(): Promise<DBBlock | null> { - return this.collection - .chain() - .find({ - fork: false, - dividend: { $gt: 0 } - }) - .simplesort('number', true) - .data()[0] - } - - @MonitorExecutionTime() - async saveBlock(block: DBBlock): Promise<DBBlock> { - if (!this.current || this.current.number < block.number) { - this.current = block; - } - return this.insertOrUpdate(block, false) - } - - @MonitorExecutionTime() - async saveSideBlock(block: DBBlock): Promise<DBBlock> { - return this.insertOrUpdate(block, true) - } - - @MonitorExecutionTime() - async insertOrUpdate(block: DBBlock, isFork:boolean): Promise<DBBlock> { - block.fork = isFork - const conditions = { number: block.number, hash: block.hash } - const existing = (await this.findRaw(conditions))[0] - if (existing && existing.fork !== isFork) { - // Existing block: we only allow to change the fork flag - this.collection - .chain() - .find(conditions) - .update(b => { - b.fork = isFork - b.monetaryMass = block.monetaryMass - b.dividend = block.dividend - }) - } - else if (!existing) { - await this.insert(block) - } - return block - } - - @MonitorExecutionTime() - async dropNonForkBlocksAbove(number: number): Promise<void> { - this.collection - .chain() - .find({ - fork: false, - number: { $gt: number } - }) - .remove() - } - - @MonitorExecutionTime() - async setSideBlock(number: number, previousBlock: DBBlock | null): Promise<void> { - this.collection - .chain() - .find({ - number - }) - .update((b:DBBlock) => { - b.fork = true - }) - // Also update the cache if concerned - if (this.current && this.current.number === number) { - if (previousBlock && this.current.previousHash === previousBlock.hash) { - this.current = previousBlock - } else { - this.current = null - } - } - } - - @MonitorExecutionTime() - async getNonForkChunk(start: number, end: number): Promise<DBBlock[]> { - return this.collection - .chain() - .find({ - fork: false, - number: { $between: [start, end ]} - }) - .simplesort('number') - .data() - } -} diff --git a/app/lib/dal/indexDAL/loki/LokiCollection.ts b/app/lib/dal/indexDAL/loki/LokiCollection.ts deleted file mode 100644 index e160608b6..000000000 --- a/app/lib/dal/indexDAL/loki/LokiCollection.ts +++ /dev/null @@ -1,59 +0,0 @@ -import {LokiChainableFind, LokiCollection} from "./LokiTypes" -import {NewLogger} from "../../../logger" -import {getMicrosecondsTime} from "../../../../ProcessCpuProfiler" - -const logger = NewLogger() - -export class LokiProxyCollection<T> implements LokiCollection<T> { - - constructor(public collection:LokiCollection<T>, private collectionName:string) { - } - - get data() { - return this.collection.data - } - - length(): number { - return this.collection.data.length - } - - insert(entity:T) { - this.collection.insert(entity) - } - - remove(entity:T) { - this.collection.remove(entity) - } - - find(criterion:{ [t in keyof T|'$or'|'$and']?: any }) { - const now = getMicrosecondsTime() - const res = this.collection.find(criterion) - // logger.trace('[loki][%s][find] %sµs', this.collectionName, getDurationInMicroSeconds(now), criterion) - return res - } - - chain(): LokiChainableFind<T> { - return this.collection.chain() - } - - setChangesApi(enabled: boolean) { - this.collection.setChangesApi(enabled) - // This is a Loki bug: `disableDeltaChangesApi` should be impacted just like `disableChangesApi`: - ;(this.collection as any).disableDeltaChangesApi = !enabled - } - - // Returns the real Loki property - get disableChangesApi() { - return this.collection.disableChangesApi - } - - // Returns the real Loki property - get disableDeltaChangesApi() { - return this.collection.disableDeltaChangesApi - } - - get changes() { - return this.collection.changes - } - -} diff --git a/app/lib/dal/indexDAL/loki/LokiCollectionManager.ts b/app/lib/dal/indexDAL/loki/LokiCollectionManager.ts deleted file mode 100755 index f322003bb..000000000 --- a/app/lib/dal/indexDAL/loki/LokiCollectionManager.ts +++ /dev/null @@ -1,54 +0,0 @@ -import {LokiCollection} from "./LokiTypes" -import {LokiProxyCollection} from "./LokiCollection" -import {NewLogger} from "../../../logger" -import {LokiDAO} from "./LokiDAO" -import {cliprogram} from "../../../common-libs/programOptions" -import {Initiable} from "../../sqliteDAL/Initiable" - -const logger = NewLogger() - -export abstract class LokiCollectionManager<T> implements LokiDAO, Initiable { - - protected collection:LokiCollection<T> - protected collectionIsInitialized: Promise<void> - protected resolveCollection: () => void - - public constructor( - protected loki:any, - protected collectionName:'iindex'|'mindex'|'cindex'|'sindex'|'bindex'|'blockchain'|'txs'|'wallet'|'peer'|'dividend', - protected indices: (keyof T)[]) { - this.collectionIsInitialized = new Promise<void>(res => this.resolveCollection = res) - } - - get lokiCollection(): LokiCollection<T> { - return this.collection - } - - public triggerInit() { - const coll = this.loki.addCollection(this.collectionName, { - indices: this.indices, - disableChangesApi: cliprogram.isSync - }) - this.collection = new LokiProxyCollection(coll, this.collectionName) - this.resolveCollection() - } - - public enableChangesAPI() { - this.collection.setChangesApi(true) - } - - public disableChangesAPI() { - this.collection.setChangesApi(false) - } - - async init(): Promise<void> { - await this.collectionIsInitialized - logger.info('Collection %s ready', this.collectionName) - } - - async close(): Promise<void> { - } - - cleanCache(): void { - } -} diff --git a/app/lib/dal/indexDAL/loki/LokiDAO.ts b/app/lib/dal/indexDAL/loki/LokiDAO.ts deleted file mode 100644 index b060804d7..000000000 --- a/app/lib/dal/indexDAL/loki/LokiDAO.ts +++ /dev/null @@ -1,10 +0,0 @@ -import {LokiCollection} from "./LokiTypes" - -export interface LokiDAO { - - enableChangesAPI(): void - - disableChangesAPI(): void - - lokiCollection: LokiCollection<any> -} diff --git a/app/lib/dal/indexDAL/loki/LokiDividend.ts b/app/lib/dal/indexDAL/loki/LokiDividend.ts deleted file mode 100644 index 73b489820..000000000 --- a/app/lib/dal/indexDAL/loki/LokiDividend.ts +++ /dev/null @@ -1,157 +0,0 @@ -import {LokiIndex} from "./LokiIndex" -import {DividendDAO, DividendEntry, UDSource} from "../abstract/DividendDAO" -import {IindexEntry, SimpleTxInput, SimpleUdEntryForWallet, SindexEntry} from "../../../indexer" -import {DataErrors} from "../../../common-libs/errors" -import {MonitorExecutionTime} from "../../../debug/MonitorExecutionTime" -import {DividendDaoHandler} from "../common/DividendDaoHandler" - -export class LokiDividend extends LokiIndex<DividendEntry> implements DividendDAO { - - constructor(loki:any) { - super(loki, 'dividend', ['pub']) - } - - @MonitorExecutionTime() - async createMember(pub: string): Promise<void> { - const existing = this.collection.find({ pub })[0] - if (!existing) { - await this.insert(DividendDaoHandler.getNewDividendEntry(pub)) - } else { - await this.setMember(true, pub) - } - } - - @MonitorExecutionTime() - async setMember(member: boolean, pub: string) { - await this.collection - .chain() - .find({ pub }) - .update(r => { - r.member = member - }) - } - - @MonitorExecutionTime() - async deleteMember(pub: string): Promise<void> { - this.collection - .chain() - .find({ pub }) - .remove() - } - - @MonitorExecutionTime() - async produceDividend(blockNumber: number, dividend: number, unitbase: number, local_iindex: IindexEntry[]): Promise<SimpleUdEntryForWallet[]> { - const dividends: SimpleUdEntryForWallet[] = [] - // Then produce the UD - this.collection - .chain() - .find({ member: true }) - .update(r => DividendDaoHandler.produceDividend(r, blockNumber, dividend, unitbase, dividends)) - return dividends - } - - @MonitorExecutionTime() - async consume(filter: SindexEntry[]): Promise<void> { - for (const dividendToConsume of filter) { - this.collection - .chain() - // We look at the dividends of this member - .find({ - pub: dividendToConsume.identifier - }) - // Then we try to consume the dividend being spent - .update(m => DividendDaoHandler.consume(m, dividendToConsume)) - } - } - - @MonitorExecutionTime() - async getUDSources(pub: string): Promise<UDSource[]> { - const member = this.collection - .chain() - .find({ pub }) - .data()[0] - if (!member) { - return [] - } - return DividendDaoHandler.udSources(member) - } - - @MonitorExecutionTime() - async findUdSourceByIdentifierPosAmountBase(identifier: string, pos: number, amount: number, base: number): Promise<SimpleTxInput[]> { - const member = this.collection.find({ pub: identifier })[0] - return DividendDaoHandler.getUDSourceByIdPosAmountBase(member, identifier, pos, amount, base) - } - - @MonitorExecutionTime() - async getUDSource(identifier: string, pos: number): Promise<SimpleTxInput|null> { - const member = this.collection.find({ pub: identifier })[0] - return DividendDaoHandler.getUDSource(member, identifier, pos) - } - - @MonitorExecutionTime() - async getWrittenOn(blockstamp: string): Promise<DividendEntry[]> { - throw Error(DataErrors[DataErrors.DIVIDEND_GET_WRITTEN_ON_SHOULD_NOT_BE_USED_DIVIDEND_DAO]) - } - - @MonitorExecutionTime() - async getWrittenOnUDs(number: number): Promise<SimpleUdEntryForWallet[]> { - const res: SimpleUdEntryForWallet[] = [] - this.collection - .chain() - .find({ availables: { $contains: number } }) - .data() - .map(m => DividendDaoHandler.getWrittenOnUDs(m, number, res)) - return res - } - - @MonitorExecutionTime() - async removeBlock(blockstamp: string): Promise<void> { - throw Error(DataErrors[DataErrors.DIVIDEND_REMOVE_BLOCK_SHOULD_NOT_BE_USED_BY_DIVIDEND_DAO]) - } - - /** - * Remove UD data produced in a block, either UD production or UD consumption. - * @param {number} number Block number to revert the created UDs. - * @returns {Promise<{createdUDsDestroyedByRevert: SimpleUdEntryForWallet[]}>} - */ - @MonitorExecutionTime() - async revertUDs(number: number): Promise<{ - createdUDsDestroyedByRevert: SimpleUdEntryForWallet[] - consumedUDsRecoveredByRevert: SimpleUdEntryForWallet[] - }> { - const createdUDsDestroyedByRevert: SimpleUdEntryForWallet[] = [] - const consumedUDsRecoveredByRevert: SimpleUdEntryForWallet[] = [] - // Remove produced dividends at this block - this.collection - .chain() - .find({ availables: { $contains: number }}) - .update(m => DividendDaoHandler.removeDividendsProduced(m, number, createdUDsDestroyedByRevert)) - // Unconsumed dividends consumed at this block - this.collection - .chain() - .find({ consumed: { $contains: number }}) - .update(m => DividendDaoHandler.unconsumeDividends(m, number, consumedUDsRecoveredByRevert)) - return { - createdUDsDestroyedByRevert, - consumedUDsRecoveredByRevert, - } - } - - @MonitorExecutionTime() - async findForDump(criterion: any): Promise<SindexEntry[]> { - return DividendDaoHandler.toDump(await this.findRaw(criterion)) - } - - @MonitorExecutionTime() - async trimConsumedUDs(belowNumber: number): Promise<void> { - // Remove dividends consumed before `belowNumber` - this.collection - .chain() - .find({}) - .update(m => DividendDaoHandler.trimConsumed(m, belowNumber)) - } - - async listAll(): Promise<DividendEntry[]> { - return this.collection.find({}) - } -} diff --git a/app/lib/dal/indexDAL/loki/LokiIIndex.ts b/app/lib/dal/indexDAL/loki/LokiIIndex.ts deleted file mode 100644 index 1c9f98f59..000000000 --- a/app/lib/dal/indexDAL/loki/LokiIIndex.ts +++ /dev/null @@ -1,158 +0,0 @@ -import {FullIindexEntry, IindexEntry, Indexer} from "../../../indexer" -import {IIndexDAO} from "../abstract/IIndexDAO" -import {LokiPubkeySharingIndex} from "./LokiPubkeySharingIndex" -import {OldIindexEntry} from "../../../db/OldIindexEntry" -import {MonitorExecutionTime} from "../../../debug/MonitorExecutionTime" -import {OldTransformers} from "../common/OldTransformer" - -export class LokiIIndex extends LokiPubkeySharingIndex<IindexEntry> implements IIndexDAO { - - constructor(loki:any) { - super(loki, 'iindex', [ - 'pub', - 'uid', - 'member', - ]) - } - - @MonitorExecutionTime() - async reducable(pub: string): Promise<IindexEntry[]> { - return this.findByPub(pub) - } - - @MonitorExecutionTime() - async findByPub(pub: string): Promise<IindexEntry[]> { - return this.collection.chain() - .find({ pub }) - .simplesort('writtenOn') - .data() - } - - @MonitorExecutionTime() - async findByUid(uid: string): Promise<IindexEntry[]> { - return this.collection.chain() - .find({ uid }) - .simplesort('writtenOn') - .data() - } - - @MonitorExecutionTime() - async getMembers(): Promise<{ pubkey: string; uid: string|null }[]> { - return this.collection - // Those who are still marked member somewhere - .find({ member: true }) - // We reduce them - .map(r => { - return Indexer.DUP_HELPERS.reduce( - this.collection - .chain() - .find({ pub: r.pub }) - .simplesort('writtenOn') - .data() - ) - }) - // We keep only the real members (because we could have excluded) - .filter(r => r.member) - // We map - .map(OldTransformers.toOldIindexEntry) - } - - @MonitorExecutionTime() - async getFromPubkey(pub: string): Promise<FullIindexEntry | null> { - return this.retrieveIdentityOnPubOrNull( - { pub } - ) as Promise<FullIindexEntry|null> - } - - @MonitorExecutionTime() - async getFromUID(uid: string): Promise<FullIindexEntry | null> { - return this.retrieveIdentityOnPubOrNull( - this.collection - .chain() - .find({ uid }) - .data()[0] - ) as Promise<FullIindexEntry|null> - } - - @MonitorExecutionTime() - async getFromPubkeyOrUid(search: string): Promise<FullIindexEntry | null> { - const idty = await this.getFromPubkey(search) - if (idty) { - return idty - } - return this.getFromUID(search) as Promise<FullIindexEntry|null> - } - - @MonitorExecutionTime() - async searchThoseMatching(search: string): Promise<OldIindexEntry[]> { - const reducables = Indexer.DUP_HELPERS.reduceBy(this.collection - .chain() - .find({ - $or: [ - { pub: { $contains: search } }, - { uid: { $contains: search } }, - ] - }) - .data() - , ['pub']) - // We get the full representation for each member - return await Promise.all(reducables.map(async (entry) => { - return OldTransformers.toOldIindexEntry(Indexer.DUP_HELPERS.reduce(await this.reducable(entry.pub))) - })) - } - - @MonitorExecutionTime() - async getFullFromUID(uid: string): Promise<FullIindexEntry> { - return (await this.getFromUID(uid)) as FullIindexEntry - } - - @MonitorExecutionTime() - async getFullFromPubkey(pub: string): Promise<FullIindexEntry> { - return (await this.getFromPubkey(pub)) as FullIindexEntry - } - - @MonitorExecutionTime() - async getFullFromHash(hash: string): Promise<FullIindexEntry> { - return this.retrieveIdentityOnPubOrNull( - this.collection - .chain() - .find({ hash }) - .data()[0] - ) as Promise<FullIindexEntry> - } - - @MonitorExecutionTime() - async retrieveIdentityOnPubOrNull(entry:{ pub:string }|null) { - if (!entry) { - return null - } - return OldTransformers.iindexEntityOrNull( - this.collection - .chain() - .find({ pub: entry.pub }) - .simplesort('writtenOn') - .data() - ) as Promise<FullIindexEntry|null> - } - - @MonitorExecutionTime() - async getToBeKickedPubkeys(): Promise<string[]> { - return this.collection - // Those who are still marked member somewhere - .find({ kick: true }) - // We reduce them - .map(r => { - return Indexer.DUP_HELPERS.reduce( - this.collection - .chain() - .find({ pub: r.pub }) - .simplesort('writtenOn') - .data() - ) - }) - // We keep only the real members (because we could have excluded) - .filter(r => r.kick) - // We map - .map(r => r.pub) - } -} diff --git a/app/lib/dal/indexDAL/loki/LokiIndex.ts b/app/lib/dal/indexDAL/loki/LokiIndex.ts deleted file mode 100644 index db9f53bf9..000000000 --- a/app/lib/dal/indexDAL/loki/LokiIndex.ts +++ /dev/null @@ -1,39 +0,0 @@ -import {GenericDAO} from "../abstract/GenericDAO" -import {LokiCollectionManager} from "./LokiCollectionManager" -import {MonitorLokiExecutionTime} from "../../../debug/MonitorLokiExecutionTime" - -export abstract class LokiIndex<T> extends LokiCollectionManager<T> implements GenericDAO<T> { - - cleanCache(): void { - } - - async insert(record: T): Promise<void> { - this.collection.insert(record) - } - - @MonitorLokiExecutionTime(true) - async findRaw(criterion?:any) { - return this.collection.find(criterion) - } - - @MonitorLokiExecutionTime(true) - async findRawWithOrder(criterion:any, sort:((string|((string|boolean)[]))[])) { - const res = this.collection - .chain() - .find(criterion) - .compoundsort(sort) - return res.data() - } - - @MonitorLokiExecutionTime() - async insertBatch(records: T[]): Promise<void> { - records.map(r => this.insert(r)) - } - - abstract getWrittenOn(blockstamp: string): Promise<T[]> - abstract removeBlock(blockstamp: string): Promise<void> - - async count() { - return this.collection.data.length - } -} diff --git a/app/lib/dal/indexDAL/loki/LokiMIndex.ts b/app/lib/dal/indexDAL/loki/LokiMIndex.ts deleted file mode 100644 index bbd189abc..000000000 --- a/app/lib/dal/indexDAL/loki/LokiMIndex.ts +++ /dev/null @@ -1,104 +0,0 @@ -import {FullMindexEntry, Indexer, MindexEntry} from "../../../indexer" -import {MIndexDAO} from "../abstract/MIndexDAO" -import {LokiPubkeySharingIndex} from "./LokiPubkeySharingIndex" -import {MonitorExecutionTime} from "../../../debug/MonitorExecutionTime" - -export class LokiMIndex extends LokiPubkeySharingIndex<MindexEntry> implements MIndexDAO { - - constructor(loki:any) { - super(loki, 'mindex', ['pub']) - } - - @MonitorExecutionTime() - async findByPubAndChainableOnGt(pub: string, medianTime: number): Promise<MindexEntry[]> { - return this.collection - .find({ - $and: [ - { pub }, - { chainable_on: { $gt: medianTime } }, - ] - }) - } - - @MonitorExecutionTime() - async findExpiresOnLteAndRevokesOnGt(medianTime: number): Promise<string[]> { - return this.collection - .find({ - $and: [ - { expires_on: { $lte: medianTime } }, - { revokes_on: { $gt: medianTime } }, - ] - }) - .map(e => e.pub) - } - - @MonitorExecutionTime() - async findRevokesOnLteAndRevokedOnIsNull(medianTime: number): Promise<string[]> { - return this.collection - .find({ - $and: [ - { revokes_on: { $lte: medianTime } }, - { revoked_on: null }, - ] - }) - .map(e => e.pub) - } - @MonitorExecutionTime() - async getReducedMS(pub: string): Promise<FullMindexEntry | null> { - const reducable = (await this.reducable(pub)) as (FullMindexEntry)[] - if (reducable.length) { - return Indexer.DUP_HELPERS.reduce(reducable) - } - return null - } - - @MonitorExecutionTime() - async getReducedMSForImplicitRevocation(pub: string): Promise<FullMindexEntry | null> { - const reducable = (await this.reducable(pub)) as (FullMindexEntry)[] - if (reducable.length) { - return Indexer.DUP_HELPERS.reduce(reducable) - } - return null - } - - @MonitorExecutionTime() - async getReducedMSForMembershipExpiry(pub: string): Promise<FullMindexEntry | null> { - const reducable = (await this.reducable(pub)) as (FullMindexEntry)[] - if (reducable.length) { - return Indexer.DUP_HELPERS.reduce(reducable) - } - return null - } - - async findPubkeysThatShouldExpire(medianTime: number): Promise<{ pub: string; created_on: string }[]> { - const results: { pub: string; created_on: string }[] = [] - const pubkeys = await this.findExpiresOnLteAndRevokesOnGt(medianTime) - for (const pub of pubkeys) { - const MS = await this.getReducedMS(pub) as FullMindexEntry // We are sure because `memberships` already comes from the MINDEX - const hasRenewedSince = MS.expires_on > medianTime; - if (!MS.expired_on && !hasRenewedSince) { - results.push({ - pub: MS.pub, - created_on: MS.created_on, - }) - } - } - return results - } - - @MonitorExecutionTime() - async getRevokedPubkeys(): Promise<string[]> { - return this.collection - .find({ revoked_on: { $gt: 0 } }) - // We map - .map(r => r.pub) - } - - async reducable(pub: string): Promise<MindexEntry[]> { - return this.collection.chain() - .find({ pub }) - .simplesort('writtenOn') - .data() - } - -} diff --git a/app/lib/dal/indexDAL/loki/LokiProtocolIndex.ts b/app/lib/dal/indexDAL/loki/LokiProtocolIndex.ts deleted file mode 100644 index dafbba713..000000000 --- a/app/lib/dal/indexDAL/loki/LokiProtocolIndex.ts +++ /dev/null @@ -1,27 +0,0 @@ -import {GenericDAO} from "../abstract/GenericDAO" -import {MonitorLokiExecutionTime} from "../../../debug/MonitorLokiExecutionTime" -import {LokiIndex} from "./LokiIndex" - -export interface IndexData { - written_on: string - writtenOn: number -} - -export abstract class LokiProtocolIndex<T extends IndexData> extends LokiIndex<T> implements GenericDAO<T> { - - @MonitorLokiExecutionTime(true) - async getWrittenOn(blockstamp: string): Promise<T[]> { - const criterion:any = { writtenOn: parseInt(blockstamp) } - return this.collection.find(criterion) - } - - @MonitorLokiExecutionTime(true) - async removeBlock(blockstamp: string): Promise<void> { - const data = await this.getWrittenOn(blockstamp) - data.map(d => this.collection.remove(d)) - } - - async count() { - return this.collection.data.length - } -} diff --git a/app/lib/dal/indexDAL/loki/LokiPubkeySharingIndex.ts b/app/lib/dal/indexDAL/loki/LokiPubkeySharingIndex.ts deleted file mode 100644 index 4fb0a3fd8..000000000 --- a/app/lib/dal/indexDAL/loki/LokiPubkeySharingIndex.ts +++ /dev/null @@ -1,39 +0,0 @@ -import {Indexer} from "../../../indexer" -import {MonitorLokiExecutionTime} from "../../../debug/MonitorLokiExecutionTime" -import {LokiProtocolIndex} from "./LokiProtocolIndex" - -export class LokiPubkeySharingIndex<T extends { written_on:string, writtenOn:number, pub:string }> extends LokiProtocolIndex<T> { - - @MonitorLokiExecutionTime(true) - async trimRecords(belowNumber: number): Promise<void> { - // TODO: may be optimized by only selecting new offseted records - const criterion:any = { - writtenOn: { - $lt: belowNumber - } - } - const trimmable = await this.collection - .chain() - .find(criterion) - .simplesort('writtenOn') - .data() - const trimmableByPub: { [pub:string]: T[] } = {} - for (const t of trimmable) { - if (!trimmableByPub[t.pub]) { - trimmableByPub[t.pub] = [] - } - trimmableByPub[t.pub].push(t) - } - for (const pub of Object.keys(trimmableByPub)) { - if (trimmableByPub[pub].length > 1) { - // Remove the existing records - for (const t of trimmableByPub[pub]) { - this.collection.remove(t) - } - // Insert a new one that gathers them - const reduced = Indexer.DUP_HELPERS.reduce(trimmableByPub[pub]) - this.collection.insert(reduced) - } - } - } -} diff --git a/app/lib/dal/indexDAL/loki/LokiSIndex.ts b/app/lib/dal/indexDAL/loki/LokiSIndex.ts deleted file mode 100644 index f055d6f3c..000000000 --- a/app/lib/dal/indexDAL/loki/LokiSIndex.ts +++ /dev/null @@ -1,129 +0,0 @@ -import {FullSindexEntry, Indexer, SimpleTxEntryForWallet, SindexEntry} from "../../../indexer" -import {SIndexDAO} from "../abstract/SIndexDAO" -import {Underscore} from "../../../common-libs/underscore" -import {MonitorLokiExecutionTime} from "../../../debug/MonitorLokiExecutionTime" -import {LokiProtocolIndex} from "./LokiProtocolIndex" -import {MonitorExecutionTime} from "../../../debug/MonitorExecutionTime" - -export class LokiSIndex extends LokiProtocolIndex<SindexEntry> implements SIndexDAO { - - constructor(loki:any) { - super(loki, 'sindex', ['identifier', 'conditions', 'writtenOn']) - } - - @MonitorExecutionTime() - async findTxSourceByIdentifierPosAmountBase(identifier: string, pos: number, amount: number, base: number): Promise<SindexEntry[]> { - return this.collection - .chain() - .find({ identifier, pos, amount, base }) - .simplesort('writtenOn') - .data() - .map(src => { - src.type = src.tx ? 'T' : 'D' - return src - }) - } - - @MonitorExecutionTime() - async getAvailableForConditions(conditionsStr: string): Promise<SindexEntry[]> { - const sources = this.collection - .chain() - .find({ conditions: conditionsStr }) - .simplesort('writtenOn') - .data() - .filter(s => this.collection.find({ identifier: s.identifier, pos: s.pos, consumed: true }).length === 0) - .map(src => { - src.type = src.tx ? 'T' : 'D' - return src - }) - return Underscore.sortBy(sources, (row:SindexEntry) => row.type == 'D' ? 0 : 1) - } - - @MonitorExecutionTime() - async getAvailableForPubkey(pubkey: string): Promise<{ amount: number; base: number, conditions: string, identifier: string, pos: number }[]> { - return this.collection - .chain() - .find({ conditions: { $regex: 'SIG\\(' + pubkey + '\\)' } }) - .simplesort('writtenOn') - .data() - .filter(s => this.collection.find({ identifier: s.identifier, pos: s.pos, consumed: true }).length === 0) - .map(src => { - src.type = src.tx ? 'T' : 'D' - return src - }) - } - - @MonitorExecutionTime() - async getTxSource(identifier: string, pos: number): Promise<FullSindexEntry | null> { - const reducables = this.collection - .chain() - .find({ identifier, pos }) - .compoundsort([['writtenOn', false], ['op', false]]) - .data() - .map(src => { - src.type = src.tx ? 'T' : 'D' - return src - }) - if (reducables.length === 0) { - return null - } - return Indexer.DUP_HELPERS.reduce(reducables) - } - - @MonitorExecutionTime() - @MonitorLokiExecutionTime(true) - async trimConsumedSource(belowNumber: number): Promise<void> { - const consumed = this.collection - .find({ writtenOn: { $lt: belowNumber }}) - .filter(s => s.consumed) - for (const e of consumed) { - this.collection - .chain() - .find({ - identifier: e.identifier, - pos: e.pos - }) - .remove() - } - } - - /** - * For SINDEX, trimming records <=> removing the consumed sources. - * @param {number} belowNumber Number below which a consumed source must be removed. - * @returns {Promise<void>} - */ - @MonitorExecutionTime() - async trimRecords(belowNumber: number): Promise<void> { - return this.trimConsumedSource(belowNumber) - } - - @MonitorExecutionTime() - async getWrittenOnTxs(blockstamp: string): Promise<SimpleTxEntryForWallet[]> { - const entries = (await this.getWrittenOn(blockstamp)) - const res: SimpleTxEntryForWallet[] = [] - entries.forEach(s => { - res.push({ - srcType: 'T', - op: s.op, - conditions: s.conditions, - amount: s.amount, - base: s.base, - identifier: s.identifier, - pos: s.pos - }) - }) - return res - } - - @MonitorExecutionTime() - async findByIdentifier(identifier: string): Promise<SindexEntry[]> { - return this.findRaw({ identifier }) - } - - @MonitorExecutionTime() - async findByPos(pos: number): Promise<SindexEntry[]> { - return this.findRaw({ pos }) - } - - -} diff --git a/app/lib/dal/indexDAL/loki/LokiTypes.ts b/app/lib/dal/indexDAL/loki/LokiTypes.ts deleted file mode 100644 index 816b87353..000000000 --- a/app/lib/dal/indexDAL/loki/LokiTypes.ts +++ /dev/null @@ -1,47 +0,0 @@ - -export interface RealLokiCollection<T> { - - data: T[] - - length(): number - - insert(entity:T): void - - remove(entity:T): void - - find(criterion:{ [t in keyof T|'$or'|'$and']?: any }): T[] - - chain(): LokiChainableFind<T> - - setChangesApi(disable: boolean): void - - disableChangesApi: boolean - - disableDeltaChangesApi: boolean - - changes: any[] -} - -export interface LokiCollection<T> extends RealLokiCollection<T> { - - collection: RealLokiCollection<T> -} - -export interface LokiChainableFind<T> { - - find(criterion:{ [t in keyof T|'$or'|'$and'|'pub']?: any }): LokiChainableFind<T> - - simplesort(prop:keyof T, desc?:boolean): LokiChainableFind<T> - - limit(l:number): LokiChainableFind<T> - - update(cb:(t:T) => void): LokiChainableFind<T> - - where(filter:(t:T) => boolean): LokiChainableFind<T> - - remove(): LokiChainableFind<T> - - compoundsort(sort:((string|((string|boolean)[]))[])): LokiChainableFind<T> - - data(): T[] -} \ No newline at end of file diff --git a/app/lib/dal/indexDAL/loki/LokiWallet.ts b/app/lib/dal/indexDAL/loki/LokiWallet.ts deleted file mode 100644 index 2d074d5a7..000000000 --- a/app/lib/dal/indexDAL/loki/LokiWallet.ts +++ /dev/null @@ -1,48 +0,0 @@ -import {WalletDAO} from "../abstract/WalletDAO" -import {LokiCollectionManager} from "./LokiCollectionManager" -import {DBWallet} from "../../../db/DBWallet" -import {MonitorExecutionTime} from "../../../debug/MonitorExecutionTime" -import {LokiDAO} from "./LokiDAO" - -export class LokiWallet extends LokiCollectionManager<DBWallet> implements WalletDAO, LokiDAO { - - constructor(loki:any) { - super(loki, 'wallet', ['conditions']) - } - - cleanCache(): void { - } - - @MonitorExecutionTime() - async getWallet(conditions: string): Promise<DBWallet|null> { - return this.collection - .find({ conditions })[0] - } - - @MonitorExecutionTime() - async insertBatch(records: DBWallet[]): Promise<void> { - for (const w of records) { - this.collection.insert(w) - } - } - - @MonitorExecutionTime() - async saveWallet(wallet: DBWallet): Promise<DBWallet> { - let updated = false - this.collection - .chain() - .find({ conditions: wallet.conditions }) - .update(w => { - w.balance = wallet.balance - updated = true - }) - if (!updated) { - await this.insertBatch([wallet]) - } - return wallet - } - - async listAll(): Promise<DBWallet[]> { - return this.collection.find({}) - } -} diff --git a/test/fast/dal/basic-loki.ts b/test/fast/dal/basic-loki.ts deleted file mode 100644 index 2f26a3b84..000000000 --- a/test/fast/dal/basic-loki.ts +++ /dev/null @@ -1,50 +0,0 @@ -// Source file from duniter: Crypto-currency software to manage libre currency such as Ğ1 -// Copyright (C) 2018 Cedric Moreau <cem.moreau@gmail.com> -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -import * as assert from "assert" -import {LokiIndex} from "../../../app/lib/dal/indexDAL/loki/LokiIndex" -import {LokiProtocolIndex} from "../../../app/lib/dal/indexDAL/loki/LokiProtocolIndex" - -const loki = require('lokijs') - -interface TestEntity { - name: string - written_on: string - writtenOn: number -} - -let lokiIndex:LokiIndex<TestEntity> - -class TheIndex extends LokiProtocolIndex<TestEntity> { -} - -describe("Basic LokiJS database", () => { - - before(async () => { - lokiIndex = new TheIndex(new loki('index.db'), 'iindex', []) - await lokiIndex.triggerInit() - await lokiIndex.init() - }) - - it('should be able instanciate the index', async () => { - assert.notEqual(null, lokiIndex) - assert.notEqual(undefined, lokiIndex) - }) - - it('should be able add new records', async () => { - assert.equal(0, (await lokiIndex.findRaw()).length) - await lokiIndex.insert({ written_on: '9-ABC', writtenOn: 9, name: 'A' }) - assert.equal(1, (await lokiIndex.findRaw()).length) - }) - -}) diff --git a/test/fast/dal/iindex-loki.ts b/test/fast/dal/iindex-loki.ts deleted file mode 100644 index 107f32193..000000000 --- a/test/fast/dal/iindex-loki.ts +++ /dev/null @@ -1,77 +0,0 @@ -// Source file from duniter: Crypto-currency software to manage libre currency such as Ğ1 -// Copyright (C) 2018 Cedric Moreau <cem.moreau@gmail.com> -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. - -import * as assert from "assert" -import {LokiIIndex} from "../../../app/lib/dal/indexDAL/loki/LokiIIndex" - -const loki = require('lokijs') - -let lokiIndex:LokiIIndex - -describe("IIndex LokiJS", () => { - - before(async () => { - lokiIndex = new LokiIIndex(new loki('index.db')) - await lokiIndex.triggerInit() - await lokiIndex.init() - }) - - it('should be able instanciate the index', async () => { - assert.notEqual(null, lokiIndex) - assert.notEqual(undefined, lokiIndex) - }) - - it('should be able to add new records', async () => { - assert.equal(0, (await lokiIndex.findRaw()).length) - await lokiIndex.insert({ - index: 'iindex', - op: 'CREATE', - uid: 'test-uid', - pub: 'test-pub', - hash: 'test-hash', - sig: 'test-sig', - created_on: '1-HASH_1', - written_on: '2-HASH_2', - writtenOn: 2, - age: 0, - member: true, - wasMember: true, - kick: false, - wotb_id: null - }) - await lokiIndex.insert({ - index: 'iindex', - op: 'UPDATE', - uid: null, - pub: 'test-pub', - hash: null, - sig: null, - created_on: '1-HASH_1', - written_on: '3-HASH_3', - writtenOn: 3, - age: 0, - member: false, - wasMember: true, - kick: false, - wotb_id: null - }) - assert.equal(2, (await lokiIndex.findRaw()).length) - }) - - it('should be able to trim records', async () => { - assert.equal(2, (await lokiIndex.findRaw()).length) - await lokiIndex.trimRecords(4) - assert.equal(1, (await lokiIndex.findRaw()).length) - }) - -}) -- GitLab