From 15752f8b25d7933d466bbacb0137c3ebe0060239 Mon Sep 17 00:00:00 2001
From: cgeek <cem.moreau@gmail.com>
Date: Mon, 23 Apr 2018 17:55:34 +0200
Subject: [PATCH] Test sharded storage
---
app/lib/blockchain/DuniterBlockchain.ts | 4 ++-
app/lib/dal/fileDAL.ts | 2 ++
app/lib/dal/indexDAL/abstract/IIndexDAO.ts | 2 ++
app/lib/dal/indexDAL/loki/LokiIIndex.ts | 7 +++++
.../indexDAL/loki/LokiPubkeySharingIndex.ts | 12 ++++----
app/lib/indexer.ts | 28 +++++++++----------
app/lib/other_constants.ts | 2 +-
app/modules/reapply.ts | 12 ++++++++
8 files changed, 47 insertions(+), 22 deletions(-)
diff --git a/app/lib/blockchain/DuniterBlockchain.ts b/app/lib/blockchain/DuniterBlockchain.ts
index e7961febd..da6a576ba 100644
--- a/app/lib/blockchain/DuniterBlockchain.ts
+++ b/app/lib/blockchain/DuniterBlockchain.ts
@@ -143,7 +143,9 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// BR_G85
if (Indexer.ruleMembershipExcludedIsMember(iindex) === false) throw Error('ruleMembershipExcludedIsMember');
// BR_G86
- if ((await Indexer.ruleToBeKickedArePresent(iindex, dal)) === false) throw Error('ruleToBeKickedArePresent');
+ if ((await Indexer.ruleToBeKickedArePresent(iindex, dal)) === false) {
+ throw Error('ruleToBeKickedArePresent')
+ }
// BR_G103
if (Indexer.ruleTxWritability(sindex) === false) throw Error('ruleTxWritability');
// BR_G87
diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts
index ed49de19c..397f99ae2 100644
--- a/app/lib/dal/fileDAL.ts
+++ b/app/lib/dal/fileDAL.ts
@@ -54,6 +54,7 @@ import {NewLogger} from "../logger"
const fs = require('fs')
const loki = require('lokijs')
+const lokiAdapter = require('./lokifsadapater')
const path = require('path')
const readline = require('readline')
const _ = require('underscore');
@@ -104,6 +105,7 @@ export class FileDAL {
this.profile = 'DAL'
const that = this
this.loki = new loki(path.join(this.rootPath, Directory.INDEX_DB_FILE), {
+ adapter: new lokiAdapter(),
autoload: true,
autoloadCallback : () => {
const dals = [
diff --git a/app/lib/dal/indexDAL/abstract/IIndexDAO.ts b/app/lib/dal/indexDAL/abstract/IIndexDAO.ts
index 22ef18d56..d85975ce3 100644
--- a/app/lib/dal/indexDAL/abstract/IIndexDAO.ts
+++ b/app/lib/dal/indexDAL/abstract/IIndexDAO.ts
@@ -29,4 +29,6 @@ export interface IIndexDAO extends ReduceableDAO<IindexEntry> {
getMembersPubkeys(): Promise<{ pub:string }[]>
getToBeKickedPubkeys(): Promise<string[]>
+
+ findAllByWrittenOn(): Promise<IindexEntry[]>
}
diff --git a/app/lib/dal/indexDAL/loki/LokiIIndex.ts b/app/lib/dal/indexDAL/loki/LokiIIndex.ts
index 74f510a23..1179ef77d 100644
--- a/app/lib/dal/indexDAL/loki/LokiIIndex.ts
+++ b/app/lib/dal/indexDAL/loki/LokiIIndex.ts
@@ -17,6 +17,13 @@ export class LokiIIndex extends LokiPubkeySharingIndex<IindexEntry> implements I
return this.findByPub(pub)
}
+ async findAllByWrittenOn(): Promise<IindexEntry[]> {
+ return this.collection.chain()
+ .find({})
+ .simplesort('writtenOn')
+ .data()
+ }
+
async findByPub(pub: string): Promise<IindexEntry[]> {
return this.collection.chain()
.find({ pub })
diff --git a/app/lib/dal/indexDAL/loki/LokiPubkeySharingIndex.ts b/app/lib/dal/indexDAL/loki/LokiPubkeySharingIndex.ts
index 36236c07f..5693a0ce9 100644
--- a/app/lib/dal/indexDAL/loki/LokiPubkeySharingIndex.ts
+++ b/app/lib/dal/indexDAL/loki/LokiPubkeySharingIndex.ts
@@ -25,12 +25,12 @@ export class LokiPubkeySharingIndex<T extends { written_on:string, writtenOn:num
for (const pub of Object.keys(trimmableByPub)) {
if (trimmableByPub[pub].length > 1) {
// Remove the existing records
- for (const t of trimmableByPub[pub]) {
- this.collection.remove(t)
- }
- // Insert a new one that gathers them
- const reduced = Indexer.DUP_HELPERS.reduce(trimmableByPub[pub])
- this.collection.insert(reduced)
+ // for (const t of trimmableByPub[pub]) {
+ // this.collection.remove(t)
+ // }
+ // // Insert a new one that gathers them
+ // const reduced = Indexer.DUP_HELPERS.reduce(trimmableByPub[pub])
+ // this.collection.insert(reduced)
}
}
}
diff --git a/app/lib/indexer.ts b/app/lib/indexer.ts
index 8e17bf2b6..11c1f1ffa 100644
--- a/app/lib/indexer.ts
+++ b/app/lib/indexer.ts
@@ -567,20 +567,20 @@ export class Indexer {
// BR_G16
await Indexer.prepareSpeed(HEAD, head, conf)
- // // BR_G19
- // await Indexer.prepareIdentitiesAge(iindex, HEAD, HEAD_1, conf, dal);
-
- // // BR_G22
- // await Indexer.prepareMembershipsAge(mindex, HEAD, HEAD_1, conf, dal);
- //
- // // BR_G37
- // await Indexer.prepareCertificationsAge(cindex, HEAD, HEAD_1, conf, dal);
- //
- // // BR_G104
- // await Indexer.ruleIndexCorrectMembershipExpiryDate(HEAD, mindex, dal);
- //
- // // BR_G105
- // await Indexer.ruleIndexCorrectCertificationExpiryDate(HEAD, cindex, dal);
+ // BR_G19
+ await Indexer.prepareIdentitiesAge(iindex, HEAD, HEAD_1, conf, dal);
+
+ // BR_G22
+ await Indexer.prepareMembershipsAge(mindex, HEAD, HEAD_1, conf, dal);
+
+ // BR_G37
+ await Indexer.prepareCertificationsAge(cindex, HEAD, HEAD_1, conf, dal);
+
+ // BR_G104
+ await Indexer.ruleIndexCorrectMembershipExpiryDate(HEAD, mindex, dal);
+
+ // BR_G105
+ await Indexer.ruleIndexCorrectCertificationExpiryDate(HEAD, cindex, dal);
return HEAD;
}
diff --git a/app/lib/other_constants.ts b/app/lib/other_constants.ts
index d10528e99..6fc9b81f4 100644
--- a/app/lib/other_constants.ts
+++ b/app/lib/other_constants.ts
@@ -14,7 +14,7 @@
export const OtherConstants = {
MUTE_LOGS_DURING_UNIT_TESTS: false,
- SQL_TRACES: false,
+ SQL_TRACES: true,
BC_EVENT: {
SWITCHED: 'switched',
diff --git a/app/modules/reapply.ts b/app/modules/reapply.ts
index 0fcb1b206..ac58ece38 100644
--- a/app/modules/reapply.ts
+++ b/app/modules/reapply.ts
@@ -34,6 +34,18 @@ module.exports = {
await server.disconnect();
}
}
+ }, {
+ name: 'db-dump',
+ desc: 'Dump some db data',
+ preventIfRunning: true,
+ onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
+ const data = await server.dal.iindexDAL.findAllByWrittenOn()
+ for (const d of data) {
+ if (d.pub === "9DDn592RMWfka6fPtTGkmAS54CkYxohDGuk41EECxioD") {
+ console.log("%s %s", d.pub, d.kick)
+ }
+ }
+ }
}]
}
}
--
GitLab