From e7da47ed406e528b2a47c98df0240210649b92d1 Mon Sep 17 00:00:00 2001
From: cgeek <cem.moreau@gmail.com>
Date: Thu, 8 Nov 2018 21:10:58 +0100
Subject: [PATCH] [enh] Allow full transactions history storage

---
 app/lib/common-libs/programOptions.ts         |  1 +
 app/lib/dto/ConfDTO.ts                        | 13 ++++++++--
 app/modules/config.ts                         | 24 ++++++++++++++++++-
 .../crawler/lib/sync/v2/GlobalIndexStream.ts  |  8 +++++--
 4 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/app/lib/common-libs/programOptions.ts b/app/lib/common-libs/programOptions.ts
index 2c1949959..5605bcea7 100644
--- a/app/lib/common-libs/programOptions.ts
+++ b/app/lib/common-libs/programOptions.ts
@@ -27,6 +27,7 @@ export interface ProgramOptions {
   loglevel?: string
   sqlTraces?: boolean
   memory?: boolean
+  storeTxs?: boolean
 }
 
 export const cliprogram: ProgramOptions = {
diff --git a/app/lib/dto/ConfDTO.ts b/app/lib/dto/ConfDTO.ts
index ac1625ad1..b17a17ee6 100644
--- a/app/lib/dto/ConfDTO.ts
+++ b/app/lib/dto/ConfDTO.ts
@@ -22,6 +22,12 @@ export interface Keypair {
   sec: string
 }
 
+export interface StorageDTO {
+  storage?: {
+    transactions?:boolean
+  }
+}
+
 export interface PowDTO {
   powNoSecurity:boolean
 }
@@ -103,7 +109,7 @@ export interface WS2PConfDTO {
   }
 }
 
-export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO, BranchingDTO, WS2PConfDTO, PowDTO {
+export class ConfDTO implements StorageDTO, CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO, BranchingDTO, WS2PConfDTO, PowDTO {
 
   constructor(
     public loglevel: string,
@@ -182,7 +188,10 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
       maxPrivate?:number
       syncLimit?:number
     },
-    public powNoSecurity = false
+    public powNoSecurity = false,
+    public storage = {
+      transactions: false
+    },
 ) {}
 
   static mock() {
diff --git a/app/modules/config.ts b/app/modules/config.ts
index 0442cb20a..5eee626cf 100644
--- a/app/modules/config.ts
+++ b/app/modules/config.ts
@@ -17,21 +17,43 @@ import {Server} from "../../server"
 import {CommonConstants} from "../lib/common-libs/constants"
 import {Directory} from "../lib/system/directory"
 import {Underscore} from "../lib/common-libs/underscore"
+import {ProgramOptions} from "../lib/common-libs/programOptions"
 
 module.exports = {
   duniter: {
 
+    cliOptions: [
+      { value: '--store-txs', desc: 'Enable full transaction history storage.' },
+    ],
+
     config: {
-      onLoading: async (conf:ConfDTO) => {
+      onLoading: async (conf:ConfDTO, program: ProgramOptions) => {
         conf.msPeriod = conf.msWindow
         conf.switchOnHeadAdvance = CommonConstants.SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS
+
+        // Transactions storage
+        if (program.storeTxs) {
+          if (!conf.storage) {
+            conf.storage = { transactions: true }
+          }
+          else {
+            conf.storage.transactions = true
+          }
+        }
       },
       beforeSave: async (conf:ConfDTO) => {
         conf.msPeriod = conf.msWindow
         conf.switchOnHeadAdvance = CommonConstants.SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS
+        if (!conf.storage) {
+          conf.storage = {
+            transactions: false
+          }
+        }
       }
     },
 
+
+
     cli: [{
       name: 'config',
       desc: 'Register configuration in database',
diff --git a/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts b/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts
index de32a3a97..cb369b75a 100644
--- a/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts
+++ b/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts
@@ -9,7 +9,7 @@ import {
   SimpleUdEntryForWallet,
   SindexEntry
 } from "../../../../../lib/indexer"
-import {CurrencyConfDTO} from "../../../../../lib/dto/ConfDTO"
+import {ConfDTO, CurrencyConfDTO} from "../../../../../lib/dto/ConfDTO"
 import {FileDAL} from "../../../../../lib/dal/fileDAL"
 import {DuniterBlockchain} from "../../../../../lib/blockchain/DuniterBlockchain"
 import {BlockDTO} from "../../../../../lib/dto/BlockDTO"
@@ -80,7 +80,7 @@ export class GlobalIndexStream extends Duplex {
 
   private mapInjection: { [k: string]: any } = {}
 
-  constructor(private conf: any,
+  constructor(private conf: ConfDTO,
               private dal:FileDAL,
               private to: number,
               private localNumber:number,
@@ -332,6 +332,10 @@ export class GlobalIndexStream extends Duplex {
       return block
     }))
 
+    if (this.conf.storage && this.conf.storage.transactions) {
+      await Promise.all(blocks.map(block => this.dal.saveTxsInFiles(block.transactions, block.number, block.medianTime)))
+    }
+
     // We only keep a bunch of days of blocks in memory, so memory consumption keeps approximately constant during the sync
     await this.dal.blockDAL.trimBlocks(blocks[blocks.length - 1].number - CommonConstants.BLOCKS_IN_MEMORY_MAX)
   }
-- 
GitLab