diff --git a/app/lib/common-libs/programOptions.ts b/app/lib/common-libs/programOptions.ts index 2c1949959de9a52e76d1bff3acf029847ef9f0f9..5605bcea7107cf571a389fcd037c6abf649509fe 100644 --- a/app/lib/common-libs/programOptions.ts +++ b/app/lib/common-libs/programOptions.ts @@ -27,6 +27,7 @@ export interface ProgramOptions { loglevel?: string sqlTraces?: boolean memory?: boolean + storeTxs?: boolean } export const cliprogram: ProgramOptions = { diff --git a/app/lib/dto/ConfDTO.ts b/app/lib/dto/ConfDTO.ts index ac1625ad11c3e4851b56d5b1f3e1684b0c574bb5..b17a17ee6ca1e8dd4ea2ed05b64637beb0f3e3b7 100644 --- a/app/lib/dto/ConfDTO.ts +++ b/app/lib/dto/ConfDTO.ts @@ -22,6 +22,12 @@ export interface Keypair { sec: string } +export interface StorageDTO { + storage?: { + transactions?:boolean + } +} + export interface PowDTO { powNoSecurity:boolean } @@ -103,7 +109,7 @@ export interface WS2PConfDTO { } } -export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO, BranchingDTO, WS2PConfDTO, PowDTO { +export class ConfDTO implements StorageDTO, CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO, BranchingDTO, WS2PConfDTO, PowDTO { constructor( public loglevel: string, @@ -182,7 +188,10 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO, maxPrivate?:number syncLimit?:number }, - public powNoSecurity = false + public powNoSecurity = false, + public storage = { + transactions: false + }, ) {} static mock() { diff --git a/app/modules/config.ts b/app/modules/config.ts index 0442cb20a68a968f896705e89cd4957b2de1fed7..5eee626cfdc28582cf8dd43e9e030251596b2c79 100644 --- a/app/modules/config.ts +++ b/app/modules/config.ts @@ -17,21 +17,43 @@ import {Server} from "../../server" import {CommonConstants} from "../lib/common-libs/constants" import {Directory} from "../lib/system/directory" import {Underscore} from "../lib/common-libs/underscore" +import {ProgramOptions} from "../lib/common-libs/programOptions" module.exports = { duniter: { + cliOptions: [ + { value: '--store-txs', desc: 'Enable full transaction history storage.' }, + ], + config: { - onLoading: async (conf:ConfDTO) => { + onLoading: async (conf:ConfDTO, program: ProgramOptions) => { conf.msPeriod = conf.msWindow conf.switchOnHeadAdvance = CommonConstants.SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS + + // Transactions storage + if (program.storeTxs) { + if (!conf.storage) { + conf.storage = { transactions: true } + } + else { + conf.storage.transactions = true + } + } }, beforeSave: async (conf:ConfDTO) => { conf.msPeriod = conf.msWindow conf.switchOnHeadAdvance = CommonConstants.SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS + if (!conf.storage) { + conf.storage = { + transactions: false + } + } } }, + + cli: [{ name: 'config', desc: 'Register configuration in database', diff --git a/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts b/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts index de32a3a975283d370d4e0aaa63c5e4f158a1666f..cb369b75a589526279e826911a1f1d8d993fa122 100644 --- a/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts +++ b/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts @@ -9,7 +9,7 @@ import { SimpleUdEntryForWallet, SindexEntry } from "../../../../../lib/indexer" -import {CurrencyConfDTO} from "../../../../../lib/dto/ConfDTO" +import {ConfDTO, CurrencyConfDTO} from "../../../../../lib/dto/ConfDTO" import {FileDAL} from "../../../../../lib/dal/fileDAL" import {DuniterBlockchain} from "../../../../../lib/blockchain/DuniterBlockchain" import {BlockDTO} from "../../../../../lib/dto/BlockDTO" @@ -80,7 +80,7 @@ export class GlobalIndexStream extends Duplex { private mapInjection: { [k: string]: any } = {} - constructor(private conf: any, + constructor(private conf: ConfDTO, private dal:FileDAL, private to: number, private localNumber:number, @@ -332,6 +332,10 @@ export class GlobalIndexStream extends Duplex { return block })) + if (this.conf.storage && this.conf.storage.transactions) { + await Promise.all(blocks.map(block => this.dal.saveTxsInFiles(block.transactions, block.number, block.medianTime))) + } + // We only keep a bunch of days of blocks in memory, so memory consumption keeps approximately constant during the sync await this.dal.blockDAL.trimBlocks(blocks[blocks.length - 1].number - CommonConstants.BLOCKS_IN_MEMORY_MAX) }