[enh] Add wotwizard export option

app/lib/dal/sqliteDAL/BlockDAL.ts

+// Source file from duniter: Crypto-currency software to manage libre currency such as Ğ1
+// Copyright (C) 2018  Cedric Moreau <cem.moreau@gmail.com>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+import {AbstractSQLite} from "./AbstractSQLite"
+import {SQLiteDriver} from "../drivers/SQLiteDriver"
+import {DBBlock} from "../../db/DBBlock"
+
+const constants = require('../../constants');
+
+const IS_FORK = true;
+const IS_NOT_FORK = false;
+
+export class BlockDAL extends AbstractSQLite<DBBlock> {
+
+  private current: DBBlock|null
+
+  constructor(driver:SQLiteDriver) {
+    super(
+      driver,
+      'block',
+      // PK fields
+      ['number','hash'],
+      // Fields
+      ['fork', 'hash', 'inner_hash', 'signature', 'currency', 'issuer', 'issuersCount', 'issuersFrame', 'issuersFrameVar', 'parameters', 'previousHash', 'previousIssuer', 'version', 'membersCount', 'monetaryMass', 'UDTime', 'medianTime', 'dividend', 'unitbase', 'time', 'powMin', 'number', 'nonce', 'transactions', 'certifications', 'identities', 'joiners', 'actives', 'leavers', 'revoked', 'excluded', 'len', 'legacy'],
+      // Arrays
+      ['identities','certifications','actives','revoked','excluded','leavers','joiners','transactions'],
+      // Booleans
+      ['wrong', 'legacy'],
+      // BigIntegers
+      ['monetaryMass'],
+      // Transient
+      []
+    )
+
+    /**
+     * Periodically cleans the current block cache.
+     * It seems the cache is not always correct and may stuck the node, so it is preferable to reset it periodically.
+     */
+    setInterval(this.cleanCache, constants.CURRENT_BLOCK_CACHE_DURATION);
+  }
+
+  async init() {
+    await this.exec('BEGIN;' +
+      'CREATE TABLE IF NOT EXISTS ' + this.table + ' (' +
+      'fork BOOLEAN NOT NULL,' +
+      'legacy BOOLEAN NOT NULL,' +
+      'hash VARCHAR(64) NOT NULL,' +
+      'inner_hash VARCHAR(64) NOT NULL,' +
+      'signature VARCHAR(100) NOT NULL,' +
+      'currency VARCHAR(50) NOT NULL,' +
+      'issuer VARCHAR(50) NOT NULL,' +
+      'issuersFrame INTEGER NULL,' +
+      'issuersFrameVar INTEGER NULL,' +
+      'issuersCount INTEGER NULL,' +
+      'len INTEGER NULL,' +
+      'parameters VARCHAR(255),' +
+      'previousHash VARCHAR(64),' +
+      'previousIssuer VARCHAR(50),' +
+      'version INTEGER NOT NULL,' +
+      'membersCount INTEGER NOT NULL,' +
+      'monetaryMass VARCHAR(100) DEFAULT \'0\',' +
+      'UDTime DATETIME,' +
+      'medianTime DATETIME NOT NULL,' +
+      'dividend INTEGER DEFAULT \'0\',' +
+      'unitbase INTEGER NULL,' +
+      'time DATETIME NOT NULL,' +
+      'powMin INTEGER NOT NULL,' +
+      'number INTEGER NOT NULL,' +
+      'nonce INTEGER NOT NULL,' +
+      'transactions TEXT,' +
+      'certifications TEXT,' +
+      'identities TEXT,' +
+      'joiners TEXT,' +
+      'actives TEXT,' +
+      'leavers TEXT,' +
+      'revoked TEXT,' +
+      'excluded TEXT,' +
+      'created DATETIME DEFAULT NULL,' +
+      'updated DATETIME DEFAULT NULL,' +
+      'PRIMARY KEY (number,hash)' +
+      ');' +
+      'CREATE INDEX IF NOT EXISTS idx_block_hash ON block (hash);' +
+      'CREATE INDEX IF NOT EXISTS idx_block_fork ON block (fork);' +
+      'COMMIT;')
+  }
+
+  cleanCache() {
+    this.current = null
+  }
+
+  async getCurrent() {
+    if (!this.current) {
+      this.current = (await this.query('SELECT * FROM block WHERE NOT fork ORDER BY number DESC LIMIT 1'))[0];
+    }
+    return this.current
+  }
+
+  async getBlock(number:string | number): Promise<DBBlock|null> {
+    return (await this.query('SELECT * FROM block WHERE number = ? and NOT fork', [parseInt(String(number))]))[0];
+  }
+
+  async getAbsoluteBlock(number:number, hash:string): Promise<DBBlock|null> {
+    return (await this.query('SELECT * FROM block WHERE number = ? and hash = ?', [number, hash]))[0];
+  }
+
+  getBlocks(start:number, end:number) {
+    return this.query('SELECT * FROM block WHERE number BETWEEN ? and ? and NOT fork ORDER BY number ASC', [start, end]);
+  }
+
+  async lastBlockOfIssuer(issuer:string) {
+    return (await this.query('SELECT * FROM block WHERE issuer = ? and NOT fork ORDER BY number DESC LIMIT 1', [issuer]))[0]
+  }
+
+  async getCountOfBlocksIssuedBy(issuer:string) {
+    let res: any = await this.query('SELECT COUNT(*) as quantity FROM block WHERE issuer = ? and NOT fork', [issuer]);
+    return res[0].quantity;
+  }
+
+  getPotentialForkBlocks(numberStart:number, medianTimeStart:number, maxNumber:number) {
+    return this.query('SELECT * FROM block WHERE fork AND number >= ? AND number <= ? AND medianTime >= ? ORDER BY number DESC', [numberStart, maxNumber, medianTimeStart]);
+  }
+
+  getPotentialRoots() {
+    return this.query('SELECT * FROM block WHERE fork AND number = ?', [0])
+  }
+
+  async saveBunch(blocks:DBBlock[]) {
+    let queries = "INSERT INTO block (" + this.fields.join(',') + ") VALUES ";
+    for (let i = 0, len = blocks.length; i < len; i++) {
+      let block = blocks[i];
+      queries += this.toInsertValues(block);
+      if (i + 1 < len) {
+        queries += ",\n";
+      }
+    }
+    await this.exec(queries);
+    this.cleanCache();
+  }
+
+  async saveBlock(block:DBBlock) {
+    let saved = await this.saveBlockAs(block, IS_NOT_FORK);
+    if (!this.current || this.current.number < block.number) {
+      this.current = block;
+    }
+    return saved;
+  }
+
+  saveSideBlock(block:DBBlock) {
+    return this.saveBlockAs(block, IS_FORK)
+  }
+
+  private async saveBlockAs(block:DBBlock, fork:boolean) {
+    block.fork = fork;
+    return await this.saveEntity(block);
+  }
+
+  async setSideBlock(number:number, previousBlock:DBBlock|null) {
+    await this.query('UPDATE block SET fork = ? WHERE number = ?', [true, number]);
+    this.current = previousBlock;
+  }
+
+  getNextForkBlocks(number:number, hash:string) {
+    return this.query('SELECT * FROM block WHERE fork AND number = ? AND previousHash like ? ORDER BY number', [number + 1, hash]);
+  }
+}

app/lib/dal/sqliteDAL/index/IIndexDAL.ts

+// Source file from duniter: Crypto-currency software to manage libre currency such as Ğ1
+// Copyright (C) 2018  Cedric Moreau <cem.moreau@gmail.com>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+import {SQLiteDriver} from "../../drivers/SQLiteDriver"
+import {FullIindexEntry, IindexEntry, Indexer} from "../../../indexer"
+import {AbstractSQLite} from "../AbstractSQLite"
+
+const _ = require('underscore');
+
+export interface OldIindexEntry extends IindexEntry {
+  pubkey: string
+  buid: string | null
+  revocation_sig:string | null
+}
+
+export class IIndexDAL extends AbstractSQLite<IindexEntry> {
+
+  constructor(driver:SQLiteDriver) {
+    super(
+      driver,
+      'i_index',
+      // PK fields
+      ['op', 'pub', 'created_on', 'written_on'],
+      // Fields
+      [
+        'op',
+        'uid',
+        'pub',
+        'hash',
+        'sig',
+        'created_on',
+        'written_on',
+        'writtenOn',
+        'member',
+        'wasMember',
+        'kick',
+        'wotb_id',
+        'legacy'
+      ],
+      // Arrays
+      [],
+      // Booleans
+      ['member', 'wasMember', 'kick', 'legacy'],
+      // BigIntegers
+      [],
+      // Transient
+      []
+    )
+  }
+
+  init() {
+    return this.exec('BEGIN;' +
+      'CREATE TABLE IF NOT EXISTS ' + this.table + ' (' +
+      'op VARCHAR(10) NOT NULL,' +
+      'uid VARCHAR(100) NULL,' +
+      'pub VARCHAR(50) NOT NULL,' +
+      'hash VARCHAR(80) NULL,' +
+      'sig VARCHAR(80) NULL,' +
+      'created_on VARCHAR(80) NULL,' +
+      'written_on VARCHAR(80) NOT NULL,' +
+      'writtenOn INTEGER NOT NULL,' +
+      'member BOOLEAN NULL,' +
+      'wasMember BOOLEAN NULL,' +
+      'kick BOOLEAN NULL,' +
+      'legacy BOOLEAN NOT NULL,' +
+      'wotb_id INTEGER NULL,' +
+      'PRIMARY KEY (op,pub,created_on,written_on)' +
+      ');' +
+      'CREATE INDEX IF NOT EXISTS idx_iindex_pub ON i_index (pub);' +
+      'COMMIT;')
+  }
+
+  async getMembers() {
+    // All those who has been subject to, or who are currently subject to kicking. Make one result per pubkey.
+    const pubkeys = await this.query('SELECT DISTINCT(pub) FROM ' + this.table);
+    // We get the full representation for each member
+    const reduced = await Promise.all(pubkeys.map(async (entry) => {
+      const reducable = await this.reducable(entry.pub);
+      return Indexer.DUP_HELPERS.reduce(reducable);
+    }));
+    // Filter on those to be kicked, return their pubkey
+    const filtered = _.filter(reduced, (entry:IindexEntry) => entry.member);
+    return filtered.map((t:IindexEntry) => this.toCorrectEntity(t))
+  }
+
+  getMembersPubkeys() {
+    return this.query('SELECT i1.pub ' +
+      'FROM i_index i1 ' +
+      'WHERE i1.member ' +
+      'AND CAST(i1.written_on as int) = (' +
+      ' SELECT MAX(CAST(i2.written_on as int)) ' +
+      ' FROM i_index i2 ' +
+      ' WHERE i1.pub = i2.pub ' +
+      ' AND i2.member IS NOT NULL' +
+      ')')
+  }
+
+  async getToBeKickedPubkeys() {
+    // All those who has been subject to, or who are currently subject to kicking. Make one result per pubkey.
+    const reducables = Indexer.DUP_HELPERS.reduceBy(await this.sqlFind({ kick: true }), ['pub']);
+    // We get the full representation for each member
+    const reduced = await Promise.all(reducables.map(async (entry) => {
+      const reducable = await this.reducable(entry.pub);
+      return Indexer.DUP_HELPERS.reduce(reducable);
+    }))
+    // Filter on those to be kicked, return their pubkey
+    return _.filter(reduced, (entry:IindexEntry) => entry.kick).map((entry:IindexEntry) => entry.pub);
+  }
+
+  async searchThoseMatching(search:string) {
+    const reducables = Indexer.DUP_HELPERS.reduceBy(await this.sqlFindLikeAny({
+      pub: "%" + search + "%",
+      uid: "%" + search + "%"
+    }), ['pub']);
+    // We get the full representation for each member
+    return await Promise.all(reducables.map(async (entry) => {
+      return this.toCorrectEntity(Indexer.DUP_HELPERS.reduce(await this.reducable(entry.pub)))
+    }))
+  }
+
+  getFromPubkey(pubkey:string) {
+    return this.entityOrNull('pub', pubkey) as Promise<FullIindexEntry|null>
+  }
+
+  getFromUID(uid:string, retrieveOnPubkey = false) {
+    return this.entityOrNull('uid', uid, retrieveOnPubkey)
+  }
+
+  getFullFromPubkey(pub:string): Promise<FullIindexEntry> {
+    return this.entityOrNull('pub', pub) as Promise<FullIindexEntry>
+  }
+
+  getFullFromUID(uid:string): Promise<FullIindexEntry|null> {
+    return this.entityOrNull('uid', uid, true) as Promise<FullIindexEntry|null>
+  }
+
+  getFullFromHash(hash:string): Promise<FullIindexEntry|null> {
+    return this.entityOrNull('hash', hash, true) as Promise<FullIindexEntry|null>
+  }
+
+  reducable(pub:string) {
+    return this.query('SELECT * FROM ' + this.table + ' WHERE pub = ? ORDER BY CAST(written_on as integer) ASC', [pub])
+  }
+
+  removeBlock(blockstamp:string) {
+    return this.exec('DELETE FROM ' + this.table + ' WHERE written_on = \'' + blockstamp + '\'')
+  }
+
+  private async entityOrNull(field:string, value:any, retrieveOnField:boolean = false) {
+    let reducable = await this.query('SELECT * FROM ' + this.table + ' WHERE ' + field + ' = ?', [value]);
+    if (reducable.length) {
+      if (retrieveOnField) {
+        // Force full retrieval on `pub` field
+        reducable = await this.query('SELECT * FROM ' + this.table + ' WHERE pub = ? ORDER BY CAST(written_on as int) ASC', [reducable[0].pub]);
+      }
+      return this.toCorrectEntity(Indexer.DUP_HELPERS.reduce(reducable));
+    }
+    return null;
+  }
+
+  private toCorrectEntity(row:IindexEntry): OldIindexEntry {
+    // Old field
+    return {
+      pubkey: row.pub,
+      pub: row.pub,
+      buid: row.created_on,
+      revocation_sig: null,
+      uid: row.uid,
+      hash: row.hash,
+      sig: row.sig,
+      created_on: row.created_on,
+      member: row.member,
+      wasMember: row.wasMember,
+      kick: row.kick,
+      wotb_id: row.wotb_id,
+      age: row.age,
+      index: row.index,
+      op: row.op,
+      writtenOn: row.writtenOn,
+      written_on: row.written_on
+    }
+  }
+
+  async getFromPubkeyOrUid(search: string) {
+    const idty = await this.getFromPubkey(search)
+    if (idty) {
+      return idty
+    }
+    return this.getFromUID(search, true) as Promise<FullIindexEntry|null>
+  }
+}

app/lib/dto/ConfDTO.ts

@@ -25,6 +25,7 @@ export interface Keypair {
 export interface StorageDTO {
   storage?: {
     transactions?:boolean
+    wotwizard?:boolean
   }
 }
 
@@ -192,7 +193,8 @@ export class ConfDTO implements StorageDTO, CurrencyConfDTO, KeypairConfDTO, Net
     },
     public powNoSecurity = false,
     public storage = {
-      transactions: false
+      transactions: false,
+      wotwizard: false,
     },
 ) {}
 

app/lib/system/directory.ts

@@ -118,6 +118,7 @@ export const MemFS = (initialTree:{ [folder:string]: { [file:string]: string }}
 export const Directory = {
 
   DATA_FILES: ['mindex.db', 'c_mindex.db', 'iindex.db', 'cindex.db', 'sindex.db', 'wallet.db', 'dividend.db', 'txs.db', 'peers.db'],
+  WW_FILES: ['wotwizard-export.db'],
   DATA_DIRS: ['data'],
 
   INSTANCE_NAME: getDomain(opts.mdb),

app/modules/dump.ts

@@ -22,6 +22,32 @@ const Table = require('cli-table')
 
 module.exports = {
   duniter: {
+
+    service: {
+      neutral: (server:Server, conf:ConfDTO) => {
+        return {
+          startService: () => {
+            if (conf.storage && conf.storage.wotwizard) {
+              let fifo: Querable<any> = querablep(Promise.resolve())
+              server
+                .on('bcEvent', (e) => {
+                  if ((e.bcEvent === OtherConstants.BC_EVENT.HEAD_CHANGED || e.bcEvent === OtherConstants.BC_EVENT.SWITCHED) && fifo.isFulfilled()) {
+                    fifo = querablep(fifo.then(async () => {
+                      try {
+                        await dumpWotWizard(server)
+                      } catch (e) {}
+                    }))
+                  }
+                })
+            }
+          },
+          stopService: () => {
+            // Never stops, just wait for blocks
+          }
+        }
+      }
+    },
+
     cli: [{
       name: 'dump [what] [name] [cond]',
       desc: 'Dumps data of the blockchain.',
@@ -32,6 +58,9 @@ module.exports = {
         const what: string = params[0] || ''
         const name: string = params[1] || ''
         const cond: string = params[2] || ''
+
+        try {
+
           switch (what) {
 
             case 'current':
@@ -50,10 +79,17 @@ module.exports = {
               await dumpHistory(server, name)
               break
 
+            case 'wotwizard':
+              await dumpWotWizard(server)
+              break
+
             default:
               console.error(`Unknown dump ${what}`)
               break
           }
+        } catch (e) {
+          console.error(e)
+        }
         // Save DB
         await server.disconnect();
       }

app/modules/dump/wotwizard/wotwizard.constants.ts

+export const WotWizardConstants = {
+
+  DB_NAME: 'wotwizard-export.db',
+  BLOCKS_SAVE_BATCH_SIZE: 10,
+}
\ No newline at end of file

app/modules/dump/wotwizard/wotwizard.copy.mempool.ts

+import {WotWizardDAL} from "./wotwizard.init.structure"
+import {Server} from "../../../../server"
+import {DBBlock} from "../../../lib/db/DBBlock"
+import {Underscore} from "../../../lib/common-libs/underscore"
+import {NewLogger} from "../../../lib/logger"
+
+export async function copyMemPool(server: Server, wwDAL: WotWizardDAL) {
+
+  const logger = NewLogger()
+
+  const identities = await server.dal.idtyDAL.sqlListAll()
+
+  // Blocks on which are based identities
+  const blocks = await Promise.all(identities.map(async idty => {
+    let b = await server.dal.getAbsoluteBlockByBlockstamp(idty.buid)
+    if (b) {
+      const b2 = await wwDAL.blockDao.getAbsoluteBlock(b.number, b.hash)
+      if (!b2) {
+        return b
+      }
+    }
+    return null
+  }))
+
+
+  const toPersist: DBBlock[] = Underscore.uniq(blocks.filter(b => b) as DBBlock[], false, b => [b.number, b.hash].join('-'))
+
+  logger.debug('Persisting %s blocks for identities...', toPersist.length)
+  await wwDAL.blockDao.insertBatch(toPersist.map(b => { (b as any).legacy = true; return b }))
+  await wwDAL.idtyDao.insertBatch(identities)
+  await wwDAL.certDao.insertBatch(await server.dal.certDAL.sqlListAll())
+  await wwDAL.msDao.insertBatch(await server.dal.msDAL.sqlListAll())
+}
\ No newline at end of file

app/modules/dump/wotwizard/wotwizard.delete.ts

+import {WotWizardDAL} from "./wotwizard.init.structure"
+
+export async function deleteNonLegacy(wwDAL: WotWizardDAL) {
+
+  await wwDAL.iindexDao.exec('DELETE FROM i_index WHERE NOT legacy')
+  await wwDAL.blockDao.exec('DELETE FROM block WHERE NOT legacy')
+  await wwDAL.idtyDao.sqlDeleteAll()
+  await wwDAL.certDao.sqlDeleteAll()
+  await wwDAL.msDao.sqlDeleteAll()
+}
\ No newline at end of file

app/modules/dump/wotwizard/wotwizard.dump.ts

+import {Server} from "../../../../server"
+import {createExportStructure} from "./wotwizard.init.structure"
+import {WotWizardConstants} from "./wotwizard.constants"
+import {addLegacyBlocks} from "./wotwizard.legacy.blocks"
+import {addNewBlocks} from "./wotwizard.new.blocks"
+import {deleteNonLegacy} from "./wotwizard.delete"
+import {copyMemPool} from "./wotwizard.copy.mempool"
+
+export async function dumpWotWizard(server: Server) {
+
+  // 1. Create dump structure if it does not exist
+  const wwDAL = await createExportStructure(WotWizardConstants.DB_NAME)
+
+  // 2. Integrate legacy blocks (= non-forkable)
+  await addLegacyBlocks(server, wwDAL)
+
+  // 3. Delete non-legacy data
+  await deleteNonLegacy(wwDAL)
+
+  // 4. Integrate new blocks (= forkable)
+  await addNewBlocks(server, wwDAL)
+
+  // 5. Copy mempool
+  await copyMemPool(server, wwDAL)
+}

app/modules/dump/wotwizard/wotwizard.init.structure.ts

+import {Directory} from "../../../lib/system/directory"
+import {IdentityDAL} from "../../../lib/dal/sqliteDAL/IdentityDAL"
+import {MembershipDAL} from "../../../lib/dal/sqliteDAL/MembershipDAL"
+import {CertDAL} from "../../../lib/dal/sqliteDAL/CertDAL"
+import {BlockDAL} from "../../../lib/dal/sqliteDAL/BlockDAL"
+import {IIndexDAL} from "../../../lib/dal/sqliteDAL/index/IIndexDAL"
+
+export interface WotWizardDAL {
+  idtyDao: IdentityDAL
+  certDao: CertDAL
+  msDao: MembershipDAL
+  blockDao: BlockDAL
+  iindexDao: IIndexDAL
+}
+
+export async function createExportStructure(dbName: string): Promise<WotWizardDAL> {
+  const driver = await Directory.getHomeDB(false, dbName)
+
+  // DAOs
+  const idtyDao = new IdentityDAL(driver)
+  const certDao = new CertDAL(driver)
+  const msDao = new MembershipDAL(driver)
+  const blockDao = new BlockDAL(driver)
+  const iindexDao = new IIndexDAL(driver)
+
+  // Create tables
+  await idtyDao.init()
+  await certDao.init()
+  await msDao.init()
+  await blockDao.init()
+  await iindexDao.init()
+
+  const data = await blockDao.query('SELECT COUNT(*) as count FROM block')
+  const blocksCount = parseInt(String((data[0] as any).count))
+
+  // If first DB initialization
+  if (blocksCount === 0) {
+    // Manual updates (which are normally present in MetaDAL)
+    await idtyDao.exec('ALTER TABLE idty ADD COLUMN expired INTEGER NULL')
+    await idtyDao.exec('ALTER TABLE idty ADD COLUMN revoked_on INTEGER NULL')
+    await idtyDao.exec('ALTER TABLE idty ADD COLUMN removed BOOLEAN NULL DEFAULT 0')
+    await certDao.exec('ALTER TABLE cert ADD COLUMN expired INTEGER NULL')
+    await msDao.exec('ALTER TABLE membership ADD COLUMN expired INTEGER NULL')
+  }
+
+  return {
+    idtyDao,
+    certDao,
+    msDao,
+    blockDao,
+    iindexDao,
+  }
+}

app/modules/dump/wotwizard/wotwizard.legacy.blocks.ts

+import {WotWizardDAL} from "./wotwizard.init.structure"
+import {Server} from "../../../../server"
+import {CommonConstants} from "../../../lib/common-libs/constants"
+import {DBBlock} from "../../../lib/db/DBBlock"
+import {NewLogger} from "../../../lib/logger"
+
+export async function addLegacyBlocks(server: Server, wwDAL: WotWizardDAL) {
+
+  const logger = NewLogger()
+
+  const currentWW = await wwDAL.blockDao.getCurrent()
+  const current = await server.dal.blockDAL.getCurrent()
+  const start = currentWW && currentWW.number + 1 || 0
+  const end = current && Math.max(-1, current.number - 100) || -1
+
+  const blocksSaved: DBBlock[] = []
+  logger.debug('Reading blocks...')
+
+  // We loop taking care of archives structure
+  for (let i = start; i <= end; i += CommonConstants.ARCHIVES_BLOCKS_CHUNK) {
+    const blocks = await server.dal.getBlocksBetween(i, Math.min(end, i + CommonConstants.ARCHIVES_BLOCKS_CHUNK) - 1)
+    const filtered = blocks.filter(b => b.joiners.length
+      || b.actives.length
+      || b.leavers.length
+      || b.revoked.length
+      || b.excluded.length
+      || b.certifications.length
+    )
+    if (filtered.length) {
+      const legacies = filtered.map(f => { (f as any).legacy = true; return f })
+      legacies.forEach(l => blocksSaved.push(l))
+      // blocksSaved.push(wwDAL.blockDao.insertBatch(legacies))
+    }
+  }
+
+  logger.debug('Saving blocks...')
+  await wwDAL.blockDao.insertBatch(blocksSaved)
+
+  await Promise.all(blocksSaved)
+
+  const iindexRows = (await server.dal.iindexDAL.findRawWithOrder({}, [['writtenOn', false], ['wotb_id', false]]))
+    .filter(r => r.hash && r.writtenOn >= start && r.writtenOn <= end)
+
+  logger.debug('Saving %s iindex rows...', iindexRows.length)
+  const legacies = iindexRows.map(f => { (f as any).legacy = true; return f })
+  await wwDAL.iindexDao.insertBatch(legacies)
+}
\ No newline at end of file

app/modules/dump/wotwizard/wotwizard.new.blocks.ts

+import {WotWizardDAL} from "./wotwizard.init.structure"
+import {Server} from "../../../../server"
+import {CommonConstants} from "../../../lib/common-libs/constants"
+import {NewLogger} from "../../../lib/logger"
+
+export async function addNewBlocks(server: Server, wwDAL: WotWizardDAL) {
+
+  const logger = NewLogger()
+
+  wwDAL.blockDao.cleanCache()
+
+  const currentWW = await wwDAL.blockDao.getCurrent()
+  const current = await server.dal.blockDAL.getCurrent()
+  const start = currentWW && currentWW.number + 1 || 0
+  const end = current && current.number || -1
+
+  const blocksSaved: Promise<any>[] = []
+
+  // We loop taking care of archives structure
+  for (let i = start; i <= end; i += CommonConstants.ARCHIVES_BLOCKS_CHUNK) {
+    const beginAt = i
+    const endAt = Math.min(end, i + CommonConstants.ARCHIVES_BLOCKS_CHUNK) - 1
+    const blocks = await server.dal.getBlocksBetween(beginAt, endAt)
+    const forks = await server.dal.getPotentialForkBlocks(beginAt, 0, endAt)
+    const all =  blocks.concat(forks).map(f => { (f as any).legacy = false; return f })
+    logger.debug('Saving %s pending blocks...', all.length)
+    blocksSaved.push(wwDAL.blockDao.insertBatch(all))
+  }
+
+  await Promise.all(blocksSaved)
+
+  const iindexRows = (await server.dal.iindexDAL.findRawWithOrder({}, [['writtenOn', false], ['wotb_id', false]]))
+    .filter(r => r.writtenOn >= start)
+
+  logger.debug('Saving %s iindex rows...', iindexRows.length)
+  const legacies = iindexRows.map(f => { (f as any).legacy = false; return f })
+  await wwDAL.iindexDao.insertBatch(legacies)
+}
\ No newline at end of file

server.ts

@@ -393,6 +393,7 @@ export class Server extends stream.Duplex implements HookableServer {
     await this.resetConfigHook()
     const files = ['stats', 'cores', 'current', Directory.DUNITER_DB_NAME, Directory.DUNITER_DB_NAME + '.db', Directory.DUNITER_DB_NAME + '.log', Directory.WOTB_FILE, 'export.zip', 'import.zip', 'conf']
       .concat(Directory.DATA_FILES)
+      .concat(Directory.WW_FILES)
     const dirs  = ['archives', 'loki', 'blocks', 'blockchain', 'ud_history', 'branches', 'certs', 'txs', 'cores', 'sources', 'links', 'ms', 'identities', 'peers', 'indicators', 'leveldb']
       .concat(Directory.DATA_DIRS)
     return this.resetFiles(files, dirs, done);
@@ -402,6 +403,7 @@ export class Server extends stream.Duplex implements HookableServer {
     await this.resetDataHook()
     const files = ['stats', 'cores', 'current', Directory.DUNITER_DB_NAME, Directory.DUNITER_DB_NAME + '.db', Directory.DUNITER_DB_NAME + '.log', Directory.WOTB_FILE]
       .concat(Directory.DATA_FILES)
+      .concat(Directory.WW_FILES)
     const dirs  = ['archives', 'loki', 'blocks', 'ud_history', 'branches', 'certs', 'txs', 'cores', 'sources', 'links', 'ms', 'identities', 'peers', 'indicators', 'leveldb']
       .concat(Directory.DATA_DIRS)
     await this.resetFiles(files, dirs, done);
@@ -453,15 +455,6 @@ export class Server extends stream.Duplex implements HookableServer {
     })
   }
 
-  async cleanDBData() {
-    await this.dal.cleanCaches();
-    this.dal.wotb.resetWoT();
-    const files = ['stats', 'cores', 'current', Directory.DUNITER_DB_NAME, Directory.DUNITER_DB_NAME + '.db', Directory.DUNITER_DB_NAME + '.log'];
-    const dirs  = ['loki', 'blocks', 'ud_history', 'branches', 'certs', 'txs', 'cores', 'sources', 'links', 'ms', 'identities', 'peers', 'indicators', 'leveldb']
-      .concat(Directory.DATA_DIRS)
-    return this.resetFiles(files, dirs);
-  }
-
   private async resetFiles(files:string[], dirs:string[], done:any = null) {
     try {
       const params = await this.paramsP;