Commit 3dc2d3a9 authored by Cédric Moreau's avatar Cédric Moreau
Browse files

Merge branch 'fix/1396-revert-removes-certifications' into 1.7

parents 46ec4572 e4988bbd
......@@ -239,13 +239,7 @@ export class DuniterBlockchain {
await this.updateWallets(indexes.sindex, indexes.dividends, dal)
if (trim) {
const TAIL = await dal.bindexDAL.tail();
const MAX_BINDEX_SIZE = requiredBindexSizeForTail(TAIL, conf)
const currentSize = indexes.HEAD.number - TAIL.number + 1
if (currentSize > MAX_BINDEX_SIZE) {
await dal.archiveBlocks()
await dal.trimIndexes(indexes.HEAD.number - MAX_BINDEX_SIZE);
}
await DuniterBlockchain.trimIndexes(dal, indexes.HEAD, conf)
}
const dbb = DBBlock.fromBlockDTO(block)
......@@ -573,6 +567,15 @@ export class DuniterBlockchain {
throw err;
}
}
public static async trimIndexes(dal: FileDAL, HEAD: { number: number }, conf: ConfDTO) {
const TAIL = await dal.bindexDAL.tail();
const MAX_BINDEX_SIZE = requiredBindexSizeForTail(TAIL, conf)
const currentSize = HEAD.number - TAIL.number + 1
if (currentSize > MAX_BINDEX_SIZE) {
await dal.trimIndexes(HEAD.number - MAX_BINDEX_SIZE);
}
}
}
export function requiredBindexSizeForTail(TAIL: { issuersCount: number, issuersFrame: number }, conf: { medianTimeBlocks: number, dtDiffEval: number, forksize: number }) {
......
import {MonitorExecutionTime} from "../../../debug/MonitorExecutionTime"
import {CindexEntry, FullCindexEntry, Indexer, reduceBy} from "../../../indexer"
import {CindexEntry, FullCindexEntry, Indexer, reduce, reduceBy} from "../../../indexer"
import {LevelUp} from 'levelup'
import {LevelDBTable} from "./LevelDBTable"
import {Underscore} from "../../../common-libs/underscore"
......@@ -146,13 +146,20 @@ export class LevelDBCindex extends LevelDBTable<LevelDBCindexEntry> implements C
}
}
// Remove the "received" arrays
await Promise.all(toRemove.map(async e => {
const entry = await this.get(e.receiver)
// Remove the certification
entry.received = entry.received.filter(issuer => issuer !== e.issuer)
// Persist
await this.put(e.receiver, entry)
}))
for (const e of toRemove) {
const receiver = await this.get(e.receiver)
const issuer = await this.get(e.issuer)
const certification = reduce(issuer.issued.filter(i => i.receiver === e.receiver))
// We remove ONLY IF no valid link still exist, i.e. we remove if the link **has expired** (we may be here because
// of a certification replay before term that is being reverted ==> in such case, even after the revert, the link
// between issuer and receiver is still valid. So don't remove it.
if (certification.expired_on) {
// Remove the certification
receiver.received = receiver.received.filter(issuer => issuer !== e.issuer)
// Persist
await this.put(e.receiver, receiver)
}
}
// Remove the expires_on index entries
const expires = Underscore.uniq(toRemove.filter(e => e.expires_on).map(e => e.expires_on))
await Promise.all(expires.map(async e => this.indexForExpiresOn.del(LevelDBCindex.trimExpiredOnKey(e))))
......
import {CindexEntry} from "../indexer"
const Table = require('cli-table')
export function dumpBindex(rows: CindexEntry[]) {
return dump(rows, ['version','bsize','hash','issuer','time','number','membersCount','issuersCount','issuersFrame','issuersFrameVar','issuerDiff','avgBlockSize','medianTime','dividend','mass','unitBase','powMin','udTime','udReevalTime','diffNumber','speed','massReeval'])
}
export function dumpIindex(rows: CindexEntry[]) {
return dump(rows, ['op','uid','pub','hash','sig','created_on','written_on','member','wasMember','kick','wotb_id'])
}
export function dumpCindex(rows: CindexEntry[]) {
return dump(rows, ['op','issuer','receiver','created_on','written_on','sig','expires_on','expired_on','chainable_on','from_wid','to_wid','replayable_on'])
}
export function dumpCindexPretty(rows: CindexEntry[], getUid: (pub: string) => Promise<string>) {
return dump(rows, ['row','op','issuer','created_on','written_on','expires_on','expired_on','chainable_on','replayable_on'], async (f, v) => {
if (f === 'issuer') {
return await getUid(v)
}
if (f === 'written_on') {
return String(v).substr(0, 15)
}
return v
})
}
export function dumpMindex(rows: CindexEntry[]) {
return dump(rows, ['op','pub','created_on','written_on','expires_on','expired_on','revokes_on','revoked_on','leaving','revocation','chainable_on'])
}
export function dumpSindex(rows: CindexEntry[]) {
return dump(rows, ['op','tx','identifier','pos','created_on','amount','base','locktime','consumed','conditions', 'writtenOn'])
}
async function dump(rows: any[], columns: string[], transform: (field: string, value: any) => Promise<string> = (f, v) => Promise.resolve(v)) {
// Table columns
const t = new Table({
head: columns, chars: {'mid': '', 'left-mid': '', 'mid-mid': '', 'right-mid': ''}
});
let i = 0;
for (const row of rows) {
t.push(await Promise.all(columns.map(async (c) => {
if (c === 'row') {
return i
}
else if (row[c] === null) {
return "NULL"
}
else if (row[c] === undefined) {
return 'NULL'
}
else if (typeof row[c] === 'boolean') {
const v = await transform(c, row[c] ? 1 : 0)
return v
}
const v = await transform(c, row[c])
return v
})));
i++
}
try {
const dumped = t.toString()
console.log(dumped)
} catch (e) {
console.error(e)
}
}
......@@ -11,11 +11,12 @@
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
import {exec} from "child_process"
import {ConfDTO} from "../lib/dto/ConfDTO"
import {Server} from "../../server"
import {moment} from "../lib/common-libs/moment"
import {DBBlock} from "../lib/db/DBBlock"
import {SindexEntry} from "../lib/indexer"
import {FullIindexEntry, IindexEntry, SindexEntry} from "../lib/indexer"
import {BlockDTO} from "../lib/dto/BlockDTO"
import {Underscore} from "../lib/common-libs/underscore"
import {dumpWotWizard} from "./dump/wotwizard/wotwizard.dump"
......@@ -23,6 +24,13 @@ import {OtherConstants} from "../lib/other_constants"
import {Querable, querablep} from "../lib/common-libs/querable"
import {dumpBlocks, dumpForks} from "./dump/blocks/dump.blocks"
import {newResolveTimeoutPromise} from "../lib/common-libs/timeout-promise"
import {LevelDBIindex} from "../lib/dal/indexDAL/leveldb/LevelDBIindex"
import {dumpBindex, dumpCindex, dumpCindexPretty, dumpIindex, dumpMindex, dumpSindex} from "../lib/debug/dump"
import {readFileSync} from "fs"
import {IdentityDTO} from "../lib/dto/IdentityDTO"
import {CertificationDTO, ShortCertificationDTO} from "../lib/dto/CertificationDTO"
import {MembershipDTO} from "../lib/dto/MembershipDTO"
import {RevocationDTO, ShortRevocation} from "../lib/dto/RevocationDTO"
const Table = require('cli-table')
......@@ -55,16 +63,46 @@ module.exports = {
},
cli: [{
name: 'current',
desc: 'Shows current block\'s blockstamp',
logs: false,
preventIfRunning: true,
onDatabaseExecute: async (server:Server) => {
const current = await server.dal.getCurrentBlockOrNull()
if (!current) {
return console.log('None')
}
const blockstamp = `${current.number}-${current.hash}`
console.log(blockstamp)
// Save DB
await server.disconnect();
}
}, {
name: 'trim-indexes',
desc: 'Force trimming of indexes',
logs: true,
preventIfRunning: true,
onConfiguredExecute: async (server:Server) => {
await server.dal.init(server.conf)
await server.BlockchainService.trimIndexes()
// Save DB
await server.disconnect();
}
}, {
name: 'dump [what] [name] [cond]',
desc: 'Dumps data of the blockchain.',
logs: false,
preventIfRunning: true,
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
onConfiguredExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
const what: string = params[0] || ''
const name: string = params[1] || ''
const cond: string = params[2] || ''
await server.dal.init(server.conf)
try {
switch (what) {
......@@ -111,6 +149,64 @@ module.exports = {
// Save DB
await server.disconnect();
}
}, {
name: 'search [pattern]',
desc: 'Dumps data of the blockchain matching given pattern.',
logs: false,
preventIfRunning: true,
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
const pattern: string = params[0] || ''
try {
const files: string[] = await new Promise<string[]>((res, rej) => exec(`grep -r ${pattern} ${server.home}/${server.conf.currency} -l | grep .json`, (err, stdout) => {
if (err) return rej(err)
console.log(stdout)
res(stdout.split('\n').filter(l => l))
}))
const blocks = Underscore.sortBy(await findBlocksMatching(pattern, files), b => b.number)
const events: { b: BlockDTO, event: (IdentityDTO|ShortCertificationDTO|MembershipDTO|ShortRevocation|{ type: 'exclusion', pub: string }) }[] = []
for (const b of blocks) {
b.identities.filter(i => i.includes(pattern)).forEach(i => {
events.push({ b, event: IdentityDTO.fromInline(i) })
})
b.certifications.filter(c => c.includes(pattern)).forEach(c => {
events.push({ b, event: CertificationDTO.fromInline(c) })
})
b.joiners.concat(b.actives).concat(b.leavers).filter(m => m.includes(pattern)).forEach(m => {
events.push({ b, event: MembershipDTO.fromInline(m) })
})
b.revoked.filter(m => m.includes(pattern)).forEach(r => {
events.push({ b, event: RevocationDTO.fromInline(r) })
})
b.excluded.filter(m => m.includes(pattern)).forEach(r => {
events.push({ b, event: { type: 'exclusion', pub: r } })
})
}
for (const e of events) {
if ((e.event as IdentityDTO).uid) {
const date = await getDateForBlock(e.b)
const idty = e.event as IdentityDTO
console.log('%s: new identity %s (created on %s)', date, idty.uid, await getDateFor(server, idty.buid as string))
}
if ((e.event as { type: 'exclusion', pub: string }).type === 'exclusion') {
const date = await getDateForBlock(e.b)
console.log('%s: excluded', date)
}
}
console.log(events.map(e => e.event))
} catch (e) {
console.error(e)
}
// Save DB
await server.disconnect();
}
}, {
name: 'dump-ww',
desc: 'Dumps WotWizard export.',
......@@ -121,6 +217,21 @@ module.exports = {
}
}
async function findBlocksMatching(pattern: string, files: string[]) {
const matchingBlocks: BlockDTO[] = []
for (const f of files) {
const blocks: any[] = JSON.parse(await readFileSync(f, 'utf8')).blocks
for (const jsonBlock of blocks) {
const b = BlockDTO.fromJSONObject(jsonBlock)
const raw = b.getRawSigned()
if (raw.includes(pattern)) {
matchingBlocks.push(b)
}
}
}
return matchingBlocks
}
async function dumpCurrent(server: Server) {
const current = await server.dal.getCurrentBlockOrNull()
if (!current) {
......@@ -164,8 +275,7 @@ async function dumpTable(server: Server, name: string, condition?: string) {
switch (name) {
case 'b_index':
rows = await server.dal.bindexDAL.findRawWithOrder(criterion, [['number', false]])
dump(rows, ['version','bsize','hash','issuer','time','number','membersCount','issuersCount','issuersFrame','issuersFrameVar','issuerDiff','avgBlockSize','medianTime','dividend','mass','unitBase','powMin','udTime','udReevalTime','diffNumber','speed','massReeval'])
break
return dumpBindex(rows)
/**
* Dumps issuers visible in current bindex
......@@ -178,59 +288,39 @@ async function dumpTable(server: Server, name: string, condition?: string) {
case 'i_index':
rows = await server.dal.iindexDAL.findRawWithOrder(criterion, [['writtenOn', false], ['wotb_id', false]])
dump(rows, ['op','uid','pub','hash','sig','created_on','written_on','member','wasMember','kick','wotb_id'])
break
return dumpIindex(rows)
case 'm_index':
rows = await server.dal.mindexDAL.findRawWithOrder(criterion, [['writtenOn', false], ['pub', false]])
dump(rows, ['op','pub','created_on','written_on','expires_on','expired_on','revokes_on','revoked_on','leaving','revocation','chainable_on'])
break
return dumpMindex(rows)
case 'c_index':
rows = await server.dal.cindexDAL.findRawWithOrder(criterion, [['writtenOn', false], ['issuer', false], ['receiver', false]])
dump(rows, ['op','issuer','receiver','created_on','written_on','sig','expires_on','expired_on','chainable_on','from_wid','to_wid','replayable_on'])
return dumpCindex(rows)
break
case 's_index':
const rowsTX = await server.dal.sindexDAL.findRawWithOrder(criterion, [['writtenOn', false], ['identifier', false], ['pos', false]])
const rowsUD = await server.dal.dividendDAL.findForDump(criterion)
rows = rowsTX.concat(rowsUD)
sortSindex(rows)
dump(rows, ['op','tx','identifier','pos','created_on','amount','base','locktime','consumed','conditions', 'writtenOn'])
break
return dumpSindex(rows)
case 'c_index_pretty':
rows = await server.dal.cindexDAL.findRawWithOrder(criterion, [['writtenOn', false], ['issuer', false], ['receiver', false]])
rows = rows.filter((row: any) => Object.entries(criterion).reduce((ok, crit: any) => ok && row[crit[0]] === crit[1], true))
await dumpCindexPretty(rows, async (pub) => {
const iindexEntry = await server.dal.getWrittenIdtyByPubkey(pub)
return (iindexEntry as IindexEntry).uid as string
})
default:
console.error(`Unknown dump table ${name}`)
break
}
}
function dump(rows: any[], columns: string[]) {
// Table columns
const t = new Table({
head: columns
});
for (const row of rows) {
t.push(columns.map((c) => {
if (row[c] === null) {
return "NULL"
}
else if (row[c] === undefined) {
return 'NULL'
}
else if (typeof row[c] === 'boolean') {
return row[c] ? 1 : 0
}
return row[c]
}));
}
try {
const dumped = t.toString()
console.log(dumped)
} catch (e) {
console.error(e)
}
}
async function dumpHistory(server: Server, pub: string) {
const irows = await server.dal.iindexDAL.findRawWithOrder({ pub }, [['writtenOn', false]])
const mrows = await server.dal.mindexDAL.findRawWithOrder({ pub }, [['writtenOn', false]])
const irows = (await server.dal.iindexDAL.findRawWithOrder({ pub }, [['writtenOn', false]])).filter(r => pub ? r.pub === pub : true)
const mrows = (await server.dal.mindexDAL.findRawWithOrder({ pub }, [['writtenOn', false]])).filter(r => pub ? r.pub === pub : true)
const crows = (await server.dal.cindexDAL.findRawWithOrder({ pub }, [['writtenOn', false]])).filter(r => pub ? r.issuer === pub || r.receiver === pub: true)
console.log('----- IDENTITY -----')
for (const e of irows) {
const date = await getDateFor(server, e.written_on)
......@@ -259,6 +349,28 @@ async function dumpHistory(server: Server, pub: string) {
console.log('Non displayable MINDEX entry')
}
}
console.log('----- CERTIFICATION -----')
crows.forEach(crow => {
console.log(JSON.stringify(crow))
})
for (const e of crows) {
const dateW = await getDateFor(server, e.written_on)
const dateC = await getDateForBlockNumber(server, e.created_on)
if (e.receiver === pub) {
const issuer = await server.dal.getWrittenIdtyByPubkey(e.issuer) as FullIindexEntry
if (e.op === 'UPDATE') {
console.log('%s : %s: from %s (update)', dateC, dateW, issuer.uid)
}
else {
console.log('%s : %s: from %s', dateC, dateW, issuer.uid)
}
// } else if (e.issuer === pub) {
// const receiver = await server.dal.getWrittenIdtyByPubkey(e.receiver) as FullIindexEntry
// console.log('%s: to ', date, receiver.uid)
} else {
// console.log('Non displayable CINDEX entry')
}
}
}
async function dumpWot(server: Server) {
......@@ -274,6 +386,19 @@ async function getDateFor(server: Server, blockstamp: string) {
return formatTimestamp(b.medianTime) + ' (#' + bnumberPadded + ')'
}
async function getDateForBlockNumber(server: Server, number: number) {
const b = (await server.dal.getBlock(number)) as DBBlock
const s = " " + b.number
const bnumberPadded = s.substr(s.length - 6)
return formatTimestamp(b.medianTime) + ' (#' + bnumberPadded + ')'
}
async function getDateForBlock(b: BlockDTO) {
const s = " " + b.number
const bnumberPadded = s.substr(s.length - 6)
return formatTimestamp(b.medianTime) + ' (#' + bnumberPadded + ')'
}
function formatTimestamp(ts: number) {
return moment(ts * 1000).format('YYYY-MM-DD hh:mm:ss')
}
......
......@@ -460,7 +460,7 @@ export class BlockGenerator {
exclusions:any,
wereExcluded:any,
transactions:any,
manualValues:any) {
manualValues:ForcedBlockValues) {
if (manualValues && manualValues.excluded) {
exclusions = manualValues.excluded;
......@@ -630,6 +630,16 @@ export class BlockGenerator {
}
}
// Forced joiners (by tests)
if (manualValues && manualValues.joiners) {
block.joiners = block.joiners.concat(manualValues.joiners.map(j => j.inline()))
}
// Forced certifications (by tests)
if (manualValues && manualValues.certifications) {
block.certifications = block.certifications.concat(manualValues.certifications.map(c => c.inline()))
}
// Final number of members
block.membersCount = previousCount + block.joiners.length - block.excluded.length;
......@@ -679,7 +689,7 @@ export class BlockGenerator {
block.issuersFrameVar = vHEAD.issuersFrameVar;
// Manual values before hashing
if (manualValues) {
Underscore.extend(block, Underscore.omit(manualValues, 'time'));
Underscore.extend(block, Underscore.omit(manualValues, 'time', 'certifications', 'joiners'));
}
// InnerHash
block.time = block.medianTime;
......@@ -847,3 +857,13 @@ class ManualRootGenerator implements BlockGeneratorInterface {
}
}
}
export interface ForcedBlockValues {
time?: number
version?: number
medianTime?: number
excluded?: string[]
revoked?: string[]
certifications?: CertificationDTO[]
joiners?: MembershipDTO[]
}
\ No newline at end of file
......@@ -459,4 +459,10 @@ export class BlockchainService extends FIFOService {
return this.dal.getBlocksBetween(from, from + count - 1);
}
async trimIndexes() {
const HEAD = await this.dal.getCurrentBlockOrNull()
if (HEAD) {
return DuniterBlockchain.trimIndexes(this.dal, HEAD, this.conf)
}
}
}
// Source file from duniter: Crypto-currency software to manage libre currency such as Ğ1
// Copyright (C) 2018 Cedric Moreau <cem.moreau@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
import {assertEqual, writeBasicTestWithConfAnd2Users} from "../tools/test-framework"
import {CommonConstants} from "../../../app/lib/common-libs/constants"
const currentVersion = CommonConstants.BLOCK_GENERATED_VERSION
describe('Block revert with a comebacker in it', () => writeBasicTestWithConfAnd2Users({
sigQty: 2,
sigReplay: 0,
sigPeriod: 0,
sigValidity: 10,
dtDiffEval: 1,
forksize: 0,
}, (test) => {
const now = 1500000000
test('(t = 0) should init with a 4 members WoT with bidirectionnal certs', async (s1, cat, tac, toc) => {
CommonConstants.BLOCK_GENERATED_VERSION = 11
await cat.createIdentity()
await tac.createIdentity()
await toc.createIdentity()
await cat.cert(tac)
await cat.cert(toc)
await tac.cert(cat)
await tac.cert(toc)
await toc.cert(cat)
await toc.cert(tac)
await cat.join()
await tac.join()
await toc.join()
const b0 = await s1.commit({ time: now })
assertEqual(b0.certifications.length, 6)
const b1 = await s1.commit({ time: now })
assertEqual(b1.membersCount, 3)
})
test('(t = 5) cat & tac certify each other', async (s1, cat, tac, toc) => {
await s1.commit({ time: now + 5 })
await s1.commit({ time: now + 5 })
await new Promise(resolve => setTimeout(resolve, 500))
// cat and tac certify each other to stay in the WoT
await tac.cert(cat)
await toc.cert(cat) // <-- toc adds the 2nd certification
const b1 = await s1.commit({ time: now + 6 })
assertEqual(b1.certifications.length, 2)
await cat.cert(tac)
await toc.cert(tac) // <-- toc adds the 2nd certification
const b2 = await s1.commit({ time: now + 6 })
assertEqual(b2.certifications.length, 2)
// // /!\/!\/!\
// // toc gets certified by cat, to a have a remaining valid certification in the blockchain: THIS WONT BE ENOUGH!
await cat.cert(toc)
// // /!\/!\/!\
const b4 = await s1.commit({ time: now + 6 })
assertEqual(b4.certifications.length, 1)
})
test('(t = 12) toc is excluded for lack of certifications', async (s1, cat, tac, toc) => {
await s1.commit({ time: now + 12 })
await s1.commit({ time: now + 12 })
const b = await s1.commit({ time: now + 12 })
assertEqual(b.excluded.length, 1)
assertEqual(b.excluded[0], toc.pub)
})
test('(t = 12) we want some blocs to trim CINDEX', async (s1) => {
for (let i = 0; i < 10; i++) {
await s1.commit({ time: now + 12 })
}
})
test('(t = 13 #1) toc is coming back with 2 certs, whose 1 is a renewal', async (s1, cat, tac, toc) => {
await s1.commit({ time: now + 13 })
await s1.commit({ time: now + 13 })
const c1 = await cat.makeCert(<