fileDAL.ts 44.5 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
// Source file from duniter: Crypto-currency software to manage libre currency such as Ğ1
// Copyright (C) 2018  Cedric Moreau <cem.moreau@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.

14
15
import * as fs from 'fs'
import * as path from 'path'
16
17
18
19
20
21
import {SQLiteDriver} from "./drivers/SQLiteDriver"
import {ConfDAL} from "./fileDALs/ConfDAL"
import {StatDAL} from "./fileDALs/StatDAL"
import {ConfDTO} from "../dto/ConfDTO"
import {BlockDTO} from "../dto/BlockDTO"
import {DBHead} from "../db/DBHead"
22
import {DBIdentity, IdentityDAL} from "./sqliteDAL/IdentityDAL"
Cédric Moreau's avatar
Cédric Moreau committed
23
import {
24
25
  CindexEntry,
  FullCindexEntry,
26
  FullIindexEntry,
27
28
29
  FullMindexEntry,
  IindexEntry,
  IndexEntry,
30
  MindexEntry,
31
32
  SimpleTxInput,
  SimpleUdEntryForWallet,
Cédric Moreau's avatar
Cédric Moreau committed
33
34
  SindexEntry
} from "../indexer"
35
import {TransactionDTO} from "../dto/TransactionDTO"
36
import {CertDAL, DBCert} from "./sqliteDAL/CertDAL"
37
import {DBBlock} from "../db/DBBlock"
38
import {DBMembership, MembershipDAL} from "./sqliteDAL/MembershipDAL"
39
import {MerkleDTO} from "../dto/MerkleDTO"
40
import {CommonConstants} from "../common-libs/constants"
41
import {PowDAL} from "./fileDALs/PowDAL";
42
import {Initiable} from "./sqliteDAL/Initiable"
43
import {MetaDAL} from "./sqliteDAL/MetaDAL"
44
45
import {DataErrors} from "../common-libs/errors"
import {BasicRevocableIdentity, IdentityDTO} from "../dto/IdentityDTO"
46
import {FileSystem} from "../system/directory"
47
import {WoTBInstance} from "../wot"
Cédric Moreau's avatar
Cédric Moreau committed
48
49
50
51
52
53
54
55
56
57
58
59
import {IIndexDAO} from "./indexDAL/abstract/IIndexDAO"
import {LokiIIndex} from "./indexDAL/loki/LokiIIndex"
import {BIndexDAO} from "./indexDAL/abstract/BIndexDAO"
import {MIndexDAO} from "./indexDAL/abstract/MIndexDAO"
import {SIndexDAO} from "./indexDAL/abstract/SIndexDAO"
import {CIndexDAO} from "./indexDAL/abstract/CIndexDAO"
import {IdentityForRequirements} from "../../service/BlockchainService"
import {LokiSIndex} from "./indexDAL/loki/LokiSIndex"
import {LokiCIndex} from "./indexDAL/loki/LokiCIndex"
import {LokiMIndex} from "./indexDAL/loki/LokiMIndex";
import {LokiBIndex} from "./indexDAL/loki/LokiBIndex"
import {NewLogger} from "../logger"
60
61
62
63
64
import {LokiBlockchain} from "./indexDAL/loki/LokiBlockchain"
import {BlockchainDAO} from "./indexDAL/abstract/BlockchainDAO"
import {LokiTransactions} from "./indexDAL/loki/LokiTransactions"
import {TxsDAO} from "./indexDAL/abstract/TxsDAO"
import {LokiJsDriver} from "./drivers/LokiJsDriver"
65
66
import {WalletDAO} from "./indexDAL/abstract/WalletDAO"
import {LokiWallet} from "./indexDAL/loki/LokiWallet"
67
68
import {PeerDAO} from "./indexDAL/abstract/PeerDAO"
import {LokiPeer} from "./indexDAL/loki/LokiPeer"
69
70
import {DBTx} from "../db/DBTx"
import {DBWallet} from "../db/DBWallet"
71
import {Tristamp} from "../common/Tristamp"
72
73
74
import {CFSBlockchainArchive} from "./indexDAL/CFSBlockchainArchive"
import {CFSCore} from "./fileDALs/CFSCore"
import {BlockchainArchiveDAO} from "./indexDAL/abstract/BlockchainArchiveDAO"
75
76
import {Underscore} from "../common-libs/underscore"
import {DBPeer} from "../db/DBPeer"
77
78
import {MonitorFlushedIndex} from "../debug/MonitorFlushedIndex"
import {cliprogram} from "../common-libs/programOptions"
79
80
81
import {DividendDAO, UDSource} from "./indexDAL/abstract/DividendDAO"
import {LokiDividend} from "./indexDAL/loki/LokiDividend"
import {HttpSource, HttpUD} from "../../modules/bma/lib/dtos"
82
83
import {GenericDAO} from "./indexDAL/abstract/GenericDAO"
import {LokiDAO} from "./indexDAL/loki/LokiDAO"
84

Cédric Moreau's avatar
Cédric Moreau committed
85
86
const readline = require('readline')
const indexer = require('../indexer').Indexer
Cédric Moreau's avatar
Cédric Moreau committed
87
const logger = require('../logger').NewLogger('filedal');
Cédric Moreau's avatar
Cédric Moreau committed
88
89
90
91
const constants = require('../constants');

export interface FileDALParams {
  home:string
92
  fs:FileSystem
Cédric Moreau's avatar
Cédric Moreau committed
93
  dbf:() => SQLiteDriver
94
  dbf2: () => LokiJsDriver
95
  wotb:WoTBInstance
Cédric Moreau's avatar
Cédric Moreau committed
96
97
}

98
99
100
101
102
103
104
export interface IndexBatch {
  mindex: MindexEntry[]
  iindex: IindexEntry[]
  sindex: SindexEntry[]
  cindex: CindexEntry[]
}

Cédric Moreau's avatar
Cédric Moreau committed
105
106
107
export class FileDAL {

  rootPath:string
108
  fs: FileSystem
109
  loki:LokiJsDriver
Cédric Moreau's avatar
Cédric Moreau committed
110
  sqliteDriver:SQLiteDriver
111
  wotb:WoTBInstance
Cédric Moreau's avatar
Cédric Moreau committed
112
113
  profile:string

114
  // Simple file accessors
115
  powDAL:PowDAL
116
  confDAL:ConfDAL
117
  statDAL:StatDAL
118
  blockchainArchiveDAL:BlockchainArchiveDAO<DBBlock>
119
120
121

  // SQLite DALs
  metaDAL:MetaDAL
122
  idtyDAL:IdentityDAL
123
  certDAL:CertDAL
124
  msDAL:MembershipDAL
125
126
127
128

  // New DAO entities
  blockDAL:BlockchainDAO
  txsDAL:TxsDAO
129
  peerDAL:PeerDAO
130
  walletDAL:WalletDAO
Cédric Moreau's avatar
Cédric Moreau committed
131
132
133
134
135
  bindexDAL:BIndexDAO
  mindexDAL:MIndexDAO
  iindexDAL:IIndexDAO
  sindexDAL:SIndexDAO
  cindexDAL:CIndexDAO
136
  dividendDAL:DividendDAO
137
  newDals:{ [k:string]: Initiable }
138
139
  private dals:(BlockchainArchiveDAO<any>|PeerDAO|WalletDAO|GenericDAO<any>)[]
  private daos:LokiDAO[]
Cédric Moreau's avatar
Cédric Moreau committed
140
141
142
143
144
145
146

  loadConfHook: (conf:ConfDTO) => Promise<void>
  saveConfHook: (conf:ConfDTO) => Promise<ConfDTO>

  constructor(params:FileDALParams) {
    this.rootPath = params.home
    this.sqliteDriver = params.dbf()
147
    this.loki = params.dbf2()
Cédric Moreau's avatar
Cédric Moreau committed
148
149
    this.wotb = params.wotb
    this.profile = 'DAL'
150
    this.fs = params.fs
Cédric Moreau's avatar
Cédric Moreau committed
151
152

    // DALs
153
154
    this.powDAL = new PowDAL(this.rootPath, params.fs)
    this.confDAL = new ConfDAL(this.rootPath, params.fs)
Cédric Moreau's avatar
Cédric Moreau committed
155
    this.metaDAL = new (require('./sqliteDAL/MetaDAL').MetaDAL)(this.sqliteDriver);
156
    this.blockchainArchiveDAL = new CFSBlockchainArchive(new CFSCore(path.join(this.rootPath, '/archives'), params.fs), CommonConstants.ARCHIVES_BLOCKS_CHUNK)
157
158
    this.blockDAL = new LokiBlockchain(this.loki.getLokiInstance())
    this.txsDAL = new LokiTransactions(this.loki.getLokiInstance())
159
    this.statDAL = new StatDAL(this.rootPath, params.fs)
Cédric Moreau's avatar
Cédric Moreau committed
160
161
162
    this.idtyDAL = new (require('./sqliteDAL/IdentityDAL').IdentityDAL)(this.sqliteDriver);
    this.certDAL = new (require('./sqliteDAL/CertDAL').CertDAL)(this.sqliteDriver);
    this.msDAL = new (require('./sqliteDAL/MembershipDAL').MembershipDAL)(this.sqliteDriver);
163
    this.peerDAL = new LokiPeer(this.loki.getLokiInstance())
164
    this.walletDAL = new LokiWallet(this.loki.getLokiInstance())
165
166
167
168
169
    this.bindexDAL = new LokiBIndex(this.loki.getLokiInstance())
    this.mindexDAL = new LokiMIndex(this.loki.getLokiInstance())
    this.iindexDAL = new LokiIIndex(this.loki.getLokiInstance())
    this.sindexDAL = new LokiSIndex(this.loki.getLokiInstance())
    this.cindexDAL = new LokiCIndex(this.loki.getLokiInstance())
170
    this.dividendDAL = new LokiDividend(this.loki.getLokiInstance())
Cédric Moreau's avatar
Cédric Moreau committed
171
172

    this.newDals = {
173
      'powDAL': this.powDAL,
Cédric Moreau's avatar
Cédric Moreau committed
174
175
176
177
178
179
180
181
182
183
184
185
186
187
      'metaDAL': this.metaDAL,
      'blockDAL': this.blockDAL,
      'certDAL': this.certDAL,
      'msDAL': this.msDAL,
      'idtyDAL': this.idtyDAL,
      'txsDAL': this.txsDAL,
      'peerDAL': this.peerDAL,
      'confDAL': this.confDAL,
      'statDAL': this.statDAL,
      'walletDAL': this.walletDAL,
      'bindexDAL': this.bindexDAL,
      'mindexDAL': this.mindexDAL,
      'iindexDAL': this.iindexDAL,
      'sindexDAL': this.sindexDAL,
188
      'cindexDAL': this.cindexDAL,
189
      'dividendDAL': this.dividendDAL,
190
      'blockchainArchiveDAL': this.blockchainArchiveDAL,
Cédric Moreau's avatar
Cédric Moreau committed
191
192
193
    }
  }

194
195
196
197
198
199
200
201
  public enableChangesAPI() {
    this.daos.map(d => d.enableChangesAPI())
  }

  public disableChangesAPI() {
    this.daos.map(d => d.disableChangesAPI())
  }

Cédric Moreau's avatar
Cédric Moreau committed
202
  async init(conf:ConfDTO) {
203
204
    // Init LokiJS
    await this.loki.loadDatabase()
205
206
207
208
209
210
211
212
213
214
215
216
217
    this.daos = [
      this.blockDAL,
      this.txsDAL,
      this.peerDAL,
      this.walletDAL,
      this.bindexDAL,
      this.mindexDAL,
      this.iindexDAL,
      this.sindexDAL,
      this.cindexDAL,
      this.dividendDAL
    ]
    this.dals = [
218
219
      this.blockDAL,
      this.txsDAL,
220
      this.peerDAL,
221
      this.walletDAL,
222
223
224
225
226
      this.bindexDAL,
      this.mindexDAL,
      this.iindexDAL,
      this.sindexDAL,
      this.cindexDAL,
227
      this.dividendDAL,
228
      this.blockchainArchiveDAL,
229
    ]
230
    for (const indexDAL of this.dals) {
231
232
      indexDAL.triggerInit()
    }
233
    const dalNames = Underscore.keys(this.newDals);
Cédric Moreau's avatar
Cédric Moreau committed
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
    for (const dalName of dalNames) {
      const dal = this.newDals[dalName];
      await dal.init();
    }
    logger.debug("Upgrade database...");
    await this.metaDAL.upgradeDatabase(conf);
    // Update the maximum certifications count a member can issue into the C++ addon
    const currencyParams = await this.getParameters();
    if (currencyParams && currencyParams.sigStock !== undefined && currencyParams.sigStock !== null) {
      this.wotb.setMaxCert(currencyParams.sigStock);
    }
  }

  getDBVersion() {
    return this.metaDAL.getVersion()
  }

251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
  /**
   * Transfer a chunk of blocks from memory DB to archives if the memory DB overflows.
   * @returns {Promise<void>}
   */
  async archiveBlocks() {
    const lastArchived = await this.blockchainArchiveDAL.getLastSavedBlock()
    const current = await this.blockDAL.getCurrent()
    const lastNumber = lastArchived ? lastArchived.number : -1
    const currentNumber = current ? current.number : -1
    const difference = currentNumber - lastNumber
    if (difference > CommonConstants.BLOCKS_IN_MEMORY_MAX) {
      const CHUNK_SIZE = this.blockchainArchiveDAL.chunkSize
      const nbBlocksOverflow = difference - CommonConstants.BLOCKS_IN_MEMORY_MAX
      const chunks = (nbBlocksOverflow - (nbBlocksOverflow % CHUNK_SIZE)) / CHUNK_SIZE
      for (let i = 0; i < chunks; i++) {
        const start = lastNumber + (i*CHUNK_SIZE) + 1
        const end = lastNumber + (i*CHUNK_SIZE) + CHUNK_SIZE
        const memBlocks = await this.blockDAL.getNonForkChunk(start, end)
        if (memBlocks.length !== CHUNK_SIZE) {
          throw Error(DataErrors[DataErrors.CANNOT_ARCHIVE_CHUNK_WRONG_SIZE])
        }
        await this.blockchainArchiveDAL.archive(memBlocks)
        await this.blockDAL.trimBlocks(end)
      }
    }
  }

Cédric Moreau's avatar
Cédric Moreau committed
278
279
280
281
282
283
284
285
286
287
288
289
290
291
  writeFileOfBlock(block:DBBlock) {
    return this.blockDAL.saveBlock(block)
  }

  writeSideFileOfBlock(block:DBBlock) {
    return this.blockDAL.saveSideBlock(block)
  }

  listAllPeers() {
    return this.peerDAL.listAll()
  }

  async getPeer(pubkey:string) {
    try {
292
      return await this.peerDAL.getPeer(pubkey)
Cédric Moreau's avatar
Cédric Moreau committed
293
294
295
296
297
    } catch (err) {
      throw Error('Unknown peer ' + pubkey);
    }
  }

298
  async getWS2Peers() {
Éloïs's avatar
Éloïs committed
299
    return  this.peerDAL.getPeersWithEndpointsLike('WS2P')
300
301
  }

302
303
304
305
306
307
308
309
310
311
312
313
314
315
  getAbsoluteBlockInForkWindowByBlockstamp(blockstamp:string) {
    if (!blockstamp) throw "Blockstamp is required to find the block";
    const sp = blockstamp.split('-');
    const number = parseInt(sp[0]);
    const hash = sp[1];
    return this.getAbsoluteBlockInForkWindow(number, hash)
  }

  getAbsoluteValidBlockInForkWindowByBlockstamp(blockstamp:string) {
    if (!blockstamp) throw "Blockstamp is required to find the block";
    const sp = blockstamp.split('-');
    const number = parseInt(sp[0]);
    const hash = sp[1];
    return this.getAbsoluteValidBlockInForkWindow(number, hash)
Cédric Moreau's avatar
Cédric Moreau committed
316
317
  }

318
  async getBlockWeHaveItForSure(number:number): Promise<DBBlock> {
319
    return (await this.blockDAL.getBlock(number)) as DBBlock || (await this.blockchainArchiveDAL.getBlockByNumber(number))
320
321
  }

322
323
324
325
326
  // Duniter-UI dependency
  async getBlock(number: number): Promise<DBBlock|null> {
    return this.getFullBlockOf(number)
  }

327
  async getFullBlockOf(number: number): Promise<DBBlock|null> {
328
    return (await this.blockDAL.getBlock(number)) || (await this.blockchainArchiveDAL.getBlockByNumber(number))
Cédric Moreau's avatar
Cédric Moreau committed
329
330
  }

331
  async getBlockstampOf(number: number): Promise<string|null> {
332
    const block = await this.getTristampOf(number)
333
334
335
336
    if (block) {
      return [block.number, block.hash].join('-')
    }
    return null
337
338
  }

339
  async getTristampOf(number: number): Promise<Tristamp|null> {
340
    return (await this.blockDAL.getBlock(number)) || (await this.blockchainArchiveDAL.getBlockByNumber(number))
Cédric Moreau's avatar
Cédric Moreau committed
341
342
  }

343
  async existsAbsoluteBlockInForkWindow(number:number, hash:string): Promise<boolean> {
344
    return !!(await this.getAbsoluteBlockByNumberAndHash(number, hash))
Cédric Moreau's avatar
Cédric Moreau committed
345
346
  }

347
  async getAbsoluteBlockInForkWindow(number:number, hash:string): Promise<DBBlock|null> {
348
    return this.getAbsoluteBlockByNumberAndHash(number, hash)
Cédric Moreau's avatar
Cédric Moreau committed
349
350
  }

351
  async getAbsoluteValidBlockInForkWindow(number:number, hash:string): Promise<DBBlock|null> {
352
    const block = await this.getAbsoluteBlockByNumberAndHash(number, hash)
353
354
    if (block && !block.fork) {
      return block
Cédric Moreau's avatar
Cédric Moreau committed
355
    }
356
357
358
    return null
  }

359
  async getAbsoluteBlockByNumberAndHash(number:number, hash:string): Promise<DBBlock|null> {
360
361
362
363
364
365
    if (number > 0) {
      return (await this.blockDAL.getAbsoluteBlock(number, hash)) || (await this.blockchainArchiveDAL.getBlock(number, hash))
    } else {
      // Block#0 is special
      return (await this.blockDAL.getBlock(number)) || (await this.blockchainArchiveDAL.getBlockByNumber(number))
    }
Cédric Moreau's avatar
Cédric Moreau committed
366
367
  }

Cédric Moreau's avatar
Cédric Moreau committed
368
369
370
371
372
  async getAbsoluteBlockByBlockstamp(blockstamp: string): Promise<DBBlock|null> {
    const sp = blockstamp.split('-')
    return this.getAbsoluteBlockByNumberAndHash(parseInt(sp[0]), sp[1])
  }

Cédric Moreau's avatar
Cédric Moreau committed
373
374
375
  async existsNonChainableLink(from:string, vHEAD_1:DBHead, sigStock:number) {
    // Cert period rule
    const medianTime = vHEAD_1 ? vHEAD_1.medianTime : 0;
Cédric Moreau's avatar
Cédric Moreau committed
376
    const linksFrom:FullCindexEntry[] = await this.cindexDAL.reducablesFrom(from)
377
    const unchainables = Underscore.filter(linksFrom, (link:CindexEntry) => link.chainable_on > medianTime);
Cédric Moreau's avatar
Cédric Moreau committed
378
379
    if (unchainables.length > 0) return true;
    // Max stock rule
380
    let activeLinks = Underscore.filter(linksFrom, (link:CindexEntry) => !link.expired_on);
Cédric Moreau's avatar
Cédric Moreau committed
381
382
383
384
385
    return activeLinks.length >= sigStock;
  }


  async getCurrentBlockOrNull() {
386
    let current:DBBlock|null = null;
Cédric Moreau's avatar
Cédric Moreau committed
387
388
389
390
391
392
393
394
395
396
397
    try {
      current = await this.getBlockCurrent()
    } catch (e) {
      if (e != constants.ERROR.BLOCK.NO_CURRENT_BLOCK) {
        throw e;
      }
    }
    return current;
  }

  getPromoted(number:number) {
398
    return this.getFullBlockOf(number)
Cédric Moreau's avatar
Cédric Moreau committed
399
400
401
402
  }

  // Block

403
404
405
406
  getPotentialRootBlocks() {
    return this.blockDAL.getPotentialRoots()
  }

Cédric Moreau's avatar
Cédric Moreau committed
407
408
409
410
411
412
413
414
  lastBlockOfIssuer(issuer:string) {
    return this.blockDAL.lastBlockOfIssuer(issuer);
  }
  
  getCountOfPoW(issuer:string) {
    return this.blockDAL.getCountOfBlocksIssuedBy(issuer)
  }

415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
  async getBlocksBetween (start:number, end:number) {
    start = Math.max(0, start)
    end= Math.max(0, end)
    const blocks = await this.blockDAL.getBlocks(Math.max(0, start), end)
    if (blocks[0] && blocks[0].number === start) {
      // OK: we have all the blocks from memory
      return blocks
    }
    // Else: we have to pick them from archives
    const last = blocks[0] ? blocks[0].number - 1 : end
    const archiveBlocks = await this.blockchainArchiveDAL.getBlocks(start, last)
    const lastInArchives = archiveBlocks[archiveBlocks.length - 1] ? archiveBlocks[archiveBlocks.length - 1].number - 1 : end
    if (lastInArchives === end) {
      // OK: we have all the blocks from archives
      return archiveBlocks
    }
    // Otherwise: what is not taken in the archives are in memory
    const memBlocks = await this.blockDAL.getBlocks(archiveBlocks[archiveBlocks.length - 1].number + 1, end)
    return archiveBlocks.concat(memBlocks)
Cédric Moreau's avatar
Cédric Moreau committed
434
435
436
437
438
439
  }

  getForkBlocksFollowing(current:DBBlock) {
    return this.blockDAL.getNextForkBlocks(current.number, current.hash)
  }

440
441
  getPotentialForkBlocks(numberStart:number, medianTimeStart:number, maxNumber:number) {
    return this.blockDAL.getPotentialForkBlocks(numberStart, medianTimeStart, maxNumber)
442
443
  }

Cédric Moreau's avatar
Cédric Moreau committed
444
445
446
447
448
449
450
451
452
453
454
  async getBlockCurrent() {
    const current = await this.blockDAL.getCurrent();
    if (!current)
      throw 'No current block';
    return current;
  }

  getValidLinksTo(to:string) {
    return this.cindexDAL.getValidLinksTo(to)
  }

455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
  async getAvailableSourcesByPubkey(pubkey:string): Promise<HttpSource[]> {
    const txAvailable = await this.sindexDAL.getAvailableForPubkey(pubkey)
    const sources: UDSource[] = await this.dividendDAL.getUDSources(pubkey)
    return sources.map(d => {
      return {
        type: 'D',
        noffset: d.pos,
        identifier: pubkey,
        amount: d.amount,
        base: d.base,
        conditions: 'SIG(' + pubkey + ')'
      }
    }).concat(txAvailable.map(s => {
      return {
        type: 'T',
        noffset: s.pos,
        identifier: s.identifier,
        amount: s.amount,
        base: s.base,
        conditions: s.conditions
      }
    }))
  }

  async findByIdentifierPosAmountBase(identifier: string, pos: number, amount: number, base: number, isDividend: boolean): Promise<SimpleTxInput[]> {
    if (isDividend) {
      return this.dividendDAL.findUdSourceByIdentifierPosAmountBase(identifier, pos, amount, base)
    } else {
      return this.sindexDAL.findTxSourceByIdentifierPosAmountBase(identifier, pos, amount, base)
    }
Cédric Moreau's avatar
Cédric Moreau committed
485
486
  }

487
488
  async getGlobalIdentityByHashForExistence(hash:string): Promise<boolean> {
    const pending = await this.idtyDAL.getByHash(hash)
Cédric Moreau's avatar
Cédric Moreau committed
489
    if (!pending) {
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
      const idty = await this.iindexDAL.getFullFromHash(hash)
      if (!idty) {
        return false
      }
    }
    return true
  }

  async getGlobalIdentityByHashForHashingAndSig(hash:string): Promise<{ pubkey:string, uid:string, buid:string, sig:string }|null> {
    const pending = await this.idtyDAL.getByHash(hash)
    if (!pending) {
      const idty = await this.iindexDAL.getFullFromHash(hash)
      if (!idty) {
        return null
      }
      return {
        pubkey: idty.pub,
        uid: idty.uid,
        buid: idty.created_on,
        sig: idty.sig
      }
    }
    return pending
  }

  async getGlobalIdentityByHashForLookup(hash:string): Promise<{ pubkey:string, uid:string, buid:string, sig:string, member:boolean, wasMember:boolean }|null> {
    const pending = await this.idtyDAL.getByHash(hash)
    if (!pending) {
      const idty = await this.iindexDAL.getFullFromHash(hash)
      if (!idty) {
        return null
      }
      return {
        pubkey: idty.pub,
        uid: idty.uid,
        buid: idty.created_on,
        sig: idty.sig,
        member: idty.member,
        wasMember: idty.wasMember
      }
    }
    return pending
  }

  async getGlobalIdentityByHashForJoining(hash:string): Promise<{ pubkey:string, uid:string, buid:string, sig:string, member:boolean, wasMember:boolean, revoked:boolean }|null> {
    const pending = await this.idtyDAL.getByHash(hash)
    if (!pending) {
      const idty = await this.iindexDAL.getFullFromHash(hash)
      if (!idty) {
        return null
      }
      const membership = await this.mindexDAL.getReducedMS(idty.pub) as FullMindexEntry
      return {
        pubkey: idty.pub,
        uid: idty.uid,
        buid: idty.created_on,
        sig: idty.sig,
        member: idty.member,
        wasMember: idty.wasMember,
        revoked: !!(membership.revoked_on)
      }
    }
    return pending
  }

  async getGlobalIdentityByHashForIsMember(hash:string): Promise<{ pub:string, member:boolean }|null> {
    const pending = await this.idtyDAL.getByHash(hash)
    if (!pending) {
      const idty = await this.iindexDAL.getFullFromHash(hash)
      if (!idty) {
        return null
      }
      return {
        pub: idty.pub,
        member: idty.member
      }
    }
    return {
      pub: pending.pubkey,
      member: pending.member
    }
  }

  async getGlobalIdentityByHashForRevocation(hash:string): Promise<{ pub:string, uid:string, created_on:string, sig:string, member:boolean, wasMember:boolean, revoked:boolean, revocation_sig:string|null, expires_on:number }|null> {
    const pending = await this.idtyDAL.getByHash(hash)
    if (!pending) {
      const idty = await this.iindexDAL.getFullFromHash(hash)
      if (!idty) {
        return null
      }
      const membership = await this.mindexDAL.getReducedMS(idty.pub) as FullMindexEntry
      return {
        pub: idty.pub,
        uid: idty.uid,
        sig: idty.sig,
        member: idty.member,
        wasMember: idty.wasMember,
        expires_on: membership.expires_on,
        created_on: idty.created_on,
        revoked: !!(membership.revoked_on),
        revocation_sig: membership.revocation
      }
    }
    return {
      pub: pending.pubkey,
      uid: pending.uid,
      sig: pending.sig,
      expires_on: pending.expires_on,
      created_on: pending.buid,
      member: pending.member,
      wasMember: pending.wasMember,
      revoked: pending.revoked,
      revocation_sig: pending.revocation_sig
Cédric Moreau's avatar
Cédric Moreau committed
603
604
605
606
607
608
609
    }
  }

  getMembers() {
    return this.iindexDAL.getMembers()
  }

610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
  async getWrittenIdtyByPubkeyForHash(pubkey:string): Promise<{ hash:string }> {
    return this.getWrittenForSureIdtyByPubkey(pubkey)
  }

  async getWrittenIdtyByPubkeyForHashing(pubkey:string): Promise<{ uid:string, created_on:string, pub:string }> {
    return this.getWrittenForSureIdtyByPubkey(pubkey)
  }

  async getWrittenIdtyByPubkeyForWotbID(pubkey:string): Promise<{ wotb_id:number }> {
    return this.getWrittenForSureIdtyByPubkey(pubkey)
  }

  async getWrittenIdtyByPubkeyForUidAndPubkey(pubkey:string): Promise<{ pub:string, uid:string }> {
    return this.getWrittenForSureIdtyByPubkey(pubkey)
  }

  async getWrittenIdtyByPubkeyForIsMember(pubkey:string): Promise<{ member:boolean }|null> {
    return this.iindexDAL.getFromPubkey(pubkey)
  }

  async getWrittenIdtyByPubkeyForUidAndIsMemberAndWasMember(pubkey:string): Promise<{ uid:string, member:boolean, wasMember:boolean }|null> {
    return this.iindexDAL.getFromPubkey(pubkey)
  }

  async getWrittenIdtyByPubkeyOrUidForIsMemberAndPubkey(search:string): Promise<{ pub:string, member:boolean }|null> {
    return this.iindexDAL.getFromPubkeyOrUid(search)
  }

  async getWrittenIdtyByPubkeyOrUIdForHashingAndIsMember(search:string): Promise<{ uid:string, created_on:string, pub:string, member:boolean }|null> {
    return await this.iindexDAL.getFromPubkeyOrUid(search)
  }

  async getWrittenIdtyByPubkeyForRevocationCheck(pubkey:string): Promise<{ pub:string, uid:string, created_on:string, sig:string, revoked_on:number|null }|null> {
643
644
    const idty = await this.iindexDAL.getFromPubkey(pubkey)
    if (!idty) {
645
646
647
648
649
650
651
652
653
      return null
    }
    const membership = await this.mindexDAL.getReducedMS(pubkey) as FullMindexEntry
    return {
      pub: idty.pub,
      uid: idty.uid,
      sig: idty.sig,
      created_on: idty.created_on,
      revoked_on: membership.revoked_on
654
    }
Cédric Moreau's avatar
Cédric Moreau committed
655
656
  }

657
658
  async getWrittenIdtyByPubkeyForCertificationCheck(pubkey:string): Promise<{ pub:string, uid:string, created_on:string, sig:string }|null> {
    const idty = await this.iindexDAL.getFromPubkey(pubkey)
659
    if (!idty) {
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
      return null
    }
    return {
      pub: idty.pub,
      uid: idty.uid,
      sig: idty.sig,
      created_on: idty.created_on,
    }
  }

  async getWrittenIdtyByPubkeyForUidAndMemberAndCreatedOn(pubkey:string): Promise<{ uid:string, member:boolean, created_on:string }|null> {
    const idty = await this.iindexDAL.getFromPubkey(pubkey)
    if (!idty) {
      return null
    }
    return {
      uid: idty.uid,
      member: idty.member,
      created_on: idty.created_on,
    }
  }

  private async getWrittenForSureIdtyByPubkey(pubkey:string) {
    const idty = await this.iindexDAL.getFromPubkey(pubkey)
    if (!idty) {
      throw Error(DataErrors[DataErrors.MEMBER_NOT_FOUND])
    }
    return idty
  }

  private async getWrittenForSureIdtyByUid(pubkey:string) {
    const idty = (await this.iindexDAL.getFullFromUID(pubkey))
    if (!idty) {
      throw Error(DataErrors[DataErrors.MEMBER_NOT_FOUND])
694
    }
695
696
697
    return idty
  }

698
  // Duniter-UI dependency
699
700
  async getWrittenIdtyByPubkey(pub:string): Promise<FullIindexEntry | null> {
    return await this.iindexDAL.getFromPubkey(pub)
701
702
  }

703
704
705
706
707
708
709
710
711
712
713
714
715
716
  async getWrittenIdtyByPubkeyForExistence(uid:string) {
    return !!(await this.iindexDAL.getFromPubkey(uid))
  }

  async getWrittenIdtyByUIDForExistence(uid:string) {
    return !!(await this.iindexDAL.getFromUID(uid))
  }

  async getWrittenIdtyByUidForHashing(uid:string): Promise<{ uid:string, created_on:string, pub:string }> {
    return this.getWrittenForSureIdtyByUid(uid)
  }

  async getWrittenIdtyByUIDForWotbId(uid:string): Promise<{ wotb_id:number }> {
    return this.getWrittenForSureIdtyByUid(uid)
Cédric Moreau's avatar
Cédric Moreau committed
717
718
719
720
  }

  async findPeersWhoseHashIsIn(hashes:string[]) {
    const peers = await this.peerDAL.listAll();
721
    return Underscore.chain(peers).filter((p:DBPeer) => hashes.indexOf(p.hash) !== -1).value()
Cédric Moreau's avatar
Cédric Moreau committed
722
723
724
725
726
727
728
729
730
731
  }

  getTxByHash(hash:string) {
    return this.txsDAL.getTX(hash)
  }

  removeTxByHash(hash:string) {
    return this.txsDAL.removeTX(hash)
  }

732
  getTransactionsPending(versionMin = 0) {
Cédric Moreau's avatar
Cédric Moreau committed
733
734
735
736
737
    return this.txsDAL.getAllPending(versionMin)
  }

  async getNonWritten(pubkey:string) {
    const pending = await this.idtyDAL.getPendingIdentities();
738
    return Underscore.chain(pending).where({pubkey: pubkey}).value()
Cédric Moreau's avatar
Cédric Moreau committed
739
740
741
742
743
744
  }

  async getRevocatingMembers() {
    const revoking = await this.idtyDAL.getToRevoke();
    const toRevoke = [];
    for (const pending of revoking) {
745
746
      const idty = await this.getWrittenIdtyByPubkeyForRevocationCheck(pending.pubkey)
      if (idty && !idty.revoked_on) {
Cédric Moreau's avatar
Cédric Moreau committed
747
748
749
750
751
752
753
754
755
756
        toRevoke.push(pending);
      }
    }
    return toRevoke;
  }

  getToBeKickedPubkeys() {
    return this.iindexDAL.getToBeKickedPubkeys()
  }

Éloïs's avatar
Éloïs committed
757
758
759
760
  getRevokedPubkeys() {
    return this.mindexDAL.getRevokedPubkeys()
  }

761
  async searchJustIdentities(search:string): Promise<DBIdentity[]> {
Cédric Moreau's avatar
Cédric Moreau committed
762
763
    const pendings = await this.idtyDAL.searchThoseMatching(search);
    const writtens = await this.iindexDAL.searchThoseMatching(search);
764
765
    const nonPendings = Underscore.filter(writtens, (w:IindexEntry) => {
      return Underscore.where(pendings, { pubkey: w.pub }).length == 0;
Cédric Moreau's avatar
Cédric Moreau committed
766
    });
Cédric Moreau's avatar
Cédric Moreau committed
767
768
769
770
771
    const found = pendings.concat(nonPendings.map((i:any) => {
      // Use the correct field
      i.pubkey = i.pub
      return i
    }));
772
    return await Promise.all<DBIdentity>(found.map(async (f:any) => {
Cédric Moreau's avatar
Cédric Moreau committed
773
774
      const ms = await this.mindexDAL.getReducedMS(f.pub);
      if (ms) {
775
        f.revoked_on = ms.revoked_on ? ms.revoked_on : null;
Cédric Moreau's avatar
Cédric Moreau committed
776
777
778
779
780
781
782
783
784
785
786
787
        f.revoked = !!f.revoked_on;
        f.revocation_sig = ms.revocation || null;
      }
      return f;
    }))
  }

  async certsToTarget(pub:string, hash:string) {
    const certs = await this.certDAL.getToTarget(hash);
    const links = await this.cindexDAL.getValidLinksTo(pub);
    let matching = certs;
    await Promise.all(links.map(async (entry:any) => {
Cédric Moreau's avatar
Cédric Moreau committed
788
      matching.push(await this.cindexEntry2DBCert(entry))
Cédric Moreau's avatar
Cédric Moreau committed
789
    }))
790
    matching  = Underscore.sortBy(matching, (c:DBCert) => -c.block);
Cédric Moreau's avatar
Cédric Moreau committed
791
792
793
794
795
796
797
798
    matching.reverse();
    return matching;
  }

  async certsFrom(pubkey:string) {
    const certs = await this.certDAL.getFromPubkeyCerts(pubkey);
    const links = await this.cindexDAL.getValidLinksFrom(pubkey);
    let matching = certs;
Cédric Moreau's avatar
Cédric Moreau committed
799
800
    await Promise.all(links.map(async (entry:CindexEntry) => {
      matching.push(await this.cindexEntry2DBCert(entry))
Cédric Moreau's avatar
Cédric Moreau committed
801
    }))
802
    matching  = Underscore.sortBy(matching, (c:DBCert) => -c.block);
Cédric Moreau's avatar
Cédric Moreau committed
803
804
805
806
    matching.reverse();
    return matching;
  }

Cédric Moreau's avatar
Cédric Moreau committed
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
  async cindexEntry2DBCert(entry:CindexEntry): Promise<DBCert> {
    const idty = await this.getWrittenIdtyByPubkeyForHash(entry.receiver)
    const wbt = entry.written_on.split('-')
    const block = (await this.blockDAL.getBlock(entry.created_on)) as DBBlock
    return {
      issuers: [entry.issuer],
      linked: true,
      written: true,
      written_block: parseInt(wbt[0]),
      written_hash: wbt[1],
      sig: entry.sig,
      block_number: block.number,
      block_hash: block.hash,
      target: idty.hash,
      to: entry.receiver,
      from: entry.issuer,
      block: block.number,
      expired: !!entry.expired_on,
      expires_on: entry.expires_on,
    }
  }

Cédric Moreau's avatar
Cédric Moreau committed
829
830
831
832
833
834
835
836
837
838
839
840
841
  async isSentry(pubkey:string, conf:ConfDTO) {
    const current = await this.getCurrentBlockOrNull();
    if (current) {
      const dSen = Math.ceil(Math.pow(current.membersCount, 1 / conf.stepMax));
      const linksFrom = await this.cindexDAL.getValidLinksFrom(pubkey);
      const linksTo = await this.cindexDAL.getValidLinksTo(pubkey);
      return linksFrom.length >= dSen && linksTo.length >= dSen;
    }
    return false;
  }

  async certsFindNew() {
    const certs = await this.certDAL.getNotLinked();
842
    return Underscore.chain(certs).where({linked: false}).sortBy((c:DBCert) => -c.block).value()
Cédric Moreau's avatar
Cédric Moreau committed
843
844
845
846
  }

  async certsNotLinkedToTarget(hash:string) {
    const certs = await this.certDAL.getNotLinkedToTarget(hash);
847
    return Underscore.chain(certs).sortBy((c:any) => -c.block).value();
Cédric Moreau's avatar
Cédric Moreau committed
848
849
850
851
852
853
854
855
856
857
858
859
860
861
  }

  async getMostRecentMembershipNumberForIssuer(issuer:string) {
    const mss = await this.msDAL.getMembershipsOfIssuer(issuer);
    const reduced = await this.mindexDAL.getReducedMS(issuer);
    let max = reduced ? parseInt(reduced.created_on) : -1;
    for (const ms of mss) {
      max = Math.max(ms.number, max);
    }
    return max;
  }

  async lastJoinOfIdentity(target:string) {
    let pending = await this.msDAL.getPendingINOfTarget(target);
862
    return Underscore.sortBy(pending, (ms:any) => -ms.number)[0];
Cédric Moreau's avatar
Cédric Moreau committed
863
864
  }

865
  async findNewcomers(blockMedianTime = 0): Promise<DBMembership[]> {
Cédric Moreau's avatar
Cédric Moreau committed
866
    const pending = await this.msDAL.getPendingIN()
867
    const mss: DBMembership[] = await Promise.all<DBMembership>(pending.map(async (p:any) => {
Cédric Moreau's avatar
Cédric Moreau committed
868
869
870
871
872
873
      const reduced = await this.mindexDAL.getReducedMS(p.issuer)
      if (!reduced || !reduced.chainable_on || blockMedianTime >= reduced.chainable_on || blockMedianTime < constants.TIME_TO_TURN_ON_BRG_107) {
        return p
      }
      return null
    }))
874
    return Underscore.chain(Underscore.filter(mss, ms => !!ms) as DBMembership[])
875
      .sortBy((ms:DBMembership) => -ms.blockNumber)
Cédric Moreau's avatar
Cédric Moreau committed
876
877
878
      .value()
  }

879
  async findLeavers(blockMedianTime = 0): Promise<DBMembership[]> {
880
    const pending = await this.msDAL.getPendingOUT();
881
    const mss = await Promise.all<DBMembership|null>(pending.map(async p => {
882
883
884
885
886
887
      const reduced = await this.mindexDAL.getReducedMS(p.issuer)
      if (!reduced || !reduced.chainable_on || blockMedianTime >= reduced.chainable_on || blockMedianTime < constants.TIME_TO_TURN_ON_BRG_107) {
        return p
      }
      return null
    }))
888
889
    return Underscore.chain(Underscore.filter(mss, ms => !!ms) as DBMembership[])
      .sortBy(ms => -ms.blockNumber)
890
      .value();
Cédric Moreau's avatar
Cédric Moreau committed
891
892
893
894
895
896
  }

  existsNonReplayableLink(from:string, to:string) {
    return  this.cindexDAL.existsNonReplayableLink(from, to)
  }

897
898
899
900
901
902
  async getSource(identifier:string, pos:number, isDividend: boolean): Promise<SimpleTxInput | null> {
    if (isDividend) {
      return this.dividendDAL.getUDSource(identifier, pos)
    } else {
      return this.sindexDAL.getTxSource(identifier, pos)
    }
Cédric Moreau's avatar
Cédric Moreau committed
903
904
  }

Éloïs's avatar
Éloïs committed
905
  async isMember(pubkey:string):Promise<boolean> {
Cédric Moreau's avatar
Cédric Moreau committed
906
907
    try {
      const idty = await this.iindexDAL.getFromPubkey(pubkey);
Éloïs's avatar
Éloïs committed
908
      if (idty === null) {
909
910
        return false
      }
Éloïs's avatar
Éloïs committed
911
      return true;
Cédric Moreau's avatar
Cédric Moreau committed
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
    } catch (err) {
      return false;
    }
  }

  async isMemberAndNonLeaver(pubkey:string) {
    try {
      const idty = await this.iindexDAL.getFromPubkey(pubkey);
      if (idty && idty.member) {
        return !(await this.isLeaving(pubkey));
      }
      return false;
    } catch (err) {
      return false;
    }
  }

  async isLeaving(pubkey:string) {
    const ms = await this.mindexDAL.getReducedMS(pubkey);
    return (ms && ms.leaving) || false;
  }

  async existsCert(cert:any) {
    const existing = await this.certDAL.existsGivenCert(cert);
    if (existing) return existing;
    const existsLink = await this.cindexDAL.existsNonReplayableLink(cert.from, cert.to);
    return !!existsLink;
  }

  deleteCert(cert:any) {
    return this.certDAL.deleteCert(cert)
  }

  deleteMS(ms:any) {
    return this.msDAL.deleteMS(ms)
  }

  async setRevoked(pubkey:string) {
950
    return await this.idtyDAL.setRevoked(pubkey)
Cédric Moreau's avatar
Cédric Moreau committed
951
952
  }

953
954
955
956
957
958
959
960
  setRevocating = (idty:BasicRevocableIdentity, revocation_sig:string) => {
    const dbIdentity = IdentityDTO.fromBasicIdentity(idty)
    dbIdentity.member = idty.member
    dbIdentity.wasMember = idty.wasMember
    dbIdentity.expires_on = idty.expires_on
    dbIdentity.revocation_sig = revocation_sig
    dbIdentity.revoked = false
    return this.idtyDAL.saveIdentity(dbIdentity)
Cédric Moreau's avatar
Cédric Moreau committed
961
962
963
964
965
966
967
968
969
970
971
972
973
974
  }

  async getPeerOrNull(pubkey:string) {
    let peer = null;
    try {
      peer = await this.getPeer(pubkey);
    } catch (e) {
      if (e != constants.ERROR.BLOCK.NO_CURRENT_BLOCK) {
        throw e;
      }
    }
    return peer;
  }

975
976
977
978
  async removePeerByPubkey(pubkey:string) {
    return this.peerDAL.removePeerByPubkey(pubkey)
  }

Cédric Moreau's avatar
Cédric Moreau committed
979
  async findAllPeersBut(pubkeys:string[]) {
Cédric Moreau's avatar
Cédric Moreau committed
980
981
982
983
984
985
986
    const peers = await this.listAllPeers();
    return peers.filter((peer:DBPeer) => pubkeys.indexOf(peer.pubkey) == -1
    && ['UP'].indexOf(peer.status) !== -1);
  }

  async listAllPeersWithStatusNewUP() {
    const peers = await this.peerDAL.listAll();
987
    return Underscore.chain(peers)
Cédric Moreau's avatar
Cédric Moreau committed
988
989
990
991
992
993
        .filter((p:DBPeer) => ['UP']
            .indexOf(p.status) !== -1).value();
  }

  async listAllPeersWithStatusNewUPWithtout(pub:string) {
    const peers = await this.peerDAL.listAll();
994
    return Underscore.chain(peers).filter((p:DBPeer) => p.status == 'UP').filter((p:DBPeer) => p.pubkey !== pub).value();
Cédric Moreau's avatar
Cédric Moreau committed
995
996
  }

997
  async findPeers(pubkey:string): Promise<DBPeer[]> {
Cédric Moreau's avatar
Cédric Moreau committed
998
999
1000
    try {
      const peer = await this.getPeer(pubkey);
      return [peer];
For faster browsing, not all history is shown. View entire blame