From fdd6338cb5bbd99584e95c44f424cc0992a609ff Mon Sep 17 00:00:00 2001
From: cgeek <cem.moreau@gmail.com>
Date: Sun, 23 Dec 2018 11:36:19 +0100
Subject: [PATCH] [enh] sync: enhance milestones downloading speed
---
app/lib/common-libs/constants.ts | 1 +
app/lib/common-libs/errors.ts | 4 +-
app/modules/bma/index.ts | 25 +++++----
app/modules/bma/lib/bma.ts | 2 +
app/modules/bma/lib/constants.ts | 3 +-
app/modules/bma/lib/controllers/blockchain.ts | 8 ++-
app/modules/bma/lib/dtos.ts | 22 ++++++++
app/modules/bma/lib/parameters.ts | 11 ++++
app/modules/crawler/lib/contacter.ts | 8 +++
.../crawler/lib/sync/AbstractSynchronizer.ts | 1 +
.../crawler/lib/sync/BMARemoteContacter.ts | 8 +++
.../crawler/lib/sync/IRemoteContacter.ts | 4 ++
.../crawler/lib/sync/LocalPathSynchronizer.ts | 4 ++
.../crawler/lib/sync/P2PSyncDownloader.ts | 9 ++--
.../crawler/lib/sync/RemoteSynchronizer.ts | 44 +++++++++++++++
.../crawler/lib/sync/WS2PRemoteContacter.ts | 8 +++
.../crawler/lib/sync/p2p/p2p-candidate.ts | 8 ++-
.../crawler/lib/sync/v2/ValidatorStream.ts | 2 +-
app/modules/ws2p/lib/WS2PDocpoolPuller.ts | 2 +
app/modules/ws2p/lib/WS2PRequester.ts | 15 +++++-
.../ws2p/lib/impl/WS2PReqMapperByServer.ts | 9 ++++
.../ws2p/lib/interface/WS2PReqMapper.ts | 3 ++
.../lib/interface/WS2PServerMessageHandler.ts | 12 +++++
server.ts | 54 ++++++++++++++++++-
24 files changed, 242 insertions(+), 25 deletions(-)
diff --git a/app/lib/common-libs/constants.ts b/app/lib/common-libs/constants.ts
index 223c77795..9b15d71a6 100755
--- a/app/lib/common-libs/constants.ts
+++ b/app/lib/common-libs/constants.ts
@@ -309,6 +309,7 @@ export const CommonConstants = {
ARCHIVES_BLOCKS_CHUNK: 250,
SYNC_BLOCKS_CHUNK: 250,
+ MILESTONES_PER_PAGE: 50,
CHUNK_PREFIX: 'chunk_',
BLOCKS_IN_MEMORY_MAX: 288 * 60, // 288 = 1 day
diff --git a/app/lib/common-libs/errors.ts b/app/lib/common-libs/errors.ts
index 0db0e8a14..412c07ccd 100755
--- a/app/lib/common-libs/errors.ts
+++ b/app/lib/common-libs/errors.ts
@@ -1,4 +1,3 @@
-
export enum DataErrors {
INVALID_LEVELDB_IINDEX_DATA_WAS_KICKED,
INVALID_LEVELDB_IINDEX_DATA_TO_BE_KICKED,
@@ -26,5 +25,6 @@ export enum DataErrors {
CANNOT_REAPPLY_NO_CURRENT_BLOCK,
CANNOT_REVERT_NO_CURRENT_BLOCK,
BLOCK_TO_REVERT_NOT_FOUND,
- MEMBER_NOT_FOUND
+ MEMBER_NOT_FOUND,
+ MILESTONE_BLOCK_NOT_FOUND
}
diff --git a/app/modules/bma/index.ts b/app/modules/bma/index.ts
index 7bfd78702..b28c5025d 100644
--- a/app/modules/bma/index.ts
+++ b/app/modules/bma/index.ts
@@ -149,19 +149,6 @@ export const BmaDependency = {
if (program.upnp === true) {
conf.upnp = true;
}
-
- // Configuration errors
- if (!conf.nobma) {
- if(!conf.ipv4 && !conf.ipv6){
- throw new Error("No interface to listen to.");
- }
- if(!conf.remoteipv4 && !conf.remoteipv6 && !conf.remotehost){
- throw new Error('No interface for remote contact.');
- }
- if (!conf.remoteport) {
- throw new Error('No port for remote contact.');
- }
- }
},
beforeSave: async (conf:NetworkConfDTO, program:any) => {
@@ -175,6 +162,18 @@ export const BmaDependency = {
service: {
input: (server:Server, conf:NetworkConfDTO, logger:any) => {
+ // Configuration errors
+ if (!conf.nobma) {
+ if(!conf.ipv4 && !conf.ipv6){
+ throw new Error("BMA: no interface to listen to.");
+ }
+ if(!conf.remoteipv4 && !conf.remoteipv6 && !conf.remotehost){
+ throw new Error('BMA: no interface for remote contact.');
+ }
+ if (!conf.remoteport) {
+ throw new Error('BMA: no port for remote contact.');
+ }
+ }
if (!conf.nobma) {
server.addEndpointsDefinitions(() => Promise.resolve(getEndpoint(conf)))
server.addWrongEndpointFilter((endpoints:string[]) => getWrongEndpoints(endpoints, server.conf.pair.pub))
diff --git a/app/modules/bma/lib/bma.ts b/app/modules/bma/lib/bma.ts
index 129bcbc15..66c16dde7 100644
--- a/app/modules/bma/lib/bma.ts
+++ b/app/modules/bma/lib/bma.ts
@@ -65,6 +65,8 @@ export const bma = function(server:Server, interfaces:NetworkInterface[]|null, h
httpMethods.httpPOST( '/blockchain/block', (req:any) => blockchain.parseBlock(req), BMALimitation.limitAsHighUsage());
httpMethods.httpGET( '/blockchain/block/:number', (req:any) => blockchain.promoted(req), BMALimitation.limitAsHighUsage());
httpMethods.httpGET( '/blockchain/blocks/:count/:from', (req:any) => blockchain.blocks(req), BMALimitation.limitAsHighUsage());
+ httpMethods.httpGET( '/blockchain/milestones', (req:any) => blockchain.milestones(req), BMALimitation.limitAsHighUsage());
+ httpMethods.httpGET( '/blockchain/milestones/:page', (req:any) => blockchain.milestones(req), BMALimitation.limitAsHighUsage());
httpMethods.httpGET( '/blockchain/current', (req:any) => blockchain.current(), BMALimitation.limitAsHighUsage());
httpMethods.httpGET( '/blockchain/hardship/:search', (req:any) => blockchain.hardship(req), BMALimitation.limitAsHighUsage());
httpMethods.httpGET( '/blockchain/difficulties', (req:any) => blockchain.difficulties(), BMALimitation.limitAsHighUsage());
diff --git a/app/modules/bma/lib/constants.ts b/app/modules/bma/lib/constants.ts
index 2488dd573..4ba022f29 100644
--- a/app/modules/bma/lib/constants.ts
+++ b/app/modules/bma/lib/constants.ts
@@ -52,7 +52,8 @@ export const BMAConstants = {
NO_CURRENT_BLOCK: { httpCode: 404, uerr: { ucode: 2010, message: "No current block" }},
PEER_NOT_FOUND: { httpCode: 404, uerr: { ucode: 2012, message: "Peer not found" }},
NO_IDTY_MATCHING_PUB_OR_UID: { httpCode: 404, uerr: { ucode: 2021, message: "No identity matching this pubkey or uid" }},
- TX_NOT_FOUND: { httpCode: 400, uerr: { ucode: 2034, message: 'Transaction not found' }}
+ TX_NOT_FOUND: { httpCode: 400, uerr: { ucode: 2034, message: 'Transaction not found' }},
+ INCORRECT_PAGE_NUMBER: { httpCode: 400, uerr: { ucode: 2035, message: 'Incorrect page number' }}
// New errors: range 3000-4000
}
diff --git a/app/modules/bma/lib/controllers/blockchain.ts b/app/modules/bma/lib/controllers/blockchain.ts
index 2fc5fa748..d5afc6273 100644
--- a/app/modules/bma/lib/controllers/blockchain.ts
+++ b/app/modules/bma/lib/controllers/blockchain.ts
@@ -24,15 +24,16 @@ import {
HttpHardship,
HttpMembership,
HttpMemberships,
+ HttpMilestonePage,
HttpParameters,
HttpStat
} from "../dtos"
import {TransactionDTO} from "../../../../lib/dto/TransactionDTO"
import {DataErrors} from "../../../../lib/common-libs/errors"
import {Underscore} from "../../../../lib/common-libs/underscore"
+import * as toJson from "../tojson"
const http2raw = require('../http2raw');
-const toJson = require('../tojson');
export class BlockchainBinding extends AbstractController {
@@ -168,6 +169,11 @@ export class BlockchainBinding extends AbstractController {
}))
}
+ async milestones(req: any): Promise<HttpMilestonePage> {
+ const page = ParametersService.getPage(req)
+ return this.server.milestones(page)
+ }
+
async current(): Promise<HttpBlock> {
const current = await this.server.dal.getCurrentBlockOrNull();
if (!current) throw BMAConstants.ERRORS.NO_CURRENT_BLOCK;
diff --git a/app/modules/bma/lib/dtos.ts b/app/modules/bma/lib/dtos.ts
index d0b7f27d1..63805afe0 100644
--- a/app/modules/bma/lib/dtos.ts
+++ b/app/modules/bma/lib/dtos.ts
@@ -969,3 +969,25 @@ export interface HttpSandboxes {
export const LogLink = {
link: String
};
+
+export interface HttpMilestonePage {
+ totalPages: number
+ chunkSize: number
+ milestonesPerPage: number
+ currentPage?: number
+ blocks?: HttpBlock[]
+}
+
+export const Milestones = {
+ totalPages: Number,
+ chunkSize: Number,
+ milestonesPerPage: Number,
+ currentPage: Number,
+ "blocks": [Block]
+}
+
+export const MilestonesPage = {
+ totalPages: Number,
+ chunkSize: Number,
+ milestonesPerPage: Number,
+}
diff --git a/app/modules/bma/lib/parameters.ts b/app/modules/bma/lib/parameters.ts
index 3db765af1..9126088fa 100644
--- a/app/modules/bma/lib/parameters.ts
+++ b/app/modules/bma/lib/parameters.ts
@@ -73,6 +73,17 @@ export class ParametersService {
return parseInt(req.params.minsig)
}
+ static getPage(req:any): number|undefined {
+ if(!req.params.page){
+ return undefined
+ }
+ const matches = req.params.page.match(/\d+/)
+ if(!matches){
+ throw Error("`page` format is incorrect, must be an integer")
+ }
+ return parseInt(req.params.page)
+ }
+
static getPubkey = function (req:any, callback:any){
if(!req.params.pubkey){
callback('Parameter `pubkey` is required');
diff --git a/app/modules/crawler/lib/contacter.ts b/app/modules/crawler/lib/contacter.ts
index 75f37f0e9..105a8c2df 100644
--- a/app/modules/crawler/lib/contacter.ts
+++ b/app/modules/crawler/lib/contacter.ts
@@ -57,6 +57,14 @@ export class Contacter {
getCurrent() {
return this.get('/blockchain/current', dtos.Block)
}
+
+ getMilestonesPage() {
+ return this.get('/blockchain/milestones', dtos.MilestonesPage)
+ }
+
+ getMilestones(page: number) {
+ return this.get('/blockchain/milestones/' + page, dtos.Milestones)
+ }
getPeer() {
return this.get('/network/peering', dtos.Peer)
diff --git a/app/modules/crawler/lib/sync/AbstractSynchronizer.ts b/app/modules/crawler/lib/sync/AbstractSynchronizer.ts
index 803088c44..61455b381 100644
--- a/app/modules/crawler/lib/sync/AbstractSynchronizer.ts
+++ b/app/modules/crawler/lib/sync/AbstractSynchronizer.ts
@@ -28,6 +28,7 @@ export abstract class AbstractSynchronizer {
abstract initWithKnownLocalAndToAndCurrency(to: number, localNumber: number, currency: string): Promise<void>
abstract getCurrent(): Promise<BlockDTO|null>
abstract getBlock(number: number): Promise<BlockDTO|null>
+ abstract getMilestone(number: number): Promise<BlockDTO|null>
abstract p2pDownloader(): ISyncDownloader
abstract fsDownloader(): ISyncDownloader
abstract syncPeers(fullSync:boolean, to?:number): Promise<void>
diff --git a/app/modules/crawler/lib/sync/BMARemoteContacter.ts b/app/modules/crawler/lib/sync/BMARemoteContacter.ts
index 1a3f5be40..14fdbdeba 100644
--- a/app/modules/crawler/lib/sync/BMARemoteContacter.ts
+++ b/app/modules/crawler/lib/sync/BMARemoteContacter.ts
@@ -37,6 +37,14 @@ export class BMARemoteContacter implements IRemoteContacter {
return this.contacter.getBlocks(count, from)
}
+ getMilestones(page: number): Promise<{ chunkSize: number; totalPages: number; currentPage: number; milestonesPerPage: number; blocks: BlockDTO[] }> {
+ return this.contacter.getMilestones(page)
+ }
+
+ getMilestonesPage(): Promise<{ chunkSize: number; totalPages: number; milestonesPerPage: number }> {
+ return this.contacter.getMilestonesPage()
+ }
+
async getPeers(): Promise<(JSONDBPeer|null)[]> {
return (await this.contacter.getPeersArray()).peers
}
diff --git a/app/modules/crawler/lib/sync/IRemoteContacter.ts b/app/modules/crawler/lib/sync/IRemoteContacter.ts
index 42bbc4910..ef0c2e706 100644
--- a/app/modules/crawler/lib/sync/IRemoteContacter.ts
+++ b/app/modules/crawler/lib/sync/IRemoteContacter.ts
@@ -25,6 +25,10 @@ export interface IRemoteContacter {
getBlock(number: number): Promise<BlockDTO|null>
+ getMilestonesPage(): Promise<{ chunkSize: number, totalPages: number, milestonesPerPage: number }>
+
+ getMilestones(page: number): Promise<{ chunkSize: number, totalPages: number, currentPage: number, milestonesPerPage: number, blocks: BlockDTO[] }>
+
getBlocks(count: number, from: number): Promise<BlockDTO[]>
getRequirementsPending(number: number): Promise<HttpRequirements>
diff --git a/app/modules/crawler/lib/sync/LocalPathSynchronizer.ts b/app/modules/crawler/lib/sync/LocalPathSynchronizer.ts
index 3098f2ff1..3bb3881c0 100644
--- a/app/modules/crawler/lib/sync/LocalPathSynchronizer.ts
+++ b/app/modules/crawler/lib/sync/LocalPathSynchronizer.ts
@@ -105,6 +105,10 @@ export class LocalPathSynchronizer extends AbstractSynchronizer {
return chunk[position]
}
+ getMilestone(number: number) {
+ return this.getBlock(number)
+ }
+
async syncPeers(fullSync: boolean, to?: number): Promise<void> {
// Does nothing on LocalPathSynchronizer
}
diff --git a/app/modules/crawler/lib/sync/P2PSyncDownloader.ts b/app/modules/crawler/lib/sync/P2PSyncDownloader.ts
index 489342dfc..96db98a2d 100644
--- a/app/modules/crawler/lib/sync/P2PSyncDownloader.ts
+++ b/app/modules/crawler/lib/sync/P2PSyncDownloader.ts
@@ -56,7 +56,7 @@ export class P2PSyncDownloader extends ASyncDownloader implements ISyncDownloade
return this.p2pCandidates.filter(p => p.hasAvailableApi()).length
}
- private async waitForAvailableNodes(needed = 1): Promise<P2pCandidate[]> {
+ private async waitForAvailableNodesAndReserve(needed = 1): Promise<P2pCandidate[]> {
let nodesToWaitFor = this.p2pCandidates.slice()
let nodesAvailable: P2pCandidate[] = []
let i = 0
@@ -67,7 +67,10 @@ export class P2PSyncDownloader extends ASyncDownloader implements ISyncDownloade
nodesAvailable = nodesAvailable.concat(readyNodes)
i++
}
- return nodesAvailable
+ return nodesAvailable.slice(0, needed).map(n => {
+ n.reserve()
+ return n
+ })
}
/**
@@ -78,7 +81,7 @@ export class P2PSyncDownloader extends ASyncDownloader implements ISyncDownloade
private async getP2Pcandidates(chunkIndex: number): Promise<P2pCandidate[]> {
return this.fifoPromise.pushFIFOPromise('getP2Pcandidates_' + getNanosecondsTime(), async () => {
// We wait a bit to have some available nodes
- const readyNodes = await this.waitForAvailableNodes()
+ const readyNodes = await this.waitForAvailableNodesAndReserve()
// We remove the nodes impossible to reach (timeout)
let byAvgAnswerTime = Underscore.sortBy(readyNodes, p => p.avgResponseTime())
const parallelMax = Math.min(this.PARALLEL_PER_CHUNK, byAvgAnswerTime.length)
diff --git a/app/modules/crawler/lib/sync/RemoteSynchronizer.ts b/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
index 0d33f7977..7916d0c01 100644
--- a/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
+++ b/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
@@ -53,6 +53,10 @@ export class RemoteSynchronizer extends AbstractSynchronizer {
private localNumber: number
private watcher: Watcher
private endpoint: string = ""
+ private hasMilestonesPages: boolean|undefined
+ private milestones: { [k: number]: BlockDTO } = {}
+ private milestonesPerPage = 1
+ private maxPage = 0
constructor(
private host: string,
@@ -247,6 +251,46 @@ export class RemoteSynchronizer extends AbstractSynchronizer {
return this.node.getBlock(number)
}
+ async getMilestone(number: number): Promise<BlockDTO|null> {
+ if (this.hasMilestonesPages === undefined) {
+ try {
+ const mlPage = await this.node.getMilestonesPage()
+ this.hasMilestonesPages = mlPage.chunkSize === this.chunkSize
+ this.milestonesPerPage = mlPage.milestonesPerPage
+ this.maxPage = mlPage.totalPages
+ } catch (e) {
+ this.hasMilestonesPages = false
+ }
+ }
+ if (!this.hasMilestonesPages) {
+ return this.getBlock(number)
+ }
+ if (this.milestones[number]) {
+ return this.milestones[number]
+ }
+
+ if ((number + 1) % this.chunkSize !== 0) {
+ // Something went wrong: we cannot rely on milestones method
+ this.hasMilestonesPages = false
+ return this.getBlock(number)
+ }
+ const chunkNumber = (number + 1) / this.chunkSize
+ const pageNumber = (chunkNumber - (chunkNumber % this.milestonesPerPage)) / this.milestonesPerPage + 1
+ if (pageNumber > this.maxPage) {
+ // The page is not available: we cannot rely on milestones method at this point
+ this.hasMilestonesPages = false
+ return this.getBlock(number)
+ }
+ const mlPage = await this.node.getMilestones(pageNumber)
+ mlPage.blocks.forEach(b => this.milestones[b.number] = b)
+ if (this.milestones[number]) {
+ return this.milestones[number]
+ }
+ // Even after the download, it seems we don't have our milestone. We will download normally.
+ this.hasMilestonesPages = false
+ return this.getBlock(number)
+ }
+
static async test(host: string, port: number, keypair: Keypair): Promise<BlockDTO> {
const syncApi = await RemoteSynchronizer.getSyncAPI([{ host, port }], keypair)
const current = await syncApi.api.getCurrent()
diff --git a/app/modules/crawler/lib/sync/WS2PRemoteContacter.ts b/app/modules/crawler/lib/sync/WS2PRemoteContacter.ts
index a5ea8af5f..eeb7226cb 100644
--- a/app/modules/crawler/lib/sync/WS2PRemoteContacter.ts
+++ b/app/modules/crawler/lib/sync/WS2PRemoteContacter.ts
@@ -34,6 +34,14 @@ export class WS2PRemoteContacter implements IRemoteContacter {
return this.requester.getBlock(number)
}
+ getMilestones(page: number): Promise<{ chunkSize: number; totalPages: number; currentPage: number; milestonesPerPage: number; blocks: BlockDTO[] }> {
+ return this.requester.getMilestones(page) as any
+ }
+
+ getMilestonesPage(): Promise<{ chunkSize: number; totalPages: number; milestonesPerPage: number }> {
+ return this.requester.getMilestonesPage()
+ }
+
getCurrent(): Promise<BlockDTO | null> {
return this.requester.getCurrent()
}
diff --git a/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts b/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts
index 4ece5e233..7bdfdbe36 100644
--- a/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts
+++ b/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts
@@ -15,6 +15,7 @@ export class P2pCandidate {
private nbSuccess = 0
private isExcluded: boolean
private failures = 0
+ private reserved = false
constructor(
private p: PeerDTO,
@@ -35,7 +36,7 @@ export class P2pCandidate {
}
isReady() {
- return this.apiPromise.isResolved() && this.dlPromise.isResolved() && this.api && !this.isExcluded
+ return !this.reserved && this.apiPromise.isResolved() && this.dlPromise.isResolved() && this.api && !this.isExcluded
}
async waitAvailability(maxWait: number): Promise<boolean> {
@@ -64,6 +65,7 @@ export class P2pCandidate {
async downloadBlocks(count: number, from: number) {
const start = Date.now()
let error: Error|undefined
+ this.reserved = false
this.dlPromise = querablep((async () => {
// We try to download the blocks
let blocks: BlockDTO[]|null
@@ -125,6 +127,10 @@ export class P2pCandidate {
}
})())
}
+
+ reserve() {
+ this.reserved = true
+ }
}
interface RemoteAPI {
diff --git a/app/modules/crawler/lib/sync/v2/ValidatorStream.ts b/app/modules/crawler/lib/sync/v2/ValidatorStream.ts
index f4641a8e4..85c4c2d87 100644
--- a/app/modules/crawler/lib/sync/v2/ValidatorStream.ts
+++ b/app/modules/crawler/lib/sync/v2/ValidatorStream.ts
@@ -50,7 +50,7 @@ export class ValidatorStream extends Readable {
try {
const bNumber = Math.min(this.to, (i + 1) * this.syncStrategy.chunkSize - 1)
if (bNumber > maximumCacheNumber) {
- block = await this.syncStrategy.getBlock(bNumber)
+ block = await this.syncStrategy.getMilestone(bNumber)
} else {
block = await this.getBlockFromCache(bNumber)
}
diff --git a/app/modules/ws2p/lib/WS2PDocpoolPuller.ts b/app/modules/ws2p/lib/WS2PDocpoolPuller.ts
index a782b9fec..4178a7a49 100644
--- a/app/modules/ws2p/lib/WS2PDocpoolPuller.ts
+++ b/app/modules/ws2p/lib/WS2PDocpoolPuller.ts
@@ -35,6 +35,8 @@ export class WS2PDocpoolPuller {
getCurrent: async () => null,
getBlock: async () => null,
getBlocks: async () => [],
+ getMilestonesPage: async () => ({ chunkSize: 0, totalPages: 0, milestonesPerPage: 0 }),
+ getMilestones: async () => ({ chunkSize: 0, totalPages: 0, currentPage: 0, milestonesPerPage: 0, blocks: [] }),
hostName: ''
}, this.server, this.server.logger)
}
diff --git a/app/modules/ws2p/lib/WS2PRequester.ts b/app/modules/ws2p/lib/WS2PRequester.ts
index afaa67832..051a07d81 100644
--- a/app/modules/ws2p/lib/WS2PRequester.ts
+++ b/app/modules/ws2p/lib/WS2PRequester.ts
@@ -13,7 +13,8 @@
import {WS2PConnection} from "./WS2PConnection"
import {BlockDTO} from "../../../lib/dto/BlockDTO"
-import {PeerDTO} from "../../../lib/dto/PeerDTO";
+import {PeerDTO} from "../../../lib/dto/PeerDTO"
+import {HttpMilestonePage} from "../../bma/lib/dtos"
export enum WS2P_REQ {
KNOWN_PEERS,
@@ -21,7 +22,9 @@ export enum WS2P_REQ {
WOT_REQUIREMENTS_OF_PENDING,
BLOCKS_CHUNK,
BLOCK_BY_NUMBER,
- CURRENT
+ CURRENT,
+ MILESTONES_PAGE,
+ MILESTONES
}
export class WS2PRequester {
@@ -45,6 +48,14 @@ export class WS2PRequester {
return this.query(WS2P_REQ.CURRENT)
}
+ getMilestonesPage(): Promise<HttpMilestonePage> {
+ return this.query(WS2P_REQ.MILESTONES_PAGE)
+ }
+
+ getMilestones(page: number): Promise<HttpMilestonePage> {
+ return this.query(WS2P_REQ.MILESTONES, { page })
+ }
+
getBlock(number:number): Promise<BlockDTO> {
return this.query(WS2P_REQ.BLOCK_BY_NUMBER, { number })
}
diff --git a/app/modules/ws2p/lib/impl/WS2PReqMapperByServer.ts b/app/modules/ws2p/lib/impl/WS2PReqMapperByServer.ts
index ce6833205..2bc2c58f0 100644
--- a/app/modules/ws2p/lib/impl/WS2PReqMapperByServer.ts
+++ b/app/modules/ws2p/lib/impl/WS2PReqMapperByServer.ts
@@ -17,6 +17,7 @@ import {WS2PReqMapper} from "../interface/WS2PReqMapper"
import {BlockDTO} from "../../../../lib/dto/BlockDTO"
import {DBBlock} from "../../../../lib/db/DBBlock"
import {PeerDTO} from "../../../../lib/dto/PeerDTO"
+import {HttpMilestonePage} from "../../../bma/lib/dtos"
export class WS2PReqMapperByServer implements WS2PReqMapper {
@@ -45,6 +46,14 @@ export class WS2PReqMapperByServer implements WS2PReqMapper {
return (await this.server.dal.getBlocksBetween(from, from + count - 1)).map((b:DBBlock) => BlockDTO.fromJSONObject(b))
}
+ getMilestones(page: number): Promise<HttpMilestonePage> {
+ return this.server.milestones(page)
+ }
+
+ getMilestonesPage(): Promise<HttpMilestonePage> {
+ return this.server.milestones()
+ }
+
async getRequirementsOfPending(minsig: number): Promise<any> {
let identities:IdentityForRequirements[] = (await this.server.dal.idtyDAL.query(
'SELECT i.*, count(c.sig) as nbSig ' +
diff --git a/app/modules/ws2p/lib/interface/WS2PReqMapper.ts b/app/modules/ws2p/lib/interface/WS2PReqMapper.ts
index 83b37d29b..4a0ca36a7 100644
--- a/app/modules/ws2p/lib/interface/WS2PReqMapper.ts
+++ b/app/modules/ws2p/lib/interface/WS2PReqMapper.ts
@@ -14,6 +14,7 @@
import {BlockDTO} from "../../../../lib/dto/BlockDTO"
import {DBBlock} from "../../../../lib/db/DBBlock"
import {PeerDTO} from "../../../../lib/dto/PeerDTO"
+import {HttpMilestonePage} from "../../../bma/lib/dtos"
export interface WS2PReqMapper {
@@ -23,4 +24,6 @@ export interface WS2PReqMapper {
getRequirementsOfPending(minCert:number): Promise<any>
getPeer(): Promise<PeerDTO>
getKnownPeers(): Promise<PeerDTO[]>
+ getMilestones(page: number): Promise<HttpMilestonePage>
+ getMilestonesPage(): Promise<HttpMilestonePage>
}
\ No newline at end of file
diff --git a/app/modules/ws2p/lib/interface/WS2PServerMessageHandler.ts b/app/modules/ws2p/lib/interface/WS2PServerMessageHandler.ts
index 27f43f308..06c99b901 100644
--- a/app/modules/ws2p/lib/interface/WS2PServerMessageHandler.ts
+++ b/app/modules/ws2p/lib/interface/WS2PServerMessageHandler.ts
@@ -210,6 +210,18 @@ export class WS2PServerMessageHandler implements WS2PMessageHandler {
body = await this.mapper.getKnownPeers()
break;
+ case WS2P_REQ[WS2P_REQ.MILESTONES_PAGE]:
+ body = await this.mapper.getMilestonesPage()
+ break;
+
+ case WS2P_REQ[WS2P_REQ.MILESTONES]:
+ if (isNaN(data.params.page)) {
+ throw "Wrong param `page`"
+ }
+ const page:number = data.params.page
+ body = await this.mapper.getMilestones(page)
+ break;
+
default:
throw Error(WS2P_REQERROR[WS2P_REQERROR.UNKNOWN_REQUEST])
}
diff --git a/server.ts b/server.ts
index 60b4d3054..163b4a891 100644
--- a/server.ts
+++ b/server.ts
@@ -22,7 +22,7 @@ import * as stream from "stream"
import {KeyGen, randomKey} from "./app/lib/common-libs/crypto/keyring"
import {parsers} from "./app/lib/common-libs/parsers/index"
import {Cloneable} from "./app/lib/dto/Cloneable"
-import {DuniterDocument, duniterDocument2str} from "./app/lib/common-libs/constants"
+import {CommonConstants, DuniterDocument, duniterDocument2str} from "./app/lib/common-libs/constants"
import {GlobalFifoPromise} from "./app/service/GlobalFifoPromise"
import {BlockchainContext} from "./app/lib/computation/BlockchainContext"
import {BlockDTO} from "./app/lib/dto/BlockDTO"
@@ -42,6 +42,9 @@ import {DBPeer} from "./app/lib/db/DBPeer"
import {Underscore} from "./app/lib/common-libs/underscore"
import {SQLiteDriver} from "./app/lib/dal/drivers/SQLiteDriver"
import {LevelUp} from "levelup";
+import {BMAConstants} from "./app/modules/bma/lib/constants"
+import {HttpMilestonePage} from "./app/modules/bma/lib/dtos"
+import * as toJson from "./app/modules/bma/lib/tojson"
export interface HookableServer {
generatorGetJoinData: (...args:any[]) => Promise<any>
@@ -88,6 +91,7 @@ export class Server extends stream.Duplex implements HookableServer {
BlockchainService:BlockchainService
TransactionsService:TransactionService
private documentFIFO:GlobalFifoPromise
+ milestoneArray: DBBlock[] = []
constructor(home:string, private memoryOnly:boolean, private overrideConf:any) {
super({ objectMode: true })
@@ -698,4 +702,52 @@ export class Server extends stream.Duplex implements HookableServer {
resetConfigHook(): Promise<any> {
return Promise.resolve({})
}
+
+ async milestones(page?: number): Promise<HttpMilestonePage> {
+ const chunkSize = CommonConstants.SYNC_BLOCKS_CHUNK
+ const milestonesPerPage = CommonConstants.MILESTONES_PER_PAGE
+ const current = await this.dal.getCurrentBlockOrNull();
+ if (!current) {
+ return {
+ totalPages: 0,
+ chunkSize,
+ milestonesPerPage
+ }
+ }
+ const topNumber = current.number - this.conf.forksize
+ const nbMilestones = (topNumber - (topNumber % chunkSize)) / chunkSize
+ const totalPages = (nbMilestones - (nbMilestones % milestonesPerPage)) / milestonesPerPage
+ if (page === undefined) {
+ return {
+ totalPages,
+ chunkSize,
+ milestonesPerPage
+ }
+ }
+ if (page > totalPages || page <= 0) throw BMAConstants.ERRORS.INCORRECT_PAGE_NUMBER
+ while (this.milestoneArray.length < page * milestonesPerPage) {
+ const lastMilestoneNumber = this.milestoneArray.length
+ // Feed the milestones
+ const newMilestones: DBBlock[] = []
+ for (let i = 1; i <= milestonesPerPage && this.milestoneArray.length < page * milestonesPerPage; i++) {
+ const b = await this.dal.getBlock((lastMilestoneNumber + i) * chunkSize - 1)
+ if (!b) {
+ throw Error(DataErrors[DataErrors.MILESTONE_BLOCK_NOT_FOUND])
+ }
+ newMilestones.push(b)
+ }
+ // As the process is async, another call to "milestones()" maybe have already filled in the milestones
+ if (this.milestoneArray.length < page * milestonesPerPage) {
+ this.milestoneArray = this.milestoneArray.concat(newMilestones)
+ }
+ }
+ const blocks = this.milestoneArray.slice((page - 1) * milestonesPerPage, page * milestonesPerPage)
+ return {
+ totalPages,
+ chunkSize,
+ milestonesPerPage,
+ currentPage: page,
+ blocks: blocks.map(b => toJson.block(b))
+ }
+ }
}
--
GitLab