Commit 3be85224 authored by Cédric Moreau's avatar Cédric Moreau

[fix] Sync was not using P2P correctly

parent d37aa060
Pipeline #3715 failed with stages
in 9 minutes and 58 seconds
......@@ -316,8 +316,7 @@ export const CommonConstants = {
DEFAULT_NON_WOT_PEERS_LIMIT: 100, // Number of non-wot peers accepted in our peer document pool
REJECT_WAIT_FOR_AVAILABLE_NODES_IN_SYNC_AFTER: 20000, // Reject after 20 seconds without any change
REJECT_WAIT_FOR_AVAILABLE_NODES_IN_SYNC_MAX_FAILS: 5, // Maximum number of rejections of waiting for an available node
WAIT_P2P_CANDIDATE_HEARTBEAT: 5000, // Wait X seconds for a node to answer about its state
MAX_READING_SLOTS_FOR_FILE_SYNC: 20, // Number of file reading in parallel
}
......
......@@ -72,8 +72,8 @@ export function randomKey() {
}
const keypair = nacl.sign.keyPair.fromSeed(byteseed)
return new Key(
Base58encode(keypair.publicKey),
Base58encode(keypair.secretKey)
Base58encode(new Buffer(keypair.publicKey)),
Base58encode(new Buffer(keypair.secretKey))
)
}
......
......@@ -4,9 +4,7 @@ export enum DataErrors {
INVALID_LEVELDB_IINDEX_DATA_TO_BE_KICKED,
IDENTITY_UID_NOT_FOUND,
INVALID_TRIMMABLE_DATA,
SYNC_FAST_MEM_ERROR_DURING_INJECTION,
CANNOT_GET_VALIDATION_BLOCK_FROM_REMOTE,
REJECT_WAIT_FOR_AVAILABLE_NODES_BUT_CONTINUE,
NO_NODE_FOUND_TO_DOWNLOAD_CHUNK,
WRONG_CURRENCY_DETECTED,
NO_PEERING_AVAILABLE_FOR_SYNC,
......@@ -20,7 +18,6 @@ export enum DataErrors {
NEGATIVE_BALANCE,
BLOCK_WASNT_COMMITTED,
CANNOT_ARCHIVE_CHUNK_WRONG_SIZE,
CORRUPTED_DATABASE,
BLOCKCHAIN_NOT_INITIALIZED_YET,
CANNOT_DETERMINATE_MEMBERSHIP_AGE,
CANNOT_DETERMINATE_IDENTITY_AGE,
......
......@@ -19,7 +19,7 @@ export interface ProgramOptions {
notrim?: boolean
nosbx?: boolean
nopeers?: boolean
p2psync?: boolean
nop2p?: boolean
syncTrace?: string
isSync: boolean
noSources: boolean
......@@ -36,7 +36,7 @@ export const cliprogram: ProgramOptions = {
notrim: opts.notrim,
nosbx: opts.nosbx,
nopeers: opts.nopeers,
p2psync: opts.p2psync,
nop2p: opts.nop2p,
noSources: !!opts.nosources,
syncTrace: opts['sync-trace'],
isSync: opts._[0] === 'sync',
......
......@@ -16,3 +16,9 @@ export function newRejectTimeoutPromise(timeout: number) {
setTimeout(rej, timeout)
})
}
export function newResolveTimeoutPromise<T>(timeout: number, value: T): Promise<T> {
return new Promise(res => {
setTimeout(() => res(value), timeout)
})
}
......@@ -76,7 +76,7 @@ export const CrawlerDependency = {
{ value: '--nocautious', desc: 'Do not check blocks validity during sync.'},
{ value: '--cautious', desc: 'Check blocks validity during sync (overrides --nocautious option).'},
{ value: '--nopeers', desc: 'Do not retrieve peers during sync.'},
{ value: '--p2psync', desc: 'Force P2P downloading of blocs during sync.'},
{ value: '--nop2p', desc: 'Disables P2P downloading of blocs during sync.'},
{ value: '--nosources', desc: 'Do not parse sources (UD, TX) during sync (debug purposes).'},
{ value: '--nosbx', desc: 'Do not retrieve sandboxes during sync.'},
{ value: '--onlypeers', desc: 'Will only try to sync peers.'},
......@@ -86,8 +86,8 @@ export const CrawlerDependency = {
],
cli: [{
name: 'sync [source] [to] [currency]',
desc: 'Synchronize blockchain from a remote Duniter node',
name: 'sync [source] [to]',
desc: 'Synchronize blockchain from a remote Duniter node. [source] is [host][:port]. [to] defaults to remote current block number.',
preventIfRunning: true,
onConfiguredExecute: async (server:Server) => {
await server.resetData();
......
......@@ -15,5 +15,4 @@ export abstract class ASyncDownloader implements ISyncDownloader {
abstract maxSlots: number
abstract getChunk(i: number): Promise<BlockDTO[]>
abstract getTimesToAnswer(): Promise<{ ttas: number[] }[]>
}
......@@ -43,8 +43,4 @@ export class FsSyncDownloader extends ASyncDownloader implements ISyncDownloader
get maxSlots(): number {
return CommonConstants.MAX_READING_SLOTS_FOR_FILE_SYNC
}
async getTimesToAnswer(): Promise<{ ttas: number[] }[]> {
return [{ ttas: this.ttas }]
}
}
......@@ -5,5 +5,4 @@ export interface ISyncDownloader {
getBlock(number: number): Promise<BlockDTO|null>
maxSlots: number
chunkSize: number
getTimesToAnswer(): Promise<{ ttas: number[] }[]>
}
......@@ -185,7 +185,8 @@ export class RemoteSynchronizer extends AbstractSynchronizer {
// Peers (just for P2P download)
//=======
let peers:(JSONDBPeer|null)[] = [];
if (cliprogram.p2psync) {
const p2psync = !cliprogram.nop2p
if (p2psync) {
this.watcher.writeStatus('Peers...');
peers = await this.node.getPeers()
}
......
import {Querable, querablep} from "../../../../../lib/common-libs/querable"
import {PeerDTO} from "../../../../../lib/dto/PeerDTO"
import {Keypair} from "../../../../../lib/dto/ConfDTO"
import {RemoteSynchronizer} from "../RemoteSynchronizer"
import {IRemoteContacter} from "../IRemoteContacter"
import {BlockDTO} from "../../../../../lib/dto/BlockDTO"
import {newResolveTimeoutPromise} from "../../../../../lib/common-libs/timeout-promise"
export class P2pCandidate {
private readonly apiPromise: Querable<any>
private dlPromise: Querable<BlockDTO[]|null>
private readonly responseTimes: number[] = []
private api: IRemoteContacter|null|undefined
private nbSuccess = 0
private isExcluded: boolean
private failures = 0
constructor(
private p: PeerDTO,
private keypair: Keypair,
private logger: any
) {
this.apiPromise = this.initAPI()
this.dlPromise = querablep(Promise.resolve(null))
}
addFailure() {
this.failures++
if (this.failures >= 5 && !this.isExcluded) {
this.isExcluded = true
this.logger.warn('Excluding node %s as it returned unchainable chunks %s times', this.hostName, this.failures)
}
}
isReady() {
return this.apiPromise.isResolved() && this.dlPromise.isResolved() && this.api && !this.isExcluded
}
async waitAvailability(maxWait: number): Promise<boolean> {
return Promise.race([
// Wait for availablity
(async () => !this.isExcluded
&& (this.apiPromise.isRejected() ? await newResolveTimeoutPromise(maxWait, false) : !!(await this.apiPromise))
&& (this.dlPromise.isRejected() ? await newResolveTimeoutPromise(maxWait, false) : !!(await this.dlPromise)))(),
// Maximum wait trigger
newResolveTimeoutPromise(maxWait, false)
])
}
hasAvailableApi() {
return !!this.api
}
avgResponseTime() {
return this.responseTimes.reduce((sum, rt) => sum + rt, 0) / this.responseTimes.length
}
get hostName() {
return (this.api && this.api.hostName) || 'NO_API'
}
async downloadBlocks(count: number, from: number) {
const start = Date.now()
let error: Error|undefined
this.dlPromise = querablep((async () => {
// We try to download the blocks
let blocks: BlockDTO[]|null
try {
blocks = await (this.api as IRemoteContacter).getBlocks(count, from)
}
catch (e) {
// Unfortunately this can fail
blocks = null
error = e
}
this.responseTimes.push(Date.now() - start);
// Only keep a flow of 5 ttas for the node
if (this.responseTimes.length > 5) this.responseTimes.shift()
this.nbSuccess++
if (error) {
throw error
}
return blocks
})())
return this.dlPromise
}
private getRemoteAPIs() {
const bmaAPI = this.p.getBMA()
const ws2pAPI = this.p.getFirstNonTorWS2P()
const apis: RemoteAPI[] = []
const bmaHost = bmaAPI.dns || bmaAPI.ipv4 || bmaAPI.ipv6
if (bmaAPI.port && bmaHost) {
apis.push({
isBMA: true,
port: bmaAPI.port,
host: bmaHost
})
}
if (ws2pAPI) {
apis.push({
isWS2P: true,
host: ws2pAPI.host,
port: ws2pAPI.port,
path: ws2pAPI.path,
})
}
return apis
}
private initAPI() {
return querablep((async (): Promise<IRemoteContacter|null> => {
try {
const apis = this.getRemoteAPIs()
const syncApi = await RemoteSynchronizer.getSyncAPI(apis, this.keypair)
if ((syncApi && syncApi.api.hostName || '').match(/^(localhost|192|127)/)) {
return null
}
this.api = syncApi.api
return syncApi.api
} catch (e) {
return null
}
})())
}
}
interface RemoteAPI {
isBMA?: boolean
isWS2P?: boolean
host: string
port: number
path?: string
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment