Skip to content
Snippets Groups Projects
Commit 4be285c1 authored by Cédric Moreau's avatar Cédric Moreau
Browse files

[enh] Add `sync-mempool` and `sync-mempool-search` commands

parent f552e8e6
No related branches found
No related tags found
No related merge requests found
...@@ -21,6 +21,12 @@ import {rawer} from "../../lib/common-libs/index" ...@@ -21,6 +21,12 @@ import {rawer} from "../../lib/common-libs/index"
import {PeerDTO} from "../../lib/dto/PeerDTO" import {PeerDTO} from "../../lib/dto/PeerDTO"
import {Buid} from "../../lib/common-libs/buid" import {Buid} from "../../lib/common-libs/buid"
import {BlockDTO} from "../../lib/dto/BlockDTO" import {BlockDTO} from "../../lib/dto/BlockDTO"
import {NewLogger} from "../../lib/logger"
import {connect} from "./lib/connect"
import {applyMempoolRequirements, pullSandboxToLocalServer} from "./lib/sandbox"
const HOST_PATTERN = /^[^:/]+(:[0-9]{1,5})?$/
const FILE_PATTERN = /^(\/.+)$/
export const CrawlerDependency = { export const CrawlerDependency = {
duniter: { duniter: {
...@@ -168,6 +174,66 @@ export const CrawlerDependency = { ...@@ -168,6 +174,66 @@ export const CrawlerDependency = {
} }
} }
}, { }, {
name: 'sync-mempool <from>',
desc: 'Import all pending data from matching <search>',
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
const source: string = params[0]
if (!source || !(source.match(HOST_PATTERN) || source.match(FILE_PATTERN))) {
throw 'Source of sync is required. (host[:port])'
}
const logger = NewLogger()
const from: string = params[0]
const { host, port } = extractHostPort(from)
try {
const peer = PeerDTO.fromJSONObject({ endpoints: [['BASIC_MERKLED_API', host, port].join(' ')] })
const fromHost = peer.getHostPreferDNS();
const fromPort = peer.getPort();
logger.info('Looking at %s:%s...', fromHost, fromPort);
try {
const fromHost = await connect(peer, 60*1000)
await pullSandboxToLocalServer(server.conf.currency, fromHost, server, logger)
} catch (e) {
logger.error(e);
}
await server.disconnect();
} catch(e) {
logger.error(e);
throw Error("Exiting");
}
}
}, {
name: 'sync-mempool-search <from> <search>',
desc: 'Import all pending data from matching <search>',
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
const source: string = params[0]
const search: string = params[1]
if (!source || !(source.match(HOST_PATTERN) || source.match(FILE_PATTERN))) {
throw 'Source of sync is required. (host[:port])'
}
const logger = NewLogger()
const from: string = params[0]
const { host, port } = extractHostPort(from)
try {
const peer = PeerDTO.fromJSONObject({ endpoints: [['BASIC_MERKLED_API', host, port].join(' ')] })
const fromHost = peer.getHostPreferDNS();
const fromPort = peer.getPort();
logger.info('Looking at %s:%s...', fromHost, fromPort);
try {
const fromHost = await connect(peer)
const res = await fromHost.getRequirements(search)
await applyMempoolRequirements(server.conf.currency, res, server, logger)
} catch (e) {
logger.error(e);
}
await server.disconnect();
} catch(e) {
logger.error(e);
throw Error("Exiting");
}
}
}, {
name: 'forward <number> <fromHost> <fromPort> <toHost> <toPort>', name: 'forward <number> <fromHost> <fromPort> <toHost> <toPort>',
desc: 'Forward existing block <number> from a host to another', desc: 'Forward existing block <number> from a host to another',
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => { onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
...@@ -357,3 +423,13 @@ export const CrawlerDependency = { ...@@ -357,3 +423,13 @@ export const CrawlerDependency = {
}] }]
} }
} }
function extractHostPort(source: string) {
const sp = source.split(':')
const onHost = sp[0]
const onPort = parseInt(sp[1] ? sp[1] : '443') // Defaults to 443
return {
host: onHost,
port: onPort,
}
}
\ No newline at end of file
...@@ -22,7 +22,7 @@ export class Contacter { ...@@ -22,7 +22,7 @@ export class Contacter {
options:{ timeout:number } options:{ timeout:number }
fullyQualifiedHost:string fullyQualifiedHost:string
constructor(private host:string, private port:number, opts:any = {}) { constructor(public host:string, public port:number, opts:any = {}) {
this.options = { this.options = {
timeout: (opts && opts.timeout) || CrawlerConstants.DEFAULT_TIMEOUT timeout: (opts && opts.timeout) || CrawlerConstants.DEFAULT_TIMEOUT
} }
......
...@@ -12,46 +12,30 @@ ...@@ -12,46 +12,30 @@
// GNU Affero General Public License for more details. // GNU Affero General Public License for more details.
"use strict"; "use strict";
import {Contacter} from "./contacter"
import {Server} from "../../../../server" import {Server} from "../../../../server"
import {rawer} from "../../../lib/common-libs/index" import {rawer} from "../../../lib/common-libs/index"
import {parsers} from "../../../lib/common-libs/parsers/index" import {parsers} from "../../../lib/common-libs/parsers/index"
import {HttpRequirements} from "../../bma/lib/dtos"
export const pullSandbox = async (currency:string, fromHost:string, fromPort:number, toHost:string, toPort:number, logger:any) => { export const pullSandboxToLocalServer = async (currency:string, fromHost: {
const from = new Contacter(fromHost, fromPort); getRequirementsPending(minCert?: number): Promise<any>
const to = new Contacter(toHost, toPort); host?: string
port?: number
}, toServer:Server, logger:any, watcher:any = null, nbCertsMin = 1, notify = true) => {
let res let res
try { try {
res = await from.getRequirementsPending(1) res = await fromHost.getRequirementsPending(nbCertsMin || 1)
} catch (e) { } catch (e) {
// Silent error watcher && watcher.writeStatus('Sandbox pulling: could not fetch requirements on %s', [fromHost.host, fromHost.port].join(':'))
logger && logger.trace('Sandbox pulling: could not fetch requirements on %s', [fromHost, fromPort].join(':'))
} }
if (res) { if (res) {
const docs = getDocumentsTree(currency, res) await applyMempoolRequirements(currency, res, toServer)
for (const identity of docs.identities) {
await submitIdentity(identity, to)
}
for (const certification of docs.certifications) {
await submitCertification(certification, to)
}
for (const membership of docs.memberships) {
await submitMembership(membership, to)
}
} }
} }
export const pullSandboxToLocalServer = async (currency:string, fromHost:any, toServer:Server, logger:any, watcher:any = null, nbCertsMin = 1, notify = true) => { export async function applyMempoolRequirements(currency: string, res: HttpRequirements, toServer: Server, notify = true, logger?: any, watcher?: any) {
let res
try {
res = await fromHost.getRequirementsPending(nbCertsMin || 1)
} catch (e) {
watcher && watcher.writeStatus('Sandbox pulling: could not fetch requirements on %s', [fromHost.host, fromHost.port].join(':'))
}
if (res) {
const docs = getDocumentsTree(currency, res) const docs = getDocumentsTree(currency, res)
for (let i = 0; i < docs.identities.length; i++) { for (let i = 0; i < docs.identities.length; i++) {
...@@ -78,9 +62,8 @@ export const pullSandboxToLocalServer = async (currency:string, fromHost:any, to ...@@ -78,9 +62,8 @@ export const pullSandboxToLocalServer = async (currency:string, fromHost:any, to
await submitMembershipToServer(ms, toServer, notify, logger) await submitMembershipToServer(ms, toServer, notify, logger)
} }
} }
}
function getDocumentsTree(currency:string, res:any) { function getDocumentsTree(currency:string, res:HttpRequirements) {
const documents:any = { const documents:any = {
identities: [], identities: [],
certifications: [], certifications: [],
...@@ -136,33 +119,6 @@ function getDocumentsTree(currency:string, res:any) { ...@@ -136,33 +119,6 @@ function getDocumentsTree(currency:string, res:any) {
return documents return documents
} }
async function submitIdentity(idty:any, to:any, logger:any = null) {
try {
await to.postIdentity(idty)
logger && logger.trace('Sandbox pulling: success with identity \'%s\'', idty.uid)
} catch (e) {
// Silent error
}
}
async function submitCertification(cert:any, to:any, logger:any = null) {
try {
await to.postCert(cert)
logger && logger.trace('Sandbox pulling: success with cert key %s => %s', cert.from.substr(0, 6), cert.idty_uid)
} catch (e) {
// Silent error
}
}
async function submitMembership(ms:any, to:any, logger:any = null) {
try {
await to.postRenew(ms)
logger && logger.trace('Sandbox pulling: success with membership \'%s\'', ms.uid)
} catch (e) {
// Silent error
}
}
async function submitIdentityToServer(idty:any, toServer:any, notify:boolean, logger:any) { async function submitIdentityToServer(idty:any, toServer:any, notify:boolean, logger:any) {
try { try {
const obj = parsers.parseIdentity.syncWrite(idty) const obj = parsers.parseIdentity.syncWrite(idty)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment