Commit 4be285c1 authored by Cédric Moreau's avatar Cédric Moreau

[enh] Add `sync-mempool` and `sync-mempool-search` commands

parent f552e8e6
......@@ -21,6 +21,12 @@ import {rawer} from "../../lib/common-libs/index"
import {PeerDTO} from "../../lib/dto/PeerDTO"
import {Buid} from "../../lib/common-libs/buid"
import {BlockDTO} from "../../lib/dto/BlockDTO"
import {NewLogger} from "../../lib/logger"
import {connect} from "./lib/connect"
import {applyMempoolRequirements, pullSandboxToLocalServer} from "./lib/sandbox"
const HOST_PATTERN = /^[^:/]+(:[0-9]{1,5})?$/
const FILE_PATTERN = /^(\/.+)$/
export const CrawlerDependency = {
duniter: {
......@@ -168,6 +174,66 @@ export const CrawlerDependency = {
}
}
}, {
name: 'sync-mempool <from>',
desc: 'Import all pending data from matching <search>',
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
const source: string = params[0]
if (!source || !(source.match(HOST_PATTERN) || source.match(FILE_PATTERN))) {
throw 'Source of sync is required. (host[:port])'
}
const logger = NewLogger()
const from: string = params[0]
const { host, port } = extractHostPort(from)
try {
const peer = PeerDTO.fromJSONObject({ endpoints: [['BASIC_MERKLED_API', host, port].join(' ')] })
const fromHost = peer.getHostPreferDNS();
const fromPort = peer.getPort();
logger.info('Looking at %s:%s...', fromHost, fromPort);
try {
const fromHost = await connect(peer, 60*1000)
await pullSandboxToLocalServer(server.conf.currency, fromHost, server, logger)
} catch (e) {
logger.error(e);
}
await server.disconnect();
} catch(e) {
logger.error(e);
throw Error("Exiting");
}
}
}, {
name: 'sync-mempool-search <from> <search>',
desc: 'Import all pending data from matching <search>',
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
const source: string = params[0]
const search: string = params[1]
if (!source || !(source.match(HOST_PATTERN) || source.match(FILE_PATTERN))) {
throw 'Source of sync is required. (host[:port])'
}
const logger = NewLogger()
const from: string = params[0]
const { host, port } = extractHostPort(from)
try {
const peer = PeerDTO.fromJSONObject({ endpoints: [['BASIC_MERKLED_API', host, port].join(' ')] })
const fromHost = peer.getHostPreferDNS();
const fromPort = peer.getPort();
logger.info('Looking at %s:%s...', fromHost, fromPort);
try {
const fromHost = await connect(peer)
const res = await fromHost.getRequirements(search)
await applyMempoolRequirements(server.conf.currency, res, server, logger)
} catch (e) {
logger.error(e);
}
await server.disconnect();
} catch(e) {
logger.error(e);
throw Error("Exiting");
}
}
}, {
name: 'forward <number> <fromHost> <fromPort> <toHost> <toPort>',
desc: 'Forward existing block <number> from a host to another',
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
......@@ -357,3 +423,13 @@ export const CrawlerDependency = {
}]
}
}
function extractHostPort(source: string) {
const sp = source.split(':')
const onHost = sp[0]
const onPort = parseInt(sp[1] ? sp[1] : '443') // Defaults to 443
return {
host: onHost,
port: onPort,
}
}
\ No newline at end of file
......@@ -22,7 +22,7 @@ export class Contacter {
options:{ timeout:number }
fullyQualifiedHost:string
constructor(private host:string, private port:number, opts:any = {}) {
constructor(public host:string, public port:number, opts:any = {}) {
this.options = {
timeout: (opts && opts.timeout) || CrawlerConstants.DEFAULT_TIMEOUT
}
......
......@@ -12,75 +12,58 @@
// GNU Affero General Public License for more details.
"use strict";
import {Contacter} from "./contacter"
import {Server} from "../../../../server"
import {rawer} from "../../../lib/common-libs/index"
import {parsers} from "../../../lib/common-libs/parsers/index"
import {HttpRequirements} from "../../bma/lib/dtos"
export const pullSandbox = async (currency:string, fromHost:string, fromPort:number, toHost:string, toPort:number, logger:any) => {
const from = new Contacter(fromHost, fromPort);
const to = new Contacter(toHost, toPort);
export const pullSandboxToLocalServer = async (currency:string, fromHost: {
getRequirementsPending(minCert?: number): Promise<any>
host?: string
port?: number
}, toServer:Server, logger:any, watcher:any = null, nbCertsMin = 1, notify = true) => {
let res
try {
res = await from.getRequirementsPending(1)
res = await fromHost.getRequirementsPending(nbCertsMin || 1)
} catch (e) {
// Silent error
logger && logger.trace('Sandbox pulling: could not fetch requirements on %s', [fromHost, fromPort].join(':'))
watcher && watcher.writeStatus('Sandbox pulling: could not fetch requirements on %s', [fromHost.host, fromHost.port].join(':'))
}
if (res) {
const docs = getDocumentsTree(currency, res)
for (const identity of docs.identities) {
await submitIdentity(identity, to)
}
for (const certification of docs.certifications) {
await submitCertification(certification, to)
}
for (const membership of docs.memberships) {
await submitMembership(membership, to)
}
await applyMempoolRequirements(currency, res, toServer)
}
}
export const pullSandboxToLocalServer = async (currency:string, fromHost:any, toServer:Server, logger:any, watcher:any = null, nbCertsMin = 1, notify = true) => {
let res
try {
res = await fromHost.getRequirementsPending(nbCertsMin || 1)
} catch (e) {
watcher && watcher.writeStatus('Sandbox pulling: could not fetch requirements on %s', [fromHost.host, fromHost.port].join(':'))
}
export async function applyMempoolRequirements(currency: string, res: HttpRequirements, toServer: Server, notify = true, logger?: any, watcher?: any) {
if (res) {
const docs = getDocumentsTree(currency, res)
const docs = getDocumentsTree(currency, res)
for (let i = 0; i < docs.identities.length; i++) {
const idty = docs.identities[i];
watcher && watcher.writeStatus('Identity ' + (i+1) + '/' + docs.identities.length)
await submitIdentityToServer(idty, toServer, notify, logger)
}
for (let i = 0; i < docs.identities.length; i++) {
const idty = docs.identities[i];
watcher && watcher.writeStatus('Identity ' + (i+1) + '/' + docs.identities.length)
await submitIdentityToServer(idty, toServer, notify, logger)
}
for (let i = 0; i < docs.revocations.length; i++) {
const idty = docs.revocations[i];
watcher && watcher.writeStatus('Revocation ' + (i+1) + '/' + docs.revocations.length)
await submitRevocationToServer(idty, toServer, notify, logger)
}
for (let i = 0; i < docs.revocations.length; i++) {
const idty = docs.revocations[i];
watcher && watcher.writeStatus('Revocation ' + (i+1) + '/' + docs.revocations.length)
await submitRevocationToServer(idty, toServer, notify, logger)
}
for (let i = 0; i < docs.certifications.length; i++) {
const cert = docs.certifications[i];
watcher && watcher.writeStatus('Certification ' + (i+1) + '/' + docs.certifications.length)
await submitCertificationToServer(cert, toServer, notify, logger)
}
for (let i = 0; i < docs.certifications.length; i++) {
const cert = docs.certifications[i];
watcher && watcher.writeStatus('Certification ' + (i+1) + '/' + docs.certifications.length)
await submitCertificationToServer(cert, toServer, notify, logger)
}
for (let i = 0; i < docs.memberships.length; i++) {
const ms = docs.memberships[i];
watcher && watcher.writeStatus('Membership ' + (i+1) + '/' + docs.memberships.length)
await submitMembershipToServer(ms, toServer, notify, logger)
}
for (let i = 0; i < docs.memberships.length; i++) {
const ms = docs.memberships[i];
watcher && watcher.writeStatus('Membership ' + (i+1) + '/' + docs.memberships.length)
await submitMembershipToServer(ms, toServer, notify, logger)
}
}
function getDocumentsTree(currency:string, res:any) {
function getDocumentsTree(currency:string, res:HttpRequirements) {
const documents:any = {
identities: [],
certifications: [],
......@@ -136,33 +119,6 @@ function getDocumentsTree(currency:string, res:any) {
return documents
}
async function submitIdentity(idty:any, to:any, logger:any = null) {
try {
await to.postIdentity(idty)
logger && logger.trace('Sandbox pulling: success with identity \'%s\'', idty.uid)
} catch (e) {
// Silent error
}
}
async function submitCertification(cert:any, to:any, logger:any = null) {
try {
await to.postCert(cert)
logger && logger.trace('Sandbox pulling: success with cert key %s => %s', cert.from.substr(0, 6), cert.idty_uid)
} catch (e) {
// Silent error
}
}
async function submitMembership(ms:any, to:any, logger:any = null) {
try {
await to.postRenew(ms)
logger && logger.trace('Sandbox pulling: success with membership \'%s\'', ms.uid)
} catch (e) {
// Silent error
}
}
async function submitIdentityToServer(idty:any, toServer:any, notify:boolean, logger:any) {
try {
const obj = parsers.parseIdentity.syncWrite(idty)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment