diff --git a/app/modules/crawler/index.ts b/app/modules/crawler/index.ts index 1afe22ac6dc1b87e9c357df65bf0af403d12a788..d43d3e748e0fd36b1a7ac5b8aa969ca252232399 100644 --- a/app/modules/crawler/index.ts +++ b/app/modules/crawler/index.ts @@ -31,6 +31,12 @@ import {DataErrors} from "../../lib/common-libs/errors" import {NewLogger} from "../../lib/logger" import {CrawlerConstants} from "./lib/constants" import {ExitCodes} from "../../lib/common-libs/exit-codes" +import {connect} from "./lib/connect" +import {BMARemoteContacter} from "./lib/sync/BMARemoteContacter" +import {applyMempoolRequirements, pullSandboxToLocalServer} from "./lib/sandbox" + +const HOST_PATTERN = /^[^:/]+(:[0-9]{1,5})?$/ +const FILE_PATTERN = /^(\/.+)$/ export const CrawlerDependency = { duniter: { @@ -100,8 +106,6 @@ export const CrawlerDependency = { onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any): Promise<any> => { const source = params[0] const to = params[1] - const HOST_PATTERN = /^[^:/]+(:[0-9]{1,5})?$/ - const FILE_PATTERN = /^(\/.+)$/ if (!source || !(source.match(HOST_PATTERN) || source.match(FILE_PATTERN))) { throw 'Source of sync is required. (either a host:port or a file path)' } @@ -216,6 +220,67 @@ export const CrawlerDependency = { } } }, { + name: 'sync-mempool <from>', + desc: 'Import all pending data from matching <search>', + onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => { + const source: string = params[0] + if (!source || !(source.match(HOST_PATTERN) || source.match(FILE_PATTERN))) { + throw 'Source of sync is required. (host[:port])' + } + const logger = NewLogger() + const from: string = params[0] + const { host, port } = extractHostPort(from) + try { + const peer = PeerDTO.fromJSONObject({ endpoints: [['BASIC_MERKLED_API', host, port].join(' ')] }) + const fromHost = peer.getHostPreferDNS(); + const fromPort = peer.getPort(); + logger.info('Looking at %s:%s...', fromHost, fromPort); + try { + const fromHost = await connect(peer, 60*1000) + const api = new BMARemoteContacter(fromHost) + await pullSandboxToLocalServer(server.conf.currency, api, server, logger) + } catch (e) { + logger.error(e); + } + + await server.disconnect(); + } catch(e) { + logger.error(e); + throw Error("Exiting"); + } + } + }, { + name: 'sync-mempool-search <from> <search>', + desc: 'Import all pending data from matching <search>', + onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => { + const source: string = params[0] + const search: string = params[1] + if (!source || !(source.match(HOST_PATTERN) || source.match(FILE_PATTERN))) { + throw 'Source of sync is required. (host[:port])' + } + const logger = NewLogger() + const from: string = params[0] + const { host, port } = extractHostPort(from) + try { + const peer = PeerDTO.fromJSONObject({ endpoints: [['BASIC_MERKLED_API', host, port].join(' ')] }) + const fromHost = peer.getHostPreferDNS(); + const fromPort = peer.getPort(); + logger.info('Looking at %s:%s...', fromHost, fromPort); + try { + const fromHost = await connect(peer) + const res = await fromHost.getRequirements(search) + await applyMempoolRequirements(server.conf.currency, res, server, logger) + } catch (e) { + logger.error(e); + } + + await server.disconnect(); + } catch(e) { + logger.error(e); + throw Error("Exiting"); + } + } + }, { name: 'forward <number> <fromHost> <fromPort> <toHost> <toPort>', desc: 'Forward existing block <number> from a host to another', onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => { @@ -405,3 +470,13 @@ export const CrawlerDependency = { }] } } + +function extractHostPort(source: string) { + const sp = source.split(':') + const onHost = sp[0] + const onPort = parseInt(sp[1] ? sp[1] : '443') // Defaults to 443 + return { + host: onHost, + port: onPort, + } +} \ No newline at end of file diff --git a/app/modules/crawler/lib/sandbox.ts b/app/modules/crawler/lib/sandbox.ts index 5a92e8a87c7b0e15ae345dacada964d78cc37476..7f165b2fdbe4ac15a718120b904a26247bb3e6fb 100644 --- a/app/modules/crawler/lib/sandbox.ts +++ b/app/modules/crawler/lib/sandbox.ts @@ -12,37 +12,12 @@ // GNU Affero General Public License for more details. "use strict"; -import {Contacter} from "./contacter" import {Server} from "../../../../server" import {rawer} from "../../../lib/common-libs/index" import {parsers} from "../../../lib/common-libs/parsers/index" -import {IRemoteContacter} from "./sync/IRemoteContacter"; - -export const pullSandbox = async (currency:string, fromHost:string, fromPort:number, toHost:string, toPort:number, logger:any) => { - const from = new Contacter(fromHost, fromPort); - const to = new Contacter(toHost, toPort); - - let res - try { - res = await from.getRequirementsPending(1) - } catch (e) { - // Silent error - logger && logger.trace('Sandbox pulling: could not fetch requirements on %s', [fromHost, fromPort].join(':')) - } - - if (res) { - const docs = getDocumentsTree(currency, res) - for (const identity of docs.identities) { - await submitIdentity(identity, to) - } - for (const certification of docs.certifications) { - await submitCertification(certification, to) - } - for (const membership of docs.memberships) { - await submitMembership(membership, to) - } - } -} +import {IRemoteContacter} from "./sync/IRemoteContacter" +import {HttpRequirements} from "../../bma/lib/dtos" +import {Watcher} from "./sync/Watcher" export const pullSandboxToLocalServer = async (currency:string, fromHost:IRemoteContacter, toServer:Server, logger:any, watcher:any = null, nbCertsMin = 1, notify = true) => { let res @@ -53,44 +28,49 @@ export const pullSandboxToLocalServer = async (currency:string, fromHost:IRemote } if (res) { - const docs = getDocumentsTree(currency, res) + await applyMempoolRequirements(currency, res, toServer) + } +} - let t = 0 - let T = docs.identities.length + docs.certifications.length + docs.revocations.length + docs.memberships.length +export async function applyMempoolRequirements(currency: string, res: HttpRequirements, toServer: Server, notify = true, logger?: any, watcher?: Watcher) { - for (let i = 0; i < docs.identities.length; i++) { - const idty = docs.identities[i]; - watcher && watcher.writeStatus('Identity ' + (i+1) + '/' + docs.identities.length) - watcher && watcher.sbxPercent((t++) / T * 100) - await submitIdentityToServer(idty, toServer, notify, logger) - } + const docs = getDocumentsTree(currency, res) - for (let i = 0; i < docs.revocations.length; i++) { - const idty = docs.revocations[i]; - watcher && watcher.writeStatus('Revocation ' + (i+1) + '/' + docs.revocations.length) - watcher && watcher.sbxPercent((t++) / T * 100) - await submitRevocationToServer(idty, toServer, notify, logger) - } + let t = 0 + let T = docs.identities.length + docs.certifications.length + docs.revocations.length + docs.memberships.length - for (let i = 0; i < docs.certifications.length; i++) { - const cert = docs.certifications[i]; - watcher && watcher.writeStatus('Certification ' + (i+1) + '/' + docs.certifications.length) - watcher && watcher.sbxPercent((t++) / T * 100) - await submitCertificationToServer(cert, toServer, notify, logger) - } + for (let i = 0; i < docs.identities.length; i++) { + const idty = docs.identities[i]; + watcher && watcher.writeStatus('Identity ' + (i+1) + '/' + docs.identities.length) + watcher && watcher.sbxPercent((t++) / T * 100) + await submitIdentityToServer(idty, toServer, notify, logger) + } - for (let i = 0; i < docs.memberships.length; i++) { - const ms = docs.memberships[i]; - watcher && watcher.writeStatus('Membership ' + (i+1) + '/' + docs.memberships.length) - watcher && watcher.sbxPercent((t++) / T * 100) - await submitMembershipToServer(ms, toServer, notify, logger) - } + for (let i = 0; i < docs.revocations.length; i++) { + const idty = docs.revocations[i]; + watcher && watcher.writeStatus('Revocation ' + (i+1) + '/' + docs.revocations.length) + watcher && watcher.sbxPercent((t++) / T * 100) + await submitRevocationToServer(idty, toServer, notify, logger) + } - watcher && watcher.sbxPercent(100) + for (let i = 0; i < docs.certifications.length; i++) { + const cert = docs.certifications[i]; + watcher && watcher.writeStatus('Certification ' + (i+1) + '/' + docs.certifications.length) + watcher && watcher.sbxPercent((t++) / T * 100) + await submitCertificationToServer(cert, toServer, notify, logger) } + + for (let i = 0; i < docs.memberships.length; i++) { + const ms = docs.memberships[i]; + watcher && watcher.writeStatus('Membership ' + (i+1) + '/' + docs.memberships.length) + watcher && watcher.sbxPercent((t++) / T * 100) + await submitMembershipToServer(ms, toServer, notify, logger) + } + + watcher && watcher.sbxPercent(100) } -function getDocumentsTree(currency:string, res:any) { +function getDocumentsTree(currency:string, res:HttpRequirements) { const documents:any = { identities: [], certifications: [], @@ -146,33 +126,6 @@ function getDocumentsTree(currency:string, res:any) { return documents } -async function submitIdentity(idty:any, to:any, logger:any = null) { - try { - await to.postIdentity(idty) - logger && logger.trace('Sandbox pulling: success with identity \'%s\'', idty.uid) - } catch (e) { - // Silent error - } -} - -async function submitCertification(cert:any, to:any, logger:any = null) { - try { - await to.postCert(cert) - logger && logger.trace('Sandbox pulling: success with cert key %s => %s', cert.from.substr(0, 6), cert.idty_uid) - } catch (e) { - // Silent error - } -} - -async function submitMembership(ms:any, to:any, logger:any = null) { - try { - await to.postRenew(ms) - logger && logger.trace('Sandbox pulling: success with membership \'%s\'', ms.uid) - } catch (e) { - // Silent error - } -} - async function submitIdentityToServer(idty:any, toServer:any, notify:boolean, logger:any) { try { const obj = parsers.parseIdentity.syncWrite(idty)