diff --git a/app/modules/crawler/index.ts b/app/modules/crawler/index.ts
index 8eff8439a3633275c32dcc25c4b45f99524e6ad7..cec5c2fe5e7db9867be5be055889124f399d0d77 100644
--- a/app/modules/crawler/index.ts
+++ b/app/modules/crawler/index.ts
@@ -21,6 +21,12 @@ import {rawer} from "../../lib/common-libs/index"
import {PeerDTO} from "../../lib/dto/PeerDTO"
import {Buid} from "../../lib/common-libs/buid"
import {BlockDTO} from "../../lib/dto/BlockDTO"
+import {NewLogger} from "../../lib/logger"
+import {connect} from "./lib/connect"
+import {applyMempoolRequirements, pullSandboxToLocalServer} from "./lib/sandbox"
+
+const HOST_PATTERN = /^[^:/]+(:[0-9]{1,5})?$/
+const FILE_PATTERN = /^(\/.+)$/
export const CrawlerDependency = {
duniter: {
@@ -168,6 +174,66 @@ export const CrawlerDependency = {
}
}
}, {
+ name: 'sync-mempool <from>',
+ desc: 'Import all pending data from matching <search>',
+ onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
+ const source: string = params[0]
+ if (!source || !(source.match(HOST_PATTERN) || source.match(FILE_PATTERN))) {
+ throw 'Source of sync is required. (host[:port])'
+ }
+ const logger = NewLogger()
+ const from: string = params[0]
+ const { host, port } = extractHostPort(from)
+ try {
+ const peer = PeerDTO.fromJSONObject({ endpoints: [['BASIC_MERKLED_API', host, port].join(' ')] })
+ const fromHost = peer.getHostPreferDNS();
+ const fromPort = peer.getPort();
+ logger.info('Looking at %s:%s...', fromHost, fromPort);
+ try {
+ const fromHost = await connect(peer, 60*1000)
+ await pullSandboxToLocalServer(server.conf.currency, fromHost, server, logger)
+ } catch (e) {
+ logger.error(e);
+ }
+
+ await server.disconnect();
+ } catch(e) {
+ logger.error(e);
+ throw Error("Exiting");
+ }
+ }
+ }, {
+ name: 'sync-mempool-search <from> <search>',
+ desc: 'Import all pending data from matching <search>',
+ onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
+ const source: string = params[0]
+ const search: string = params[1]
+ if (!source || !(source.match(HOST_PATTERN) || source.match(FILE_PATTERN))) {
+ throw 'Source of sync is required. (host[:port])'
+ }
+ const logger = NewLogger()
+ const from: string = params[0]
+ const { host, port } = extractHostPort(from)
+ try {
+ const peer = PeerDTO.fromJSONObject({ endpoints: [['BASIC_MERKLED_API', host, port].join(' ')] })
+ const fromHost = peer.getHostPreferDNS();
+ const fromPort = peer.getPort();
+ logger.info('Looking at %s:%s...', fromHost, fromPort);
+ try {
+ const fromHost = await connect(peer)
+ const res = await fromHost.getRequirements(search)
+ await applyMempoolRequirements(server.conf.currency, res, server, logger)
+ } catch (e) {
+ logger.error(e);
+ }
+
+ await server.disconnect();
+ } catch(e) {
+ logger.error(e);
+ throw Error("Exiting");
+ }
+ }
+ }, {
name: 'forward <number> <fromHost> <fromPort> <toHost> <toPort>',
desc: 'Forward existing block <number> from a host to another',
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
@@ -357,3 +423,13 @@ export const CrawlerDependency = {
}]
}
}
+
+function extractHostPort(source: string) {
+ const sp = source.split(':')
+ const onHost = sp[0]
+ const onPort = parseInt(sp[1] ? sp[1] : '443') // Defaults to 443
+ return {
+ host: onHost,
+ port: onPort,
+ }
+}
\ No newline at end of file
diff --git a/app/modules/crawler/lib/contacter.ts b/app/modules/crawler/lib/contacter.ts
index c2e93c4e33f568f5147afa128f851723706e46ff..3742b09dd39ec8a28213701d954586e017a68eae 100644
--- a/app/modules/crawler/lib/contacter.ts
+++ b/app/modules/crawler/lib/contacter.ts
@@ -22,7 +22,7 @@ export class Contacter {
options:{ timeout:number }
fullyQualifiedHost:string
- constructor(private host:string, private port:number, opts:any = {}) {
+ constructor(public host:string, public port:number, opts:any = {}) {
this.options = {
timeout: (opts && opts.timeout) || CrawlerConstants.DEFAULT_TIMEOUT
}
diff --git a/app/modules/crawler/lib/sandbox.ts b/app/modules/crawler/lib/sandbox.ts
index 58d3d79a7a1f6bde1b4a37ebf106dcdff4d392c5..cf70e0ed1f93f6af7ddb5cc08d8b5b99937b9be1 100644
--- a/app/modules/crawler/lib/sandbox.ts
+++ b/app/modules/crawler/lib/sandbox.ts
@@ -12,75 +12,58 @@
// GNU Affero General Public License for more details.
"use strict";
-import {Contacter} from "./contacter"
import {Server} from "../../../../server"
import {rawer} from "../../../lib/common-libs/index"
import {parsers} from "../../../lib/common-libs/parsers/index"
+import {HttpRequirements} from "../../bma/lib/dtos"
-export const pullSandbox = async (currency:string, fromHost:string, fromPort:number, toHost:string, toPort:number, logger:any) => {
- const from = new Contacter(fromHost, fromPort);
- const to = new Contacter(toHost, toPort);
-
+export const pullSandboxToLocalServer = async (currency:string, fromHost: {
+ getRequirementsPending(minCert?: number): Promise<any>
+ host?: string
+ port?: number
+}, toServer:Server, logger:any, watcher:any = null, nbCertsMin = 1, notify = true) => {
let res
try {
- res = await from.getRequirementsPending(1)
+ res = await fromHost.getRequirementsPending(nbCertsMin || 1)
} catch (e) {
- // Silent error
- logger && logger.trace('Sandbox pulling: could not fetch requirements on %s', [fromHost, fromPort].join(':'))
+ watcher && watcher.writeStatus('Sandbox pulling: could not fetch requirements on %s', [fromHost.host, fromHost.port].join(':'))
}
if (res) {
- const docs = getDocumentsTree(currency, res)
- for (const identity of docs.identities) {
- await submitIdentity(identity, to)
- }
- for (const certification of docs.certifications) {
- await submitCertification(certification, to)
- }
- for (const membership of docs.memberships) {
- await submitMembership(membership, to)
- }
+ await applyMempoolRequirements(currency, res, toServer)
}
}
-export const pullSandboxToLocalServer = async (currency:string, fromHost:any, toServer:Server, logger:any, watcher:any = null, nbCertsMin = 1, notify = true) => {
- let res
- try {
- res = await fromHost.getRequirementsPending(nbCertsMin || 1)
- } catch (e) {
- watcher && watcher.writeStatus('Sandbox pulling: could not fetch requirements on %s', [fromHost.host, fromHost.port].join(':'))
- }
+export async function applyMempoolRequirements(currency: string, res: HttpRequirements, toServer: Server, notify = true, logger?: any, watcher?: any) {
- if (res) {
- const docs = getDocumentsTree(currency, res)
+ const docs = getDocumentsTree(currency, res)
- for (let i = 0; i < docs.identities.length; i++) {
- const idty = docs.identities[i];
- watcher && watcher.writeStatus('Identity ' + (i+1) + '/' + docs.identities.length)
- await submitIdentityToServer(idty, toServer, notify, logger)
- }
+ for (let i = 0; i < docs.identities.length; i++) {
+ const idty = docs.identities[i];
+ watcher && watcher.writeStatus('Identity ' + (i+1) + '/' + docs.identities.length)
+ await submitIdentityToServer(idty, toServer, notify, logger)
+ }
- for (let i = 0; i < docs.revocations.length; i++) {
- const idty = docs.revocations[i];
- watcher && watcher.writeStatus('Revocation ' + (i+1) + '/' + docs.revocations.length)
- await submitRevocationToServer(idty, toServer, notify, logger)
- }
+ for (let i = 0; i < docs.revocations.length; i++) {
+ const idty = docs.revocations[i];
+ watcher && watcher.writeStatus('Revocation ' + (i+1) + '/' + docs.revocations.length)
+ await submitRevocationToServer(idty, toServer, notify, logger)
+ }
- for (let i = 0; i < docs.certifications.length; i++) {
- const cert = docs.certifications[i];
- watcher && watcher.writeStatus('Certification ' + (i+1) + '/' + docs.certifications.length)
- await submitCertificationToServer(cert, toServer, notify, logger)
- }
+ for (let i = 0; i < docs.certifications.length; i++) {
+ const cert = docs.certifications[i];
+ watcher && watcher.writeStatus('Certification ' + (i+1) + '/' + docs.certifications.length)
+ await submitCertificationToServer(cert, toServer, notify, logger)
+ }
- for (let i = 0; i < docs.memberships.length; i++) {
- const ms = docs.memberships[i];
- watcher && watcher.writeStatus('Membership ' + (i+1) + '/' + docs.memberships.length)
- await submitMembershipToServer(ms, toServer, notify, logger)
- }
+ for (let i = 0; i < docs.memberships.length; i++) {
+ const ms = docs.memberships[i];
+ watcher && watcher.writeStatus('Membership ' + (i+1) + '/' + docs.memberships.length)
+ await submitMembershipToServer(ms, toServer, notify, logger)
}
}
-function getDocumentsTree(currency:string, res:any) {
+function getDocumentsTree(currency:string, res:HttpRequirements) {
const documents:any = {
identities: [],
certifications: [],
@@ -136,33 +119,6 @@ function getDocumentsTree(currency:string, res:any) {
return documents
}
-async function submitIdentity(idty:any, to:any, logger:any = null) {
- try {
- await to.postIdentity(idty)
- logger && logger.trace('Sandbox pulling: success with identity \'%s\'', idty.uid)
- } catch (e) {
- // Silent error
- }
-}
-
-async function submitCertification(cert:any, to:any, logger:any = null) {
- try {
- await to.postCert(cert)
- logger && logger.trace('Sandbox pulling: success with cert key %s => %s', cert.from.substr(0, 6), cert.idty_uid)
- } catch (e) {
- // Silent error
- }
-}
-
-async function submitMembership(ms:any, to:any, logger:any = null) {
- try {
- await to.postRenew(ms)
- logger && logger.trace('Sandbox pulling: success with membership \'%s\'', ms.uid)
- } catch (e) {
- // Silent error
- }
-}
-
async function submitIdentityToServer(idty:any, toServer:any, notify:boolean, logger:any) {
try {
const obj = parsers.parseIdentity.syncWrite(idty)