Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
duniter
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Monitor
Service Desk
Analyze
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
nodes
typescript
duniter
Commits
4be285c1
Commit
4be285c1
authored
6 years ago
by
Cédric Moreau
Browse files
Options
Downloads
Patches
Plain Diff
[enh] Add `sync-mempool` and `sync-mempool-search` commands
parent
f552e8e6
No related branches found
No related tags found
No related merge requests found
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
app/modules/crawler/index.ts
+76
-0
76 additions, 0 deletions
app/modules/crawler/index.ts
app/modules/crawler/lib/contacter.ts
+1
-1
1 addition, 1 deletion
app/modules/crawler/lib/contacter.ts
app/modules/crawler/lib/sandbox.ts
+31
-75
31 additions, 75 deletions
app/modules/crawler/lib/sandbox.ts
with
108 additions
and
76 deletions
app/modules/crawler/index.ts
+
76
−
0
View file @
4be285c1
...
@@ -21,6 +21,12 @@ import {rawer} from "../../lib/common-libs/index"
...
@@ -21,6 +21,12 @@ import {rawer} from "../../lib/common-libs/index"
import
{
PeerDTO
}
from
"
../../lib/dto/PeerDTO
"
import
{
PeerDTO
}
from
"
../../lib/dto/PeerDTO
"
import
{
Buid
}
from
"
../../lib/common-libs/buid
"
import
{
Buid
}
from
"
../../lib/common-libs/buid
"
import
{
BlockDTO
}
from
"
../../lib/dto/BlockDTO
"
import
{
BlockDTO
}
from
"
../../lib/dto/BlockDTO
"
import
{
NewLogger
}
from
"
../../lib/logger
"
import
{
connect
}
from
"
./lib/connect
"
import
{
applyMempoolRequirements
,
pullSandboxToLocalServer
}
from
"
./lib/sandbox
"
const
HOST_PATTERN
=
/^
[^
:
/]
+
(
:
[
0-9
]{1,5})?
$/
const
FILE_PATTERN
=
/^
(\/
.+
)
$/
export
const
CrawlerDependency
=
{
export
const
CrawlerDependency
=
{
duniter
:
{
duniter
:
{
...
@@ -168,6 +174,66 @@ export const CrawlerDependency = {
...
@@ -168,6 +174,66 @@ export const CrawlerDependency = {
}
}
}
}
},
{
},
{
name
:
'
sync-mempool <from>
'
,
desc
:
'
Import all pending data from matching <search>
'
,
onDatabaseExecute
:
async
(
server
:
Server
,
conf
:
ConfDTO
,
program
:
any
,
params
:
any
)
=>
{
const
source
:
string
=
params
[
0
]
if
(
!
source
||
!
(
source
.
match
(
HOST_PATTERN
)
||
source
.
match
(
FILE_PATTERN
)))
{
throw
'
Source of sync is required. (host[:port])
'
}
const
logger
=
NewLogger
()
const
from
:
string
=
params
[
0
]
const
{
host
,
port
}
=
extractHostPort
(
from
)
try
{
const
peer
=
PeerDTO
.
fromJSONObject
({
endpoints
:
[[
'
BASIC_MERKLED_API
'
,
host
,
port
].
join
(
'
'
)]
})
const
fromHost
=
peer
.
getHostPreferDNS
();
const
fromPort
=
peer
.
getPort
();
logger
.
info
(
'
Looking at %s:%s...
'
,
fromHost
,
fromPort
);
try
{
const
fromHost
=
await
connect
(
peer
,
60
*
1000
)
await
pullSandboxToLocalServer
(
server
.
conf
.
currency
,
fromHost
,
server
,
logger
)
}
catch
(
e
)
{
logger
.
error
(
e
);
}
await
server
.
disconnect
();
}
catch
(
e
)
{
logger
.
error
(
e
);
throw
Error
(
"
Exiting
"
);
}
}
},
{
name
:
'
sync-mempool-search <from> <search>
'
,
desc
:
'
Import all pending data from matching <search>
'
,
onDatabaseExecute
:
async
(
server
:
Server
,
conf
:
ConfDTO
,
program
:
any
,
params
:
any
)
=>
{
const
source
:
string
=
params
[
0
]
const
search
:
string
=
params
[
1
]
if
(
!
source
||
!
(
source
.
match
(
HOST_PATTERN
)
||
source
.
match
(
FILE_PATTERN
)))
{
throw
'
Source of sync is required. (host[:port])
'
}
const
logger
=
NewLogger
()
const
from
:
string
=
params
[
0
]
const
{
host
,
port
}
=
extractHostPort
(
from
)
try
{
const
peer
=
PeerDTO
.
fromJSONObject
({
endpoints
:
[[
'
BASIC_MERKLED_API
'
,
host
,
port
].
join
(
'
'
)]
})
const
fromHost
=
peer
.
getHostPreferDNS
();
const
fromPort
=
peer
.
getPort
();
logger
.
info
(
'
Looking at %s:%s...
'
,
fromHost
,
fromPort
);
try
{
const
fromHost
=
await
connect
(
peer
)
const
res
=
await
fromHost
.
getRequirements
(
search
)
await
applyMempoolRequirements
(
server
.
conf
.
currency
,
res
,
server
,
logger
)
}
catch
(
e
)
{
logger
.
error
(
e
);
}
await
server
.
disconnect
();
}
catch
(
e
)
{
logger
.
error
(
e
);
throw
Error
(
"
Exiting
"
);
}
}
},
{
name
:
'
forward <number> <fromHost> <fromPort> <toHost> <toPort>
'
,
name
:
'
forward <number> <fromHost> <fromPort> <toHost> <toPort>
'
,
desc
:
'
Forward existing block <number> from a host to another
'
,
desc
:
'
Forward existing block <number> from a host to another
'
,
onDatabaseExecute
:
async
(
server
:
Server
,
conf
:
ConfDTO
,
program
:
any
,
params
:
any
)
=>
{
onDatabaseExecute
:
async
(
server
:
Server
,
conf
:
ConfDTO
,
program
:
any
,
params
:
any
)
=>
{
...
@@ -357,3 +423,13 @@ export const CrawlerDependency = {
...
@@ -357,3 +423,13 @@ export const CrawlerDependency = {
}]
}]
}
}
}
}
function
extractHostPort
(
source
:
string
)
{
const
sp
=
source
.
split
(
'
:
'
)
const
onHost
=
sp
[
0
]
const
onPort
=
parseInt
(
sp
[
1
]
?
sp
[
1
]
:
'
443
'
)
// Defaults to 443
return
{
host
:
onHost
,
port
:
onPort
,
}
}
\ No newline at end of file
This diff is collapsed.
Click to expand it.
app/modules/crawler/lib/contacter.ts
+
1
−
1
View file @
4be285c1
...
@@ -22,7 +22,7 @@ export class Contacter {
...
@@ -22,7 +22,7 @@ export class Contacter {
options
:{
timeout
:
number
}
options
:{
timeout
:
number
}
fullyQualifiedHost
:
string
fullyQualifiedHost
:
string
constructor
(
p
rivate
host
:
string
,
p
rivate
port
:
number
,
opts
:
any
=
{})
{
constructor
(
p
ublic
host
:
string
,
p
ublic
port
:
number
,
opts
:
any
=
{})
{
this
.
options
=
{
this
.
options
=
{
timeout
:
(
opts
&&
opts
.
timeout
)
||
CrawlerConstants
.
DEFAULT_TIMEOUT
timeout
:
(
opts
&&
opts
.
timeout
)
||
CrawlerConstants
.
DEFAULT_TIMEOUT
}
}
...
...
This diff is collapsed.
Click to expand it.
app/modules/crawler/lib/sandbox.ts
+
31
−
75
View file @
4be285c1
...
@@ -12,46 +12,30 @@
...
@@ -12,46 +12,30 @@
// GNU Affero General Public License for more details.
// GNU Affero General Public License for more details.
"
use strict
"
;
"
use strict
"
;
import
{
Contacter
}
from
"
./contacter
"
import
{
Server
}
from
"
../../../../server
"
import
{
Server
}
from
"
../../../../server
"
import
{
rawer
}
from
"
../../../lib/common-libs/index
"
import
{
rawer
}
from
"
../../../lib/common-libs/index
"
import
{
parsers
}
from
"
../../../lib/common-libs/parsers/index
"
import
{
parsers
}
from
"
../../../lib/common-libs/parsers/index
"
import
{
HttpRequirements
}
from
"
../../bma/lib/dtos
"
export
const
pullSandbox
=
async
(
currency
:
string
,
fromHost
:
string
,
fromPort
:
number
,
toHost
:
string
,
toPort
:
number
,
logger
:
any
)
=>
{
export
const
pullSandboxToLocalServer
=
async
(
currency
:
string
,
fromHost
:
{
const
from
=
new
Contacter
(
fromHost
,
fromPort
);
getRequirementsPending
(
minCert
?:
number
):
Promise
<
any
>
const
to
=
new
Contacter
(
toHost
,
toPort
);
host
?:
string
port
?:
number
},
toServer
:
Server
,
logger
:
any
,
watcher
:
any
=
null
,
nbCertsMin
=
1
,
notify
=
true
)
=>
{
let
res
let
res
try
{
try
{
res
=
await
from
.
getRequirementsPending
(
1
)
res
=
await
from
Host
.
getRequirementsPending
(
nbCertsMin
||
1
)
}
catch
(
e
)
{
}
catch
(
e
)
{
// Silent error
watcher
&&
watcher
.
writeStatus
(
'
Sandbox pulling: could not fetch requirements on %s
'
,
[
fromHost
.
host
,
fromHost
.
port
].
join
(
'
:
'
))
logger
&&
logger
.
trace
(
'
Sandbox pulling: could not fetch requirements on %s
'
,
[
fromHost
,
fromPort
].
join
(
'
:
'
))
}
}
if
(
res
)
{
if
(
res
)
{
const
docs
=
getDocumentsTree
(
currency
,
res
)
await
applyMempoolRequirements
(
currency
,
res
,
toServer
)
for
(
const
identity
of
docs
.
identities
)
{
await
submitIdentity
(
identity
,
to
)
}
for
(
const
certification
of
docs
.
certifications
)
{
await
submitCertification
(
certification
,
to
)
}
for
(
const
membership
of
docs
.
memberships
)
{
await
submitMembership
(
membership
,
to
)
}
}
}
}
}
export
const
pullSandboxToLocalServer
=
async
(
currency
:
string
,
fromHost
:
any
,
toServer
:
Server
,
logger
:
any
,
watcher
:
any
=
null
,
nbCertsMin
=
1
,
notify
=
true
)
=>
{
export
async
function
applyMempoolRequirements
(
currency
:
string
,
res
:
HttpRequirements
,
toServer
:
Server
,
notify
=
true
,
logger
?:
any
,
watcher
?:
any
)
{
let
res
try
{
res
=
await
fromHost
.
getRequirementsPending
(
nbCertsMin
||
1
)
}
catch
(
e
)
{
watcher
&&
watcher
.
writeStatus
(
'
Sandbox pulling: could not fetch requirements on %s
'
,
[
fromHost
.
host
,
fromHost
.
port
].
join
(
'
:
'
))
}
if
(
res
)
{
const
docs
=
getDocumentsTree
(
currency
,
res
)
const
docs
=
getDocumentsTree
(
currency
,
res
)
for
(
let
i
=
0
;
i
<
docs
.
identities
.
length
;
i
++
)
{
for
(
let
i
=
0
;
i
<
docs
.
identities
.
length
;
i
++
)
{
...
@@ -78,9 +62,8 @@ export const pullSandboxToLocalServer = async (currency:string, fromHost:any, to
...
@@ -78,9 +62,8 @@ export const pullSandboxToLocalServer = async (currency:string, fromHost:any, to
await
submitMembershipToServer
(
ms
,
toServer
,
notify
,
logger
)
await
submitMembershipToServer
(
ms
,
toServer
,
notify
,
logger
)
}
}
}
}
}
function
getDocumentsTree
(
currency
:
string
,
res
:
any
)
{
function
getDocumentsTree
(
currency
:
string
,
res
:
HttpRequirements
)
{
const
documents
:
any
=
{
const
documents
:
any
=
{
identities
:
[],
identities
:
[],
certifications
:
[],
certifications
:
[],
...
@@ -136,33 +119,6 @@ function getDocumentsTree(currency:string, res:any) {
...
@@ -136,33 +119,6 @@ function getDocumentsTree(currency:string, res:any) {
return
documents
return
documents
}
}
async
function
submitIdentity
(
idty
:
any
,
to
:
any
,
logger
:
any
=
null
)
{
try
{
await
to
.
postIdentity
(
idty
)
logger
&&
logger
.
trace
(
'
Sandbox pulling: success with identity
\'
%s
\'
'
,
idty
.
uid
)
}
catch
(
e
)
{
// Silent error
}
}
async
function
submitCertification
(
cert
:
any
,
to
:
any
,
logger
:
any
=
null
)
{
try
{
await
to
.
postCert
(
cert
)
logger
&&
logger
.
trace
(
'
Sandbox pulling: success with cert key %s => %s
'
,
cert
.
from
.
substr
(
0
,
6
),
cert
.
idty_uid
)
}
catch
(
e
)
{
// Silent error
}
}
async
function
submitMembership
(
ms
:
any
,
to
:
any
,
logger
:
any
=
null
)
{
try
{
await
to
.
postRenew
(
ms
)
logger
&&
logger
.
trace
(
'
Sandbox pulling: success with membership
\'
%s
\'
'
,
ms
.
uid
)
}
catch
(
e
)
{
// Silent error
}
}
async
function
submitIdentityToServer
(
idty
:
any
,
toServer
:
any
,
notify
:
boolean
,
logger
:
any
)
{
async
function
submitIdentityToServer
(
idty
:
any
,
toServer
:
any
,
notify
:
boolean
,
logger
:
any
)
{
try
{
try
{
const
obj
=
parsers
.
parseIdentity
.
syncWrite
(
idty
)
const
obj
=
parsers
.
parseIdentity
.
syncWrite
(
idty
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment