Commit 7f0eeb07 authored by Éloïs's avatar Éloïs

[ref] remove merkle dependency

parent ff8fdda6
......@@ -35,7 +35,6 @@ import { TransactionDTO } from "../dto/TransactionDTO";
import { CertDAL, DBCert } from "./sqliteDAL/CertDAL";
import { DBBlock } from "../db/DBBlock";
import { DBMembership, MembershipDAL } from "./sqliteDAL/MembershipDAL";
import { MerkleDTO } from "../dto/MerkleDTO";
import { CommonConstants } from "../common-libs/constants";
import { PowDAL } from "./fileDALs/PowDAL";
import { Initiable } from "./sqliteDAL/Initiable";
......@@ -1337,14 +1336,6 @@ export class FileDAL implements ServerDAO {
);
}
async merkleForPeers() {
let peers = await this.listAllPeersWithStatusNewUP();
const leaves = peers.map((peer: DBPeer) => peer.hash);
const merkle = new MerkleDTO();
merkle.initialize(leaves);
return merkle;
}
savePendingIdentity(idty: DBIdentity) {
return this.idtyDAL.saveIdentity(idty);
}
......
// Source file from duniter: Crypto-currency software to manage libre currency such as Ğ1
// Copyright (C) 2018 Cedric Moreau <cem.moreau@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
"use strict";
const merkle = require("merkle");
export class MerkleDTO {
private levels: any[];
nodes: any[];
depth: number;
initialize(leaves: string[]) {
const tree = merkle("sha256").sync(leaves);
this.depth = tree.depth();
this.nodes = tree.nodes();
this.levels = [];
for (let i = 0; i < tree.levels(); i++) {
this.levels[i] = tree.level(i);
}
return this;
}
remove(leaf: string) {
// If leaf IS present
if (~this.levels[this.depth].indexOf(leaf)) {
const leaves = this.leaves();
const index = leaves.indexOf(leaf);
if (~index) {
// Replacement: remove previous hash
leaves.splice(index, 1);
}
leaves.sort();
this.initialize(leaves);
}
}
removeMany(leaves: string[]) {
leaves.forEach((leaf: string) => {
// If leaf IS present
if (~this.levels[this.depth].indexOf(leaf)) {
const theLeaves = this.leaves();
const index = theLeaves.indexOf(leaf);
if (~index) {
// Replacement: remove previous hash
theLeaves.splice(index, 1);
}
}
});
leaves.sort();
this.initialize(leaves);
}
push(leaf: string, previous: string) {
// If leaf is not present
if (this.levels[this.depth].indexOf(leaf) == -1) {
const leaves = this.leaves();
// Update or replacement ?
if (previous && leaf != previous) {
const index = leaves.indexOf(previous);
if (~index) {
// Replacement: remove previous hash
leaves.splice(index, 1);
}
}
leaves.push(leaf);
leaves.sort();
this.initialize(leaves);
}
}
pushMany(leaves: string[]) {
leaves.forEach((leaf) => {
// If leaf is not present
if (this.levels[this.depth].indexOf(leaf) == -1) {
this.leaves().push(leaf);
}
});
leaves.sort();
this.initialize(leaves);
}
root() {
return this.levels.length > 0 ? this.levels[0][0] : "";
}
leaves() {
return this.levels[this.depth];
}
count() {
return this.leaves().length;
}
}
// Source file from duniter: Crypto-currency software to manage libre currency such as Ğ1
// Copyright (C) 2018 Cedric Moreau <cem.moreau@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
export const processForURL = async (
req: any,
merkle: any,
valueCoroutine: any
) => {
// Result
const json: any = {
depth: merkle.depth,
nodesCount: merkle.nodes,
leavesCount: merkle.levels[merkle.depth].length,
root: merkle.levels[0][0] || "",
leaves: [],
};
if (req.query.leaves) {
// Leaves
json.leaves = merkle.leaves();
return json;
} else if (req.query.leaf) {
// Extract of a leaf
json.leaves = [];
const hashes = [req.query.leaf];
// This code is in a loop for historic reasons. Should be set to non-loop style.
const values = await valueCoroutine(hashes);
hashes.forEach((hash) => {
json.leaf = {
hash: hash,
value: values[hash] || "",
};
});
return json;
} else {
return json;
}
};
......@@ -41,11 +41,7 @@ export abstract class AbstractController {
get PeeringService(): PeeringService {
return this.server.PeeringService;
}
get MerkleService() {
return this.server.MerkleService;
}
async pushEntity<T>(
req: any,
rawer: (req: any) => string,
......
......@@ -21,7 +21,7 @@ import {
HttpWS2PInfo,
} from "../dtos";
import { WS2PHead } from "../../../ws2p/lib/WS2PCluster";
import { DBPeer } from "../../../../lib/db/DBPeer";
import { DBPeer, JSONDBPeer } from "../../../../lib/db/DBPeer";
const http2raw = require("../http2raw");
......@@ -34,8 +34,10 @@ export class NetworkBinding extends AbstractController {
return p.json();
}
async peersGet(req: any): Promise<HttpMerkleOfPeers> {
let merkle = await this.server.dal.merkleForPeers();
async peersGet(req: any): Promise<JSONDBPeer[]> {
const peers: JSONDBPeer[] = (await this.server.dal.peerDAL.listAll()).map((dbPeer: DBPeer): JSONDBPeer => { return DBPeer.json(dbPeer); });
return peers;
/*let merkle = await this.server.dal.merkleForPeers();
return await this.MerkleService(req, merkle, async (hashes: string[]) => {
try {
let peers = await this.server.dal.findPeersWhoseHashIsIn(hashes);
......@@ -50,7 +52,7 @@ export class NetworkBinding extends AbstractController {
} catch (e) {
throw e;
}
});
});*/
}
async peersPost(req: any): Promise<HttpPeer> {
......
......@@ -717,7 +717,7 @@ export class BlockGenerator {
if (this.selfPubkey) {
block.issuer = this.selfPubkey;
}
// Members merkle
// Members
const joiners = Underscore.keys(joinData);
joiners.sort();
const previousCount = current ? current.membersCount : 0;
......
......@@ -4,7 +4,6 @@
* [Contents](#contents)
* [Overview](#overview)
* [Merkle URLs](#merkle-urls)
* [API](#api)
* [node/](#node)
* [summary](#nodesummary)
......@@ -113,117 +112,6 @@ Data is made accessible through an HTTP API mainly inspired from [OpenUDC_exchan
|-- block
`-- peer
## Merkle URLs
Merkle URL is a special kind of URL applicable for resources:
* `network/peering/peers (GET)`
Such kind of URL returns Merkle tree hashes informations. In Duniter, Merkle trees are an easy way to detect unsynced data and where the differences come from. For example, `network/peering/peers` is a Merkle tree whose leaves are peers' key fingerprint sorted ascending way. Thus, if any new peer is added, a branch of the tree will see its hash modified and propagated to the root hash. Change is then easy to detect.
For commodity issues, this URL uses query parameters to retrieve partial data of the tree, as most of the time all the data is not required. Duniter Merkle tree has a determined number of parent nodes (given a number of leaves), which allows to ask only for interval of them.
Here is an example of members Merkle tree with 5 members (taken from [Tree Hash EXchange format (THEX)](http://web.archive.org/web/20080316033726/http://www.open-content.net/specs/draft-jchapweske-thex-02.html)):
ROOT=H(H+E)
/ \
/ \
H=H(F+G) E
/ \ \
/ \ \
F=H(A+B) G=H(C+D) E
/ \ / \ \
/ \ / \ \
A B C D E
Note: H() is some hash function
Where A,B,C,D,E are already hashed data.
With such a tree structure, Duniter consider the tree has exactly 6 nodes: `[ROOT,H,E,F,G,E]`. Nodes are just an array, and for a Lambda Server LS1, it is easy to ask for the values of another server LS2 for level 1 (`H` and `E`, the second level): it requires nodes interval `[1;2]`.
Hence it is quite easy for anyone who wants to check if a `Z` member joined the Duniter community as it would alter the `E` branch of the tree:
ROOT'=H(H+E')
/ \
/ \
H=H(F+G) E'
/ \ \
/ \ \
F=H(A+B) G=H(C+D) E'=H(E+Z)
/ \ / \ / \
/ \ / \ / \
A B C D E Z
`ROOT` changed to `ROOT'`, `E` to `E'`, but `H` did not. The whole `E'` branch should be updated with the proper new data.
For that purpose, Merkle URL defines different parameters and results:
**Parameters**
Parameter | Description
--------- | -----------
`leaves` | Defines wether or not leaves hashes should be returned too. Defaults to `false`.
`leaf` | Hash of a leaf whose content should be returned. Ignore `leaves` parameter.
**Returns**
Merkle URL result with `leaves=false`.
```json
{
"depth": 3,
"nodesCount": 6,
"leavesCount": 5,
"root": "6513D6A1582DAE614D8A3B364BF3C64C513D236B"
}
```
Merkle URL result with `leaves=true`.
```json
{
"depth": 3,
"nodesCount": 6,
"leavesCount": 5,
"root": "6513D6A1582DAE614D8A3B364BF3C64C513D236B",
"leaves": [
"32096C2E0EFF33D844EE6D675407ACE18289357D",
"50C9E8D5FC98727B4BBC93CF5D64A68DB647F04F",
"6DCD4CE23D88E2EE9568BA546C007C63D9131C1B",
"AE4F281DF5A5D0FF3CAD6371F76D5C29B6D953EC",
"E0184ADEDF913B076626646D3F52C3B49C39AD6D"
]
}
```
Merkle URL result with `leaf=AE4F281DF5A5D0FF3CAD6371F76D5C29B6D953EC`.
```json
{
"depth": 3,
"nodesCount": 6,
"leavesCount": 5,
"root": "6513D6A1582DAE614D8A3B364BF3C64C513D236B",
"leaf": {
"hash": "AE4F281DF5A5D0FF3CAD6371F76D5C29B6D953EC",
"value": // JSON value (object, string, int, ...)
}
}
```
### Duniter Merkle trees leaves
Each tree manages different data, and has a different goal. Hence, each tree has its own rules on how are generated and sorted tree leaves.
Here is a summup of such rules:
Merkle URL | Leaf | Sort
---------------------- | --------------------------| -------------
`network/peers (GET)` | Hash of the peers' pubkey | By hash string sort, ascending.
#### Unicity
It has to be noted that **possible conflict exists** for leaves, as every leaf is hash, but is rather unlikely.
## API
### node/*
......@@ -1277,7 +1165,7 @@ Peering entry of the node.
#### `network/peering/peers (GET)`
**Goal**
Merkle URL refering to peering entries of every node inside the currency network.
URL refering to peering entries of every node inside the currency network.
**Parameters**
......@@ -1285,16 +1173,6 @@ Merkle URL refering to peering entries of every node inside the currency network
**Returns**
Merkle URL result.
```json
{
"depth": 3,
"nodesCount": 6,
"leavesCount": 5,
"root": "114B6E61CB5BB93D862CA3C1DFA8B99E313E66E9"
}
```
Merkle URL leaf: peering entry
```json
{
......
......@@ -1125,7 +1125,7 @@
sodipodi:role="line"
x="1145.6346"
y="239.68961"
style="font-size:20px">requests and building of merkles trees..</tspan></text>
style="font-size:20px">requests and building..</tspan></text>
</g>
</g>
<path
......
......@@ -4056,22 +4056,6 @@
"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
"dev": true
},
"merkle": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/merkle/-/merkle-0.6.0.tgz",
"integrity": "sha1-veM/ei/nu4IX3HiQVys645kHePM=",
"requires": {
"optimist": "0.6.1",
"through": "2.3.6"
},
"dependencies": {
"through": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.6.tgz",
"integrity": "sha1-JmgcD1JGcQIdTinffDa84tDs8ug="
}
}
},
"methods": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
......
......@@ -88,7 +88,6 @@
"leveldown": "5.6.0",
"levelup": "4.3.2",
"memdown": "5.1.0",
"merkle": "0.6.0",
"moment": "2.24.0",
"morgan": "1.10.0",
"multimeter": "0.1.1",
......
......@@ -84,7 +84,6 @@ export class Server extends stream.Duplex implements HookableServer {
sign:any
blockchain:any
MerkleService:(req:any, merkle:any, valueCoroutine:any) => any
IdentityService:IdentityService
MembershipService:MembershipService
PeeringService:PeeringService
......@@ -105,7 +104,6 @@ export class Server extends stream.Duplex implements HookableServer {
this.documentFIFO = new GlobalFifoPromise()
this.MerkleService = require("./app/lib/helpers/merkle").processForURL
this.IdentityService = new IdentityService(this.documentFIFO)
this.MembershipService = new MembershipService(this.documentFIFO)
this.PeeringService = new PeeringService(this, this.documentFIFO)
......
// Source file from duniter: Crypto-currency software to manage libre currency such as Ğ1
// Copyright (C) 2018 Cedric Moreau <cem.moreau@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
import {MerkleDTO} from "../../../app/lib/dto/MerkleDTO"
const should = require('should');
const assert = require('assert');
describe("Merkle ['a', 'b', 'c', 'd', 'e']", function(){
const m = new MerkleDTO() as any
m.initialize(['a', 'b', 'c', 'd', 'e']);
it('should have root 16E6BEB3E080910740A2923D6091618CAA9968AEAD8A52D187D725D199548E2C', function(){
assert.equal(m.levels[0], '16E6BEB3E080910740A2923D6091618CAA9968AEAD8A52D187D725D199548E2C');
});
it('should have level 1,0 AB4587D9F4AD6990E0BF4A1C5A836C78CCE881C2B7C4287C0A7DA15B47B8CF1F', function(){
assert.equal(m.levels[1][0], 'AB4587D9F4AD6990E0BF4A1C5A836C78CCE881C2B7C4287C0A7DA15B47B8CF1F');
});
it('should have level 1,1 3F79BB7B435B05321651DAEFD374CDC681DC06FAA65E374E38337B88CA046DEA', function(){
assert.equal(m.levels[1][1], '3F79BB7B435B05321651DAEFD374CDC681DC06FAA65E374E38337B88CA046DEA');
});
it('should have 4 levels', function(){
assert.equal(m.levels.length, 4);
});
it('should have depth: 3', function(){
assert.equal(m.depth, 3);
});
it('should have 6 nodes', function(){
assert.equal(m.nodes, 6);
});
it('should have 5 leaves', function(){
assert.equal(m.leaves().length, 5);
});
});
describe("Merkle []", function(){
const m = new MerkleDTO() as any
m.initialize([]);
it('should have root empty', function(){
assert.equal(m.levels[0], '');
});
it('should have 1 levels', function(){
assert.equal(m.levels.length, 1);
});
it('should have depth: 0', function(){
assert.equal(m.depth, 0);
});
it('should have 0 nodes', function(){
assert.equal(m.nodes, 0);
});
it('should have 0 leaves', function(){
assert.equal(m.leaves().length, 0);
});
});
// Source file from duniter: Crypto-currency software to manage libre currency such as Ğ1
// Copyright (C) 2018 Cedric Moreau <cem.moreau@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
import {Underscore} from "../../../app/lib/common-libs/underscore"
import {HttpMerkleOfPeers} from "../../../app/modules/bma/lib/dtos"
import {NewTestingServer} from "../tools/toolbox"
import {expectAnswer, expectHttpCode} from "../tools/http-expect"
const rp = require('request-promise');
const commonConf = {
bmaWithCrawler: true,
ipv4: '127.0.0.1',
remoteipv4: '127.0.0.1',
currency: 'bb',
httpLogs: true,
forksize: 3,
sigQty: 1
};
const s1 = NewTestingServer(Underscore.extend({
name: 'bb33',
ipv4: '127.0.0.1',
port: '20501',
remoteport: '20501',
ws2p: { upnp: false },
pair: {
pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd',
sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'
},
rootoffset: 10,
sigQty: 1, dt: 0, ud0: 120
}, commonConf));
const s2 = NewTestingServer(Underscore.extend({
name: 'bb12',
port: '20502',
remoteport: '20502',
ws2p: { upnp: false },
pair: {
pub: 'DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo',
sec: '64EYRvdPpTfLGGmaX5nijLXRqWXaVz8r1Z1GtaahXwVSJGQRn7tqkxLb288zwSYzELMEG5ZhXSBYSxsTsz1m9y8F'
}
}, commonConf));
describe("Network Merkle", function() {
before(async () => {
await s1.initDalBmaConnections()
await s2.initDalBmaConnections()
await s1._server.PeeringService.generateSelfPeer(s1._server.conf, 0)
await s2._server.PeeringService.generateSelfPeer(s1._server.conf, 0)
await s1.sharePeeringWith(s2)
})
describe("Server 1 /network/peering", function() {
it('/peers?leaves=true', function() {
return expectAnswer(rp('http://127.0.0.1:20501/network/peering/peers?leaves=true', { json: true }), (res:HttpMerkleOfPeers) => {
res.should.have.property('depth').equal(0);
res.should.have.property('nodesCount').equal(0);
res.should.have.property('leavesCount').equal(1);
res.should.have.property('root').equal('C3EAB939F0BEF711461A140A1BA2649C75905107FACA3BE9C5F76F7FD1C7BC5E');
res.should.have.property('leaves').length(1);
res.leaves[0].should.equal('C3EAB939F0BEF711461A140A1BA2649C75905107FACA3BE9C5F76F7FD1C7BC5E');
});
});
it('/peers?leaf=C3EAB939F0BEF711461A140A1BA2649C75905107FACA3BE9C5F76F7FD1C7BC5E', function() {
return expectAnswer(rp('http://127.0.0.1:20501/network/peering/peers?leaf=C3EAB939F0BEF711461A140A1BA2649C75905107FACA3BE9C5F76F7FD1C7BC5E', { json: true }), (res:HttpMerkleOfPeers) => {
res.should.have.property('depth').equal(0);
res.should.have.property('nodesCount').equal(0);
res.should.have.property('leavesCount').equal(1);
res.should.have.property('root').equal('C3EAB939F0BEF711461A140A1BA2649C75905107FACA3BE9C5F76F7FD1C7BC5E');
res.should.have.property('leaves').length(0);
res.should.have.property('leaf').have.property('hash').equal('C3EAB939F0BEF711461A140A1BA2649C75905107FACA3BE9C5F76F7FD1C7BC5E');
res.should.have.property('leaf').have.property('value');
res.should.have.property('leaf').have.property('value').have.property('pubkey').equal('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
res.should.have.property('leaf').have.property('value').have.property('block').equal('0-E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855');
res.should.have.property('leaf').have.property('value').have.property('signature').equal('V4fA6+ll3aLIkh9ixhdQyd6xJxcYGcbRQhA4P9ATp3m0jCwKq3zbU5udGstBPTUn9EgCOxt08gO7teM4EYO/DQ==');