Commit cf7f6f98 authored by Cédric Moreau's avatar Cédric Moreau
Browse files

[enh] Refactoring: type FileDAL.idtyDAL

parent 34d58f9e
......@@ -17,7 +17,7 @@ import {StatDAL} from "./fileDALs/StatDAL"
import {ConfDTO} from "../dto/ConfDTO"
import {BlockDTO} from "../dto/BlockDTO"
import {DBHead} from "../db/DBHead"
import {DBIdentity} from "./sqliteDAL/IdentityDAL"
import {DBIdentity, IdentityDAL} from "./sqliteDAL/IdentityDAL"
import {CindexEntry, IindexEntry, IndexEntry, MindexEntry, SindexEntry} from "../indexer"
import {DBPeer} from "./sqliteDAL/PeerDAL"
import {TransactionDTO} from "../dto/TransactionDTO"
......@@ -28,8 +28,8 @@ import {DBBlock} from "../db/DBBlock"
import {DBMembership} from "./sqliteDAL/MembershipDAL"
import {MerkleDTO} from "../dto/MerkleDTO"
import {CommonConstants} from "../common-libs/constants"
import { ProxiesConf } from '../proxy';
import {PowDAL} from "./fileDALs/PowDAL";
import {Initiable} from "./sqliteDAL/Initiable"
const fs = require('fs')
const path = require('path')
......@@ -54,14 +54,14 @@ export class FileDAL {
wotb:any
profile:string
powDAL:PowDAL
powDAL:any
confDAL:any
metaDAL:any
peerDAL:any
blockDAL:any
txsDAL:any
statDAL:any
idtyDAL:any
idtyDAL:IdentityDAL
certDAL:any
msDAL:any
walletDAL:any
......@@ -70,7 +70,7 @@ export class FileDAL {
iindexDAL:any
sindexDAL:any
cindexDAL:any
newDals:any
newDals:{ [k:string]: Initiable }
loadConfHook: (conf:ConfDTO) => Promise<void>
saveConfHook: (conf:ConfDTO) => Promise<ConfDTO>
......@@ -292,7 +292,7 @@ export class FileDAL {
return this.sindexDAL.getAvailableForPubkey(pubkey)
}
async getIdentityByHashOrNull(hash:string) {
async getIdentityByHashOrNull(hash:string): Promise<DBIdentity|null> {
const pending = await this.idtyDAL.getByHash(hash);
if (!pending) {
return this.iindexDAL.getFromHash(hash);
......
......@@ -12,13 +12,15 @@
// GNU Affero General Public License for more details.
import {CFSCore} from "./CFSCore";
import {Initiable} from "../sqliteDAL/Initiable"
export class AbstractCFS {
export abstract class AbstractCFS extends Initiable {
protected coreFS:CFSCore
protected dal:any
constructor(rootPath:string, qioFS:any) {
super()
this.coreFS = new CFSCore(rootPath, qioFS)
}
}
......@@ -12,12 +12,13 @@
// GNU Affero General Public License for more details.
import {SQLiteDriver} from "../drivers/SQLiteDriver"
import {Initiable} from "./Initiable"
/**
* Created by cgeek on 22/08/15.
*/
const _ = require('underscore');
const co = require('co');
const colors = require('colors');
const logger = require('../../logger').NewLogger('sqlite');
......@@ -25,7 +26,7 @@ export interface BeforeSaveHook<T> {
(t:T): void
}
export abstract class AbstractSQLite<T> {
export abstract class AbstractSQLite<T> extends Initiable {
constructor(
private driver:SQLiteDriver,
......@@ -38,6 +39,7 @@ export abstract class AbstractSQLite<T> {
private transientFields: string[] = [],
private beforeSaveHook: BeforeSaveHook<T> | null = null
) {
super()
}
async query(sql:string, params: any[] = []): Promise<T[]> {
......
......@@ -13,10 +13,11 @@
import {AbstractSQLite} from "./AbstractSQLite"
import {SQLiteDriver} from "../drivers/SQLiteDriver"
import { SandBox } from './SandBox';
import {SandBox} from './SandBox';
import {IdentityDTO} from "../../dto/IdentityDTO"
import {Cloneable} from "../../dto/Cloneable";
import { DBDocument } from './DocumentDAL';
import {DBDocument} from './DocumentDAL';
const constants = require('../../constants');
export abstract class DBIdentity implements Cloneable {
......@@ -157,7 +158,7 @@ export class ExistingDBIdentity extends DBIdentity {
}
}
export interface DBSandboxIdentity extends DBIdentity,DBDocument {
export interface DBSandboxIdentity extends DBDocument {
certsCount: number
ref_block: number
}
......@@ -278,7 +279,7 @@ export class IdentityDAL extends AbstractSQLite<DBIdentity> {
getPendingIdentities() {
return this.sqlFind({
revocation_sig: { $null: false },
revocation_sig: { $null: true },
revoked: false
})
}
......
export abstract class Initiable {
abstract init(): Promise<void>
}
......@@ -30,7 +30,6 @@ import {TxsDAL} from "./TxsDAL"
const _ = require('underscore')
const logger = require('../../logger').NewLogger('metaDAL');
const constants = require('./../../constants');
export interface DBMeta {
id: number,
......
......@@ -11,21 +11,23 @@
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
import { IindexEntry } from './../../../../lib/indexer';
import {IindexEntry} from './../../../../lib/indexer';
import {AbstractController} from "./AbstractController";
import {BMAConstants} from "../constants";
import {DBIdentity} from "../../../../lib/dal/sqliteDAL/IdentityDAL";
import { IdentityForRequirements } from '../../../../service/BlockchainService';
import {IdentityForRequirements} from '../../../../service/BlockchainService';
import {
HttpCert,
HttpCertIdentity, HttpCertifications,
HttpCertIdentity,
HttpCertifications,
HttpIdentity,
HttpIdentityRequirement,
HttpLookup,
HttpMembers,
HttpMembershipList,
HttpRequirements,
HttpResult, HttpSimpleIdentity
HttpResult,
HttpSimpleIdentity
} from "../dtos";
const _ = require('underscore');
......@@ -168,16 +170,27 @@ export class WOTBinding extends AbstractController {
async requirementsOfPending(req:any): Promise<HttpRequirements> {
const minsig = ParametersService.getMinSig(req)
let identities:IdentityForRequirements[] = await this.server.dal.idtyDAL.query(
let identities:IdentityForRequirements[] = (await this.server.dal.idtyDAL.query(
'SELECT i.*, count(c.sig) as nbSig ' +
'FROM idty i, cert c ' +
'WHERE c.target = i.hash group by i.hash having nbSig >= ?',
minsig)
const members:IdentityForRequirements[] = (await this.server.dal.idtyDAL.query(
[minsig])).map(i => ({
hash: i.hash || "",
member: i.member || false,
wasMember: i.wasMember || false,
pubkey: i.pubkey,
uid: i.uid || "",
buid: i.buid || "",
sig: i.sig || "",
revocation_sig: i.revocation_sig,
revoked: i.revoked,
revoked_on: i.revoked_on ? 1 : 0
}))
const members:IdentityForRequirements[] = (await this.server.dal.iindexDAL.query(
'SELECT i.*, count(c.sig) as nbSig ' +
'FROM i_index i, cert c ' +
'WHERE c.`to` = i.pub group by i.pub having nbSig >= ?',
minsig)).map((i:IindexEntry):IdentityForRequirements => {
[minsig])).map((i:IindexEntry):IdentityForRequirements => {
return {
hash: i.hash || "",
member: i.member || false,
......
......@@ -62,7 +62,7 @@ export class ParametersService {
return req.params.hash;
};
static getMinSig(req:any){
static getMinSig(req:any): number {
if(!req.params.minsig){
return 4 // Default value
}
......
......@@ -19,14 +19,14 @@ import {Server} from "../../server"
const qfs = require('q-io/fs');
const directory = require('../lib/system/directory');
const constants = require('../lib/constants');
const path = require('path');
const Tail = require("tail").Tail
module.exports = {
duniter: {
cliOptions: [
{ value: '--loglevel <level>', desc: 'Logs level, either [error,warning,info,debug,trace]. default to `info`.' }
{ value: '--loglevel <level>', desc: 'Logs level, either [error,warning,info,debug,trace]. default to `info`.' },
{ value: '--sql-traces', desc: 'Will log every SQL query that is executed. Requires --loglevel \'trace\'.' }
],
service: {
......
......@@ -21,6 +21,7 @@ import {parsers} from "../../lib/common-libs/parsers/index"
import {PeerDTO} from "../../lib/dto/PeerDTO"
import {Server} from "../../../server"
import {BlockDTO} from "../../lib/dto/BlockDTO"
import {DBIdentity} from "../../lib/dal/sqliteDAL/IdentityDAL"
const async = require('async');
......@@ -109,7 +110,7 @@ export const ProverDependency = {
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
const difficulty = params[0]
const generator = new BlockGeneratorWhichProves(server, null);
let toDelete, catched = true;
let toDelete:DBIdentity[] = [], catched = true;
do {
try {
await generateAndSend(program, difficulty, server, () => () => generator.nextBlock())
......
......@@ -12,14 +12,12 @@
// GNU Affero General Public License for more details.
"use strict";
import {ConfDTO} from "../../../lib/dto/ConfDTO"
import {Server} from "../../../../server"
import {BlockchainContext} from "../../../lib/computation/BlockchainContext"
import {TransactionDTO} from "../../../lib/dto/TransactionDTO"
import {GLOBAL_RULES_HELPERS} from "../../../lib/rules/global_rules"
import {LOCAL_RULES_HELPERS} from "../../../lib/rules/local_rules"
import {Indexer} from "../../../lib/indexer"
import {FileDAL} from "../../../lib/dal/fileDAL"
import {DBBlock} from "../../../lib/db/DBBlock"
import {verify} from "../../../lib/common-libs/crypto/keyring"
import {rawer} from "../../../lib/common-libs/index"
......@@ -29,6 +27,9 @@ import {IdentityDTO} from "../../../lib/dto/IdentityDTO"
import {CertificationDTO} from "../../../lib/dto/CertificationDTO"
import {MembershipDTO} from "../../../lib/dto/MembershipDTO"
import {BlockDTO} from "../../../lib/dto/BlockDTO"
import {DBIdentity} from "../../../lib/dal/sqliteDAL/IdentityDAL"
import {ConfDTO} from "../../../lib/dto/ConfDTO"
import {FileDAL} from "../../../lib/dal/fileDAL"
const _ = require('underscore');
const moment = require('moment');
......@@ -38,22 +39,26 @@ const constants = CommonConstants
export class BlockGenerator {
conf:ConfDTO
dal:any
mainContext:BlockchainContext
selfPubkey:string
logger:any
constructor(private server:Server) {
this.conf = server.conf;
this.dal = server.dal;
this.mainContext = server.BlockchainService.getContext();
this.selfPubkey = (this.conf.pair && this.conf.pair.pub) || ''
this.logger = server.logger;
}
get conf(): ConfDTO {
return this.server.conf
}
get dal(): FileDAL {
return this.server.dal
}
nextBlock(manualValues:any = {}, simulationValues:any = {}) {
return this.generateNextBlock(new NextBlockGenerator(this.mainContext, this.conf, this.dal, this.logger), manualValues, simulationValues)
return this.generateNextBlock(new NextBlockGenerator(this.mainContext, this.server, this.logger), manualValues, simulationValues)
}
async manualRoot() {
......@@ -155,7 +160,7 @@ export class BlockGenerator {
const leavers:string[] = [];
memberships.forEach((ms:any) => leavers.push(ms.issuer));
for (const ms of memberships) {
const leave = { identity: null, ms: ms, key: null, idHash: '' };
const leave: { identity: DBIdentity|null, ms: any, key: any, idHash: string } = { identity: null, ms: ms, key: null, idHash: '' };
leave.idHash = (hashf(ms.userid + ms.certts + ms.issuer) + "").toUpperCase();
let block;
if (current) {
......@@ -357,7 +362,7 @@ export class BlockGenerator {
throw constants.ERRORS.TOO_OLD_IDENTITY;
}
else if (!identity.wasMember && identity.buid != CommonConstants.SPECIAL_BLOCK) {
const idtyBasedBlock = await this.dal.getBlock(identity.buid);
const idtyBasedBlock = await this.dal.getBlock(parseInt(identity.buid.split('-')[0]))
const age = current.medianTime - idtyBasedBlock.medianTime;
if (age > this.conf.idtyWindow) {
throw constants.ERRORS.TOO_OLD_IDENTITY;
......@@ -686,11 +691,18 @@ class NextBlockGenerator implements BlockGeneratorInterface {
constructor(
private mainContext:BlockchainContext,
private conf:ConfDTO,
private dal:FileDAL,
private server:Server,
private logger:any) {
}
get conf() {
return this.server.conf
}
get dal() {
return this.server.dal
}
async findNewCertsFromWoT(current:DBBlock) {
const updates:any = {};
const updatesToFrom:any = {};
......
......@@ -11,7 +11,6 @@
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
import { WS2PCluster } from './WS2PCluster';
import {Server} from "../../../../server"
import {WS2PConnection, WS2PPubkeyLocalAuth, WS2PPubkeyRemoteAuth} from "./WS2PConnection"
import {Key} from "../../../lib/common-libs/crypto/keyring"
......@@ -19,12 +18,18 @@ import {WS2PMessageHandler} from "./impl/WS2PMessageHandler"
import {WS2PConstants} from "./constants"
import {WS2PStreamer} from "./WS2PStreamer"
import {WS2PSingleWriteStream} from "./WS2PSingleWriteStream"
import { ProxiesConf } from '../../../lib/proxy';
import { server } from '../../../../test/integration/tools/toolbox';
import {ProxiesConf} from '../../../lib/proxy';
export class WS2PClient {
private constructor(public connection:WS2PConnection) {}
private constructor(
public connection:WS2PConnection,
private streamer:WS2PStreamer) {
}
disableStream() {
this.streamer.disable()
}
static async connectTo(server:Server, fullEndpointAddress:string, endpointVersion:number, expectedWS2PUID:string, messageHandler:WS2PMessageHandler, expectedPub:string, allowKey:(pub:string)=>Promise<boolean> ) {
const k2 = new Key(server.conf.pair.pub, server.conf.pair.sec)
......@@ -69,6 +74,6 @@ export class WS2PClient {
c.close()
throw e
}
return new WS2PClient(c)
return new WS2PClient(c, streamer)
}
}
\ No newline at end of file
......@@ -336,7 +336,7 @@ export class WS2PCluster {
}
}
async connectToRemoteWS(endpointVersion:number, host: string, port: number, path:string, messageHandler:WS2PMessageHandler, expectedPub:string, ws2pEndpointUUID:string = ""): Promise<WS2PConnection> {
async connectToRemoteWS(endpointVersion:number, host: string, port: number, path:string, messageHandler:WS2PMessageHandler, expectedPub:string, ws2pEndpointUUID:string = ""): Promise<WS2PClient> {
const uuid = nuuid.v4()
let pub = expectedPub.slice(0, 8)
const api:string = (host.match(WS2PConstants.HOST_ONION_REGEX) !== null) ? 'WS2PTOR':'WS2P'
......@@ -368,7 +368,7 @@ export class WS2PCluster {
to: { host, port, pubkey: pub }
})
await this.server.dal.setPeerUP(pub)
return ws2pc.connection
return ws2pc
} catch (e) {
this.server.logger.info(api+': Could not connect to peer %s using `'+api+' %s %s: %s`', pub.slice(0, 8), host, port, (e && e.message || e))
throw e
......
......@@ -12,18 +12,31 @@
// GNU Affero General Public License for more details.
import * as stream from "stream"
import { NewLogger } from "../../../lib/logger";
import { WS2PConnection } from "./WS2PConnection";
import {NewLogger} from "../../../lib/logger";
import {WS2PConnection} from "./WS2PConnection";
const logger = NewLogger()
export class WS2PStreamer extends stream.Transform {
private enabled = true
constructor(private ws2pc:WS2PConnection) {
super({ objectMode: true })
}
enable() {
this.enabled = true
}
disable() {
this.enabled = false
}
async _write(obj:any, enc:any, done:any) {
if (!this.enabled) {
return done && done()
}
try {
if (obj.joiners) {
await this.ws2pc.pushBlock(obj)
......
......@@ -11,11 +11,11 @@
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
import { IdentityForRequirements } from './../../../../service/BlockchainService';
import {IdentityForRequirements} from './../../../../service/BlockchainService';
import {Server} from "../../../../../server"
import {WS2PReqMapper} from "../interface/WS2PReqMapper"
import {BlockDTO} from "../../../../lib/dto/BlockDTO"
import { IindexEntry } from '../../../../lib/indexer';
import {IindexEntry} from '../../../../lib/indexer';
export class WS2PReqMapperByServer implements WS2PReqMapper {
......@@ -42,16 +42,27 @@ export class WS2PReqMapperByServer implements WS2PReqMapper {
}
async getRequirementsOfPending(minsig: number): Promise<any> {
let identities:IdentityForRequirements[] = await this.server.dal.idtyDAL.query(
let identities:IdentityForRequirements[] = (await this.server.dal.idtyDAL.query(
'SELECT i.*, count(c.sig) as nbSig ' +
'FROM idty i, cert c ' +
'WHERE c.target = i.hash group by i.hash having nbSig >= ?',
minsig)
const members:IdentityForRequirements[] = (await this.server.dal.idtyDAL.query(
[minsig])).map(i => ({
hash: i.hash || "",
member: i.member || false,
wasMember: i.wasMember || false,
pubkey: i.pubkey,
uid: i.uid || "",
buid: i.buid || "",
sig: i.sig || "",
revocation_sig: i.revocation_sig,
revoked: i.revoked,
revoked_on: i.revoked_on ? 1 : 0
}))
const members:IdentityForRequirements[] = (await this.server.dal.iindexDAL.query(
'SELECT i.*, count(c.sig) as nbSig ' +
'FROM i_index i, cert c ' +
'WHERE c.`to` = i.pub group by i.pub having nbSig >= ?',
minsig)).map((i:IindexEntry):IdentityForRequirements => {
[minsig])).map((i:IindexEntry):IdentityForRequirements => {
return {
hash: i.hash || "",
member: i.member || false,
......
......@@ -42,7 +42,7 @@ export interface IdentityForRequirements {
uid:string
buid:string
sig:string
revocation_sig:string
revocation_sig:string|null
revoked:boolean
revoked_on:number
}
......
......@@ -128,6 +128,7 @@ export class IdentityService extends FIFOService {
await GLOBAL_RULES_FUNCTIONS.checkIdentitiesAreWritable({ identities: [idtyObj.inline()], version: (current && current.version) || constants.BLOCK_GENERATED_VERSION }, this.conf, this.dal);
if (byAbsorption !== BY_ABSORPTION) {
if (!(await this.dal.idtyDAL.sandbox.acceptNewSandBoxEntry({
certsCount: 0,
issuers: [idty.pubkey],
ref_block: parseInt(idty.buid.split('-')[0])
}, this.conf.pair && this.conf.pair.pub))) {
......@@ -149,16 +150,16 @@ export class IdentityService extends FIFOService {
obj.currency = this.conf.currency || obj.currency;
const cert = CertificationDTO.fromJSONObject(obj)
const targetHash = cert.getTargetHash();
let idty = await this.dal.getIdentityByHashOrNull(targetHash);
let possiblyNullIdty = await this.dal.getIdentityByHashOrNull(targetHash);
let idtyAbsorbed = false
if (!idty) {
const idty: DBIdentity = possiblyNullIdty !== null ? possiblyNullIdty : await this.submitIdentity({
pubkey: cert.idty_issuer,
uid: cert.idty_uid,
buid: cert.idty_buid,
sig: cert.idty_sig
}, BY_ABSORPTION);
if (possiblyNullIdty === null) {
idtyAbsorbed = true
idty = await this.submitIdentity({
pubkey: cert.idty_issuer,
uid: cert.idty_uid,
buid: cert.idty_buid,
sig: cert.idty_sig
}, BY_ABSORPTION);
}
let anErr:any
const hash = cert.getHash()
......
......@@ -37,6 +37,8 @@ import {WS2PServer} from "../../../app/modules/ws2p/lib/WS2PServer"
import {WS2PServerMessageHandler} from "../../../app/modules/ws2p/lib/interface/WS2PServerMessageHandler"
import {TestUser} from "./TestUser"
import {RouterDependency} from "../../../app/modules/router"
import {ProverDependency} from "../../../app/modules/prover/index"
import {WS2PClient} from "../../../app/modules/ws2p/lib/WS2PClient"
const assert = require('assert');
const _ = require('underscore');
......@@ -338,6 +340,8 @@ export class TestingServer {
private port:number,
private server:Server) {
ProverDependency.duniter.methods.hookServer(server)
server.addEndpointsDefinitions(async () => {
return require('../../../app/modules/bma').BmaDependency.duniter.methods.getMainEndpoint(server.conf)
})
......@@ -681,7 +685,7 @@ export async function newWS2PBidirectionnalConnection(currency:string, k1:Key, k
})
}
export const simpleWS2PNetwork: (s1: TestingServer, s2: TestingServer) => Promise<{ w1: WS2PConnection; ws2pc: WS2PConnection; wss: WS2PServer, cluster1:WS2PCluster, cluster2:WS2PCluster }> = async (s1: TestingServer, s2: TestingServer) => {
export const simpleWS2PNetwork: (s1: TestingServer, s2: TestingServer) => Promise<{ w1: WS2PConnection; ws2pc: WS2PClient; wss: WS2PServer, cluster1:WS2PCluster, cluster2:WS2PCluster }> = async (s1: TestingServer, s2: TestingServer) => {
let port = getNewTestingPort()
const clientPub = s2.conf.pair.pub
let w1: WS2PConnection | null
......
......@@ -14,6 +14,8 @@
import {simpleTestingConf, simpleTestingServer, simpleUser, simpleWS2PNetwork, TestingServer} from "./tools/toolbox"
import {WS2PCluster} from "../../app/modules/ws2p/lib/WS2PCluster"
import {WS2PConstants} from "../../app/modules/ws2p/lib/constants"
import {WS2PClient} from "../../app/modules/ws2p/lib/WS2PClient"
import {TestUser} from "./tools/TestUser"
const assert = require('assert')
......@@ -24,10 +26,13 @@ describe("WS2P block pulling", function() {
const now = 1500000000
let s1:TestingServer, s2:TestingServer, wss:any
let ws2pc:WS2PClient
let cluster1:WS2PCluster
let cluster2:WS2PCluster
let cat:any, tac:any
let cat:TestUser, tac:TestUser, toc:TestUser
const catKeyring = { pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'}
const tacKeyring = { pub: '2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc', sec: '2HuRLWgKgED1bVio1tdpeXrf7zuUszv1yPHDsDj7kcMC4rVSN9RC58ogjtKNfTbH1eFz7rn38U1PywNs3m6Q7UxE'}
const tocKeyring = { pub: 'DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo', sec: '64EYRvdPpTfLGGmaX5nijLXRqWXaVz8r1Z1GtaahXwVSJGQRn7tqkxLb288zwSYzELMEG5ZhXSBYSxsTsz1m9y8F'}
let b0, b1, b2
......@@ -38,6 +43,7 @@ describe("WS2P block pulling", function() {
s2 = simpleTestingServer(conf2)
cat = simpleUser('cat', catKeyring, s1)
tac = simpleUser('tac', tacKeyring, s1)
toc = simpleUser('toc', tocKeyring, s2) // On S2
await s1.initDalBmaConnections()
await s2.initDalBmaConnections()