Commit bacee06d authored by Éloïs's avatar Éloïs

[ref] preparte sync command to integration tests

parent 0b799b51
......@@ -318,6 +318,7 @@ dependencies = [
"serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
"structopt 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
......
......@@ -22,16 +22,14 @@ pub mod keys;
pub mod modules;
pub mod reset;
pub mod start;
pub mod sync;
use cli::keys::KeysOpt;
pub use cli::keys::KeysOpt;
pub use dbex::*;
pub use keys::*;
pub use duniter_network::cli::sync::SyncOpt;
use log::Level;
pub use modules::*;
pub use reset::*;
pub use start::*;
pub use sync::*;
#[derive(StructOpt, Debug)]
#[structopt(
......@@ -68,11 +66,8 @@ pub enum CoreSubCommand {
/// start durs server
StartOpt(StartOpt),
#[structopt(name = "sync")]
/// synchronization from network
/// synchronization
SyncOpt(SyncOpt),
#[structopt(name = "sync_ts")]
/// synchronization via a duniter-ts database
SyncTsOpt(SyncTsOpt),
/// reset data or conf or all
#[structopt(
name = "reset",
......
......@@ -50,9 +50,10 @@ pub mod change_conf;
pub mod cli;
pub mod router;
pub use duniter_conf::{ChangeGlobalConf, DuRsConf, DuniterKeyPairs, KEYPAIRS_FILENAME};
pub use duniter_conf::{keys::*, ChangeGlobalConf, DuRsConf, DuniterKeyPairs, KEYPAIRS_FILENAME};
use duniter_module::*;
use duniter_network::{NetworkModule, SyncEndpoint, SyncParams};
use duniter_network::cli::sync::*;
use duniter_network::NetworkModule;
use durs_blockchain::{BlockchainModule, DBExQuery, DBExTxQuery, DBExWotQuery};
use durs_message::*;
use log::Level;
......@@ -61,7 +62,6 @@ use simplelog::*;
//use std::fmt::{Debug, Formatter};
use cli::keys::*;
use cli::*;
use duniter_conf::keys;
use std::fs;
use std::fs::{File, OpenOptions};
use std::sync::mpsc;
......@@ -140,7 +140,7 @@ pub enum UserCommand {
/// Start
Start(),
/// Sync (SyncEndpoint)
Sync(SyncParams),
Sync(SyncOpt),
/// List modules
ListModules(ListModulesOpt),
/// Unknow command
......@@ -307,33 +307,18 @@ impl<'a, 'b: 'a> DuniterCore<'b, 'a, DuRsConf> {
true
} else if let Some(matches) = cli_args.subcommand_matches("sync") {
let opts = SyncOpt::from_clap(matches);
let sync_endpoint = SyncEndpoint {
domain_or_ip: opts.host,
port: opts.port,
path: opts.path,
tls: false,
};
// Store sync command parameters
self.user_command = Some(UserCommand::Sync(SyncParams {
sync_endpoint,
cautious: opts.cautious_mode,
verif_hashs: opts.unsafe_mode,
}));
// Start router thread
self.router_sender = Some(router::start_router(0, profile.clone(), conf, vec![]));
true
} else if let Some(matches) = cli_args.subcommand_matches("sync_ts") {
let opts = SyncTsOpt::from_clap(matches);
let ts_profile = opts
.ts_profile
.unwrap_or_else(|| String::from("duniter_default"));
sync_ts(
profile.as_str(),
&conf,
&ts_profile,
opts.cautious_mode,
opts.unsafe_mode,
);
match opts.source_type {
SyncSourceType::Network => unimplemented!(),
SyncSourceType::TsSqlDb => {
sync_ts(
profile.as_str(),
&conf,
&opts
);
}
SyncSourceType::JsonFiles => unimplemented!(),
}
false
} else if let Some(matches) = cli_args.subcommand_matches("dbex") {
let opts = DbExOpt::from_clap(matches);
......@@ -684,12 +669,10 @@ pub fn match_profile(cli_args: &ArgMatches) -> String {
pub fn sync_ts<DC: DuniterConf>(
profile: &str,
conf: &DC,
ts_profile: &str,
cautious: bool,
verif_inner_hash: bool,
sync_opts: &SyncOpt,
) {
// Launch sync-ts
BlockchainModule::sync_ts(profile, conf, ts_profile, cautious, verif_inner_hash);
BlockchainModule::sync_ts(profile, conf, sync_opts);
}
/// Launch databases explorer
......
......@@ -16,6 +16,7 @@ durs-network-documents = { path = "../../tools/network-documents" }
serde = "1.0.*"
serde_derive = "1.0.*"
serde_json = "1.0.*"
structopt= "0.2.*"
[dev-dependencies]
......
// Copyright (C) 2018 The Duniter Project Developers.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! Durs network cli
pub mod sync;
......@@ -13,44 +13,53 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! Durs-core cli : sync subcommands.
//! Durs network cli : sync subcommands.
extern crate structopt;
use std::str::FromStr;
#[derive(StructOpt, Debug, Clone)]
#[structopt(
name = "sync",
raw(setting = "structopt::clap::AppSettings::ColoredHelp")
)]
/// synchronization from network
/// Synchronization from network
pub struct SyncOpt {
/// The domain name or ip address of the node from which to synchronize.
pub host: String,
/// The port number of the node from which to synchronize.
pub port: u16,
/// The endpoint path of the node from which to synchronize.
pub path: Option<String>,
#[structopt(short = "c", long = "cautious")]
/// The source of datas (url of the node from which to synchronize OR path to local file)
pub source: Option<String>,
/// The source type
#[structopt(short = "t", long = "type", default_value = "ts")]
pub source_type: SyncSourceType,
/// End block
#[structopt(short = "e", long = "end")]
pub end: Option<u32>,
/// cautious mode (check all protocol rules, very slow)
#[structopt(short = "c", long = "cautious")]
pub cautious_mode: bool,
#[structopt(short = "u", long = "unsafe")]
/// unsafe mode (not check blocks inner hashs, very dangerous)
#[structopt(short = "u", long = "unsafe")]
pub unsafe_mode: bool,
}
#[derive(StructOpt, Debug, Clone)]
#[structopt(
name = "sync",
raw(setting = "structopt::clap::AppSettings::ColoredHelp")
)]
/// synchronization via a duniter-ts database
pub struct SyncTsOpt {
/// Set the ts profile to use
pub ts_profile: Option<String>,
#[structopt(short = "c", long = "cautious")]
/// cautious mode (check all protocol rules, very slow)
pub cautious_mode: bool,
#[structopt(short = "u", long = "unsafe")]
/// unsafe mode (not check blocks inner hashs, very dangerous)
pub unsafe_mode: bool,
/// The source of blocks datas
#[derive(Debug, Copy, Clone)]
pub enum SyncSourceType {
/// Sync from network
Network,
/// Sync from Duniter-ts sqlite bdd
TsSqlDb,
/// Sync from json blocks in files
JsonFiles,
}
impl FromStr for SyncSourceType {
type Err = String;
fn from_str(source: &str) -> Result<Self, Self::Err> {
match source {
"n" | "network" => Ok(SyncSourceType::Network),
"ts" | "ts-sql" => Ok(SyncSourceType::TsSqlDb),
"json" => Ok(SyncSourceType::JsonFiles),
&_ => Err("Unknown source type".to_owned()),
}
}
}
......@@ -34,7 +34,10 @@ extern crate dup_crypto;
extern crate durs_network_documents;
extern crate serde;
extern crate serde_json;
#[macro_use]
extern crate structopt;
use cli::sync::SyncOpt;
use duniter_module::*;
use durs_network_documents::network_endpoint::ApiFeatures;
use durs_network_documents::network_head::NetworkHead;
......@@ -42,6 +45,7 @@ use durs_network_documents::*;
use std::fmt::Debug;
use std::sync::mpsc;
pub mod cli;
pub mod documents;
pub mod events;
pub mod requests;
......@@ -62,34 +66,10 @@ pub trait NetworkModule<DC: DuniterConf, M: ModuleMessage>: ApiModule<DC, M> {
keys: RequiredKeysContent,
module_conf: <Self as DuniterModule<DC, M>>::ModuleConf,
main_sender: mpsc::Sender<RouterThreadMessage<M>>,
sync_params: SyncParams,
sync_params: SyncOpt,
) -> Result<(), ModuleInitError>;
}
/// SyncParams
#[derive(Debug, Clone)]
pub struct SyncParams {
/// Synchronisation endpoint
pub sync_endpoint: SyncEndpoint,
/// Cautious flag
pub cautious: bool,
/// VERIF_HASHS flag
pub verif_hashs: bool,
}
#[derive(Debug, Clone)]
/// Synchronisation endpoint
pub struct SyncEndpoint {
/// Domaine name or IP
pub domain_or_ip: String,
/// Port number
pub port: u16,
/// Optionnal path
pub path: Option<String>,
/// Use TLS
pub tls: bool,
}
/// Trait to be implemented by the configuration object of the module managing the inter-node network.
pub trait NetworkConf: Debug + Copy + Clone + PartialEq {}
......
......@@ -67,6 +67,7 @@ use dubp_documents::*;
use dubp_documents::{DUBPDocument, Document};
use duniter_module::*;
use duniter_network::{
cli::sync::SyncOpt,
documents::{BlockchainDocument, NetworkBlock},
events::NetworkEvent,
requests::{NetworkResponse, OldNetworkRequest},
......@@ -209,22 +210,25 @@ impl BlockchainModule {
pub fn sync_ts<DC: DuniterConf>(
profile: &str,
conf: &DC,
ts_profile: &str,
cautious: bool,
verif_inner_hash: bool,
sync_opts: &SyncOpt,
) {
// get db_ts_path
let mut db_ts_path = match dirs::config_dir() {
Some(path) => path,
None => panic!("Impossible to get user config directory !"),
let db_ts_path = if let Some(ref ts_path) = sync_opts.source {
PathBuf::from(ts_path)
} else {
let mut db_ts_path = match dirs::config_dir() {
Some(path) => path,
None => panic!("Impossible to get user config directory !"),
};
db_ts_path.push("duniter/");
db_ts_path.push("duniter_default");
db_ts_path.push("duniter.db");
db_ts_path
};
db_ts_path.push("duniter/");
db_ts_path.push(ts_profile);
db_ts_path.push("duniter.db");
if !db_ts_path.as_path().exists() {
panic!("Fatal error : duniter-ts database don't exist !");
}
sync::sync_ts(profile, conf, db_ts_path, cautious, verif_inner_hash);
sync::sync_ts(profile, conf, db_ts_path, sync_opts.end, sync_opts.cautious_mode, !sync_opts.unsafe_mode);
}
/// Request chunk from network (chunk = group of blocks)
fn request_chunk(&self, req_id: ModuleReqId, from: u32) -> (ModuleReqId, OldNetworkRequest) {
......
......@@ -71,6 +71,7 @@ pub fn sync_ts<DC: DuniterConf>(
profile: &str,
conf: &DC,
db_ts_path: PathBuf,
end: Option<u32>,
cautious: bool,
verif_inner_hash: bool,
) {
......@@ -114,15 +115,24 @@ pub fn sync_ts<DC: DuniterConf>(
.expect("Fatal error : fail to open copy of duniter-ts database !");
info!("sync_ts : Success to open duniter-ts database.");
// Get ts current blockstamp
// Get ts target blockstamp
debug!("Get ts-db current blockstamp...");
let mut cursor: sqlite::Cursor = ts_db
let mut cursor: sqlite::Cursor = if let Some(end) = end {
let mut cursor = ts_db
.prepare("SELECT hash, number, currency FROM block WHERE fork=? AND number=? LIMIT 1;")
.expect("Request SQL get_ts_current_block is wrong !")
.cursor();
cursor.bind(&[sqlite::Value::Integer(0), sqlite::Value::Integer(i64::from(end))]).expect("Fail to get ts target block !");
cursor
} else {
let mut cursor = ts_db
.prepare("SELECT hash, number, currency FROM block WHERE fork=? ORDER BY number DESC LIMIT 1;")
.expect("Request SQL get_ts_current_block is wrong !")
.cursor();
cursor
.bind(&[sqlite::Value::Integer(0)])
.expect("Fail to get ts current block !");
cursor.bind(&[sqlite::Value::Integer(0)]).expect("Fail to get ts current block !");
cursor
};
let (currency, current_ts_blockstamp) =
if let Some(row) = cursor.next().expect("cursor error") {
let block_id = BlockId(
......@@ -201,7 +211,7 @@ pub fn sync_ts<DC: DuniterConf>(
previousIssuer, version, membersCount, monetaryMass, medianTime, dividend, unitbase,
time, powMin, number, nonce, transactions, certifications, identities, joiners,
actives, leavers, revoked, excluded, issuersFrame, issuersFrameVar, issuersCount
FROM block WHERE fork=? AND number > ? ORDER BY number ASC;",
FROM block WHERE fork=? AND number > ? AND number <= ? ORDER BY number ASC;",
)
.expect("Request SQL get_ts_blocks is wrong !")
.cursor();
......@@ -209,6 +219,7 @@ pub fn sync_ts<DC: DuniterConf>(
.bind(&[
sqlite::Value::Integer(0),
sqlite::Value::Integer(i64::from(current_blockstamp.id.0)),
sqlite::Value::Integer(i64::from(current_ts_blockstamp.id.0)),
])
.expect("0");
......@@ -376,6 +387,8 @@ pub fn sync_ts<DC: DuniterConf>(
// Increment progress bar (last chunk)
apply_pb.inc();
// Save blockchain, and fork databases
println!();
println!("Write indexs in files...");
info!("Save blockchain and forks databases in files...");
databases.save_dbs();
......@@ -650,16 +663,16 @@ pub fn sync_ts<DC: DuniterConf>(
// Wait recv two finish signals
let mut wait_jobs = *NB_SYNC_JOBS - 1;
while wait_jobs > 0 {
if let Ok(MessForSyncThread::ApplyFinish()) = recv_sync_thread.recv() {
wait_jobs -= 1;
} else {
thread::sleep(Duration::from_millis(50));
match recv_sync_thread.recv() {
Ok(MessForSyncThread::ApplyFinish()) => wait_jobs -= 1,
Ok(_) => thread::sleep(Duration::from_millis(50)),
Err(_) => wait_jobs -= 1,
}
}
info!("All sync jobs finish.");
// Log sync duration
println!("certs_count={}", certs_count);
debug!("certs_count={}", certs_count);
let sync_duration = SystemTime::now().duration_since(sync_start_time).unwrap();
println!(
"Sync {} blocks in {}.{:03} seconds.",
......
......@@ -67,6 +67,7 @@ use datas::*;
use dubp_documents::Blockstamp;
use duniter_conf::DuRsConf;
use duniter_module::*;
use duniter_network::cli::sync::SyncOpt;
use duniter_network::documents::*;
use duniter_network::events::*;
use duniter_network::requests::*;
......@@ -229,7 +230,7 @@ impl NetworkModule<DuRsConf, DursMsg> for WS2PModule {
_keys: RequiredKeysContent,
_conf: WS2PConf,
_main_sender: mpsc::Sender<RouterThreadMessage<DursMsg>>,
_sync_params: SyncParams,
_sync_params: SyncOpt,
) -> Result<(), ModuleInitError> {
println!("Downlaod blockchain from network...");
println!("Error : not yet implemented !");
......
......@@ -127,7 +127,7 @@ impl NetworkModule<DuRsConf, DursMsg> for WS2Pv2Module {
_keys: RequiredKeysContent,
_conf: WS2PConf,
_main_sender: mpsc::Sender<RouterThreadMessage<DursMsg>>,
_sync_params: SyncParams,
_sync_params: SyncOpt,
) -> Result<(), ModuleInitError> {
unimplemented!()
}
......
......@@ -52,6 +52,7 @@ pub mod services;
use constants::*;
use duniter_conf::DuRsConf;
use duniter_module::*;
use duniter_network::cli::sync::SyncOpt;
use duniter_network::*;
use durs_message::DursMsg;
use durs_network_documents::network_endpoint::*;
......@@ -127,7 +128,7 @@ impl NetworkModule<DuRsConf, DursMsg> for WS2Pv2Module {
_keys: RequiredKeysContent,
_conf: WS2PConf,
_main_sender: mpsc::Sender<RouterThreadMessage<DursMsg>>,
_sync_params: SyncParams,
_sync_params: SyncOpt,
) -> Result<(), ModuleInitError> {
unimplemented!()
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment