Commit ad7fc6ac authored by Éloïs's avatar Éloïs
Browse files

Merge branch 'feat/dex' into 'dev'

Feat/dex

See merge request nodes/typescript/duniter!1333
parents dd277f33 4357eb07
......@@ -24,7 +24,7 @@ workflow:
- server.ts
.env: &env
image: registry.duniter.org/docker/duniter-ci:v0.0.4
image: registry.duniter.org/docker/duniter-ci:v0.2.0
tags:
- redshift
before_script:
......
This diff is collapsed.
......@@ -29,8 +29,12 @@ rusty-hook = "0.11.2"
[workspace]
members = [
"neon/native",
"rust-bins/duniter-dbex",
"rust-bins/xtask",
"rust-libs/dubp-wot"
"rust-libs/dubp-wot",
"rust-libs/duniter-dbs",
"rust-libs/tools/kv_typed",
"rust-libs/tools/kv_typed_code_gen"
]
[patch.crates-io]
......@@ -38,3 +42,4 @@ members = [
#dubp-documents = { path = "../dubp-rs-libs/documents" }
#dubp-documents-parser = { path = "../dubp-rs-libs/documents-parser" }
#dubp-wallet = { path = "../dubp-rs-libs/wallet" }
#leveldb_minimal = { path = "../../../../rust/leveldb_minimal" }
[package]
name = "duniter-dbex"
version = "0.1.0"
authors = ["elois <elois@duniter.org>"]
description = "Duniter blockchain DB"
repository = "https://git.duniter.org/nodes/typescript/duniter/rust-bins/duniter-dbs-explorer"
readme = "README.md"
keywords = ["duniter", "database"]
license = "AGPL-3.0"
edition = "2018"
[[bin]]
bench = false
path = "src/main.rs"
name = "dex"
[build-dependencies]
structopt = "0.3.16"
[dependencies]
arrayvec = "0.5.1"
comfy-table = "1.0.0"
dirs = "3.0.1"
dubp-common = { version = "0.25.2", features = ["crypto_scrypt"] }
duniter-dbs = { path = "../../rust-libs/duniter-dbs", default-features = false, features = ["explorer", "leveldb_backend", "sync"] }
rayon = "1.3.1"
serde_json = "1.0.53"
structopt = "0.3.16"
[dev-dependencies]
serde = { version = "1.0.105", features = ["derive"] }
unwrap = "1.2.1"
# Duniter databases explorer (dex)
## Compile
git clone https://git.duniter.org/nodes/typescript/duniter.git
cd duniter
cargo build --release -p duniter-dbex
The binary executable is then here: `target/release/dex`
## Use
See `dex --help`
## Autocompletion
Bash autocompletion script is available here : `target/release/dex.bash`
**Several others Shell are supported : Zsh, Fish, Powershell and Elvish!**
To generate the autocompletion script for your shell, recompile with env var `COMPLETION_SHELL`.
For exemple for fish : `COMPLETION_SHELL=fish cargo build --release -p duniter-dbex`
The autocompletion script can be found in : `target/release/`
// Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
extern crate structopt;
include!("src/cli.rs");
use std::env;
use structopt::clap::Shell;
fn main() {
// Define out dir
let current_dir = match env::current_dir() {
Err(_e) => return,
Ok(current_dir) => current_dir,
};
let out_dir = current_dir.as_path().join(format!(
"../../target/{}",
env::var("PROFILE").unwrap_or_else(|_| "debug".to_owned())
));
// Define shell
let shell = if let Some(shell_str) = option_env!("COMPLETION_SHELL") {
Shell::from_str(shell_str).expect("Unknown shell")
} else {
Shell::Bash
};
let mut app = Opt::clap();
app.gen_completions(
"dex", // We need to specify the bin name manually
shell, // Then say which shell to build completions for
out_dir,
); // Then say where write the completions to
}
// Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{num::NonZeroUsize, path::PathBuf, str::FromStr};
use structopt::StructOpt;
#[derive(Debug, StructOpt)]
#[structopt(name = "duniter-dbex", about = "Duniter databases explorer.")]
pub struct Opt {
/// Duniter profile name
#[structopt(short, long)]
pub profile: Option<String>,
/// Duniter home directory
#[structopt(short, long, parse(from_os_str))]
pub home: Option<PathBuf>,
/// database
#[structopt(default_value = "bc_v1", possible_values = &["bc_v1", "bc_v2", "mp_v1"])]
pub database: Database,
#[structopt(subcommand)]
pub cmd: SubCommand,
}
#[derive(Debug)]
pub enum Database {
BcV1,
}
impl FromStr for Database {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"bc_v1" => Ok(Self::BcV1),
"bc_v2" | "mp_v1" => unimplemented!(),
_ => unreachable!(),
}
}
}
#[derive(Debug, StructOpt)]
pub enum SubCommand {
/// Count collection entries
Count { collection: String },
/// Get one value
Get { collection: String, key: String },
/// Search values by criteria
Find {
collection: String,
#[structopt(short, long)]
/// Key min
start: Option<String>,
#[structopt(short, long)]
/// Key max
end: Option<String>,
/// Filter keys by a regular expression
#[structopt(short = "k", long)]
key_regex: Option<String>,
/// Show keys only
#[structopt(long)]
keys_only: bool,
/// Filter values by a regular expression
#[structopt(short = "v", long)]
value_regex: Option<String>,
/// Maximum number of entries to be found (Slower because force sequential search)
#[structopt(short, long)]
limit: Option<usize>,
/// Browse the collection upside down
#[structopt(short, long)]
reverse: bool,
/// Step by
#[structopt(long, default_value = "1")]
step: NonZeroUsize,
/// Output format
#[structopt(short, long, default_value = "table-json", possible_values = &["csv", "json", "table-json", "table-properties"])]
output: OutputFormat,
/// Pretty json (Only for output format json or table-json)
#[structopt(long)]
pretty: bool,
/// Show only the specified properties
#[structopt(short, long)]
properties: Vec<String>,
/// Export found data to a file
#[structopt(short, long, parse(from_os_str))]
file: Option<PathBuf>,
},
/// Show database schema
Schema,
}
#[derive(Clone, Copy, Debug)]
pub enum OutputFormat {
Table,
TableJson,
Json,
Csv,
}
impl FromStr for OutputFormat {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"csv" => Ok(Self::Csv),
"json" => Ok(Self::Json),
"table-properties" => Ok(Self::Table),
"table-json" => Ok(Self::TableJson),
_ => unreachable!(),
}
}
}
// Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
#![deny(
clippy::unwrap_used,
missing_copy_implementations,
trivial_casts,
trivial_numeric_casts,
unstable_features,
unused_import_braces
)]
mod cli;
mod print_found_data;
mod stringify_json_value;
use self::cli::{Database, Opt, OutputFormat, SubCommand};
use self::stringify_json_value::stringify_json_value;
use comfy_table::Table;
use duniter_dbs::kv_typed::prelude::*;
use duniter_dbs::prelude::*;
use duniter_dbs::regex::Regex;
use duniter_dbs::serde_json::{Map, Value};
use duniter_dbs::smallvec::{smallvec, SmallVec};
use duniter_dbs::BcV1Db;
use duniter_dbs::BcV1DbWritable;
use rayon::prelude::*;
use std::{
collections::{HashMap, HashSet},
fs::File,
io::{stdin, Write},
iter::FromIterator,
time::Instant,
};
use structopt::StructOpt;
const DATA_DIR: &str = "data";
const TOO_MANY_ENTRIES_ALERT: usize = 5_000;
fn main() -> Result<(), String> {
let opt = Opt::from_args();
let home = if let Some(home) = opt.home {
home
} else {
dirs::config_dir()
.ok_or_else(|| {
"Fail to auto find duniter's home directory, please specify it explicitly."
.to_owned()
})?
.as_path()
.join("duniter")
};
let profile_name = if let Some(profile_name) = opt.profile {
profile_name
} else {
"duniter_default".to_owned()
};
let profile_path = home.as_path().join(&profile_name);
let data_path = profile_path.as_path().join(DATA_DIR);
if !data_path.exists() {
return Err(format!(
"Path '{}' don't exist !",
data_path.to_str().expect("non-UTF-8 strings not supported")
));
}
let open_db_start_time = Instant::now();
match opt.database {
Database::BcV1 => apply_subcommand(
BcV1Db::<LevelDb>::open(LevelDbConf {
db_path: data_path.as_path().join("leveldb"),
..Default::default()
})
.map_err(|e| format!("{}", e))?,
opt.cmd,
open_db_start_time,
),
}
}
fn apply_subcommand<DB: DbExplorable>(
db: DB,
cmd: SubCommand,
open_db_start_time: Instant,
) -> Result<(), String> {
let duration = open_db_start_time.elapsed();
println!(
"Database opened in {}.{:06} seconds.",
duration.as_secs(),
duration.subsec_micros()
);
let start_time = Instant::now();
match cmd {
SubCommand::Count { collection } => {
if let ExplorerActionResponse::Count(count) = db
.explore(&collection, ExplorerAction::Count, stringify_json_value)
.map_err(|e| format!("{}", e))?
.map_err(|e| e.0)?
{
let duration = start_time.elapsed();
println!(
"Count operation performed in {}.{:06} seconds.",
duration.as_secs(),
duration.subsec_micros()
);
println!("\nThis collection contains {} entries.", count);
}
}
SubCommand::Get { collection, key } => {
if let ExplorerActionResponse::Get(value_opt) = db
.explore(
&collection,
ExplorerAction::Get { key: &key },
stringify_json_value,
)
.map_err(|e| format!("{}", e))?
.map_err(|e| e.0)?
{
if let Some(value) = value_opt {
println!("\n{}", value)
} else {
println!("\nThis collection not contains this key.")
}
}
}
SubCommand::Find {
collection,
start,
end,
key_regex,
keys_only,
value_regex,
limit,
reverse,
properties,
output: output_format,
pretty: pretty_json,
file: output_file,
step,
} => {
let value_regex_opt = opt_string_to_res_opt_regex(value_regex)?;
let captures_headers = if let Some(ref value_regex) = value_regex_opt {
value_regex
.capture_names()
.skip(1)
.enumerate()
.map(|(i, name_opt)| {
if let Some(name) = name_opt {
name.to_owned()
} else {
format!("CAP{}", i + 1)
}
})
.collect()
} else {
vec![]
};
if let ExplorerActionResponse::Find(entries) = db
.explore(
&collection,
ExplorerAction::Find {
key_min: start,
key_max: end,
key_regex: opt_string_to_res_opt_regex(key_regex)?,
value_regex: value_regex_opt,
limit,
reverse,
step,
},
stringify_json_value,
)
.map_err(|e| format!("{}", e))?
.map_err(|e| e.0)?
{
let duration = start_time.elapsed();
println!(
"Search performed in {}.{:06} seconds.\n\n{} entries found.",
duration.as_secs(),
duration.subsec_micros(),
entries.len()
);
if !too_many_entries(entries.len(), output_file.is_none())
.map_err(|e| format!("{}", e))?
{
return Ok(());
}
let start_print = Instant::now();
if let Some(output_file) = output_file {
let mut file =
File::create(output_file.as_path()).map_err(|e| format!("{}", e))?;
//let mut file_buffer = BufWriter::new(file);
print_found_data::print_found_data(
&mut file,
output_format,
pretty_json,
false,
print_found_data::DataToShow {
entries,
keys_only,
only_properties: properties,
},
captures_headers,
)
.map_err(|e| format!("{}", e))?;
//file_buffer.flush().map_err(|e| format!("{}", e))?;
let export_duration = start_print.elapsed();
println!(
"Search results were written to file: '{}' in {}.{:06} seconds.",
output_file
.to_str()
.expect("output-file contains invalid utf8 characters"),
export_duration.as_secs(),
export_duration.subsec_micros(),
);
} else {
print_found_data::print_found_data(
&mut std::io::stdout(),
output_format,
pretty_json,
true,
print_found_data::DataToShow {
entries,
keys_only,
only_properties: properties,
},
captures_headers,
)
.map_err(|e| format!("{}", e))?;
let print_duration = start_print.elapsed();
println!(
"Search results were displayed in {}.{:06} seconds.",
print_duration.as_secs(),
print_duration.subsec_micros(),
);
};
}
}
SubCommand::Schema => {
show_db_schema(db.list_collections());
}
};
Ok(())
}
fn too_many_entries(entries_len: usize, output_in_term: bool) -> std::io::Result<bool> {
if entries_len > TOO_MANY_ENTRIES_ALERT {
println!(
"{} all {} entries ? (Be careful, may crash your system!) [y/N]",
if output_in_term { "Display" } else { "Export" },
entries_len
);
let mut buffer = String::new();
stdin().read_line(&mut buffer)?;
Ok(buffer == "y\n")
} else {
Ok(true)
}
}
fn show_db_schema(collections_names: Vec<(&'static str, &'static str, &'static str)>) {
let mut table = Table::new();
table.set_content_arrangement(comfy_table::ContentArrangement::Dynamic);
table.set_header(&["Collection name", "Key type", "Value type"]);
for (collection_name, key_full_type_name, value_full_type_name) in collections_names {
let key_type_name_opt = key_full_type_name.split(':').last();
let value_type_name_opt = value_full_type_name.split(':').last();
table.add_row(&[
collection_name,
key_type_name_opt.unwrap_or("?"),
value_type_name_opt.unwrap_or("?"),
]);
}
println!("{}", table);
}
#[inline]
fn opt_string_to_res_opt_regex(str_regex_opt: Option<String>) -> Result<Option<Regex>, String> {
if let Some(str_regex) = str_regex_opt {
Ok(Some(Regex::new(&str_regex).map_err(|e| format!("{}", e))?))
} else {
Ok(None)
}
}
// Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::*;
const KEY_COLUMN_NAME: &str = "Key";
const VALUE_COLUMN_NAME: &str = "Value";
pub struct DataToShow {
pub entries: Vec<EntryFound>,
pub keys_only: bool,
pub only_properties: Vec<String>,
}
pub fn print_found_data<W: Write>(
output: &mut W,
output_format: OutputFormat,
pretty_json: bool,
dynamic_content_arrangement: bool,
data_to_show: DataToShow,
captures_names: Vec<String>,
) -> std::io::Result<()> {
let DataToShow {
entries,