Commit 19483044 authored by Éloïs's avatar Éloïs
Browse files

wip dal

parent dd3b2f96
......@@ -730,6 +730,27 @@ dependencies = [
"serde",
]
[[package]]
name = "duniter-dal"
version = "0.1.0"
dependencies = [
"bincode",
"crossbeam-utils",
"dubp-common",
"dubp-wot",
"duniter-dbs",
"flate2",
"log",
"mockall",
"once_cell",
"rusqlite",
"serde",
"serde_json",
"tempdir",
"thiserror",
"unwrap",
]
[[package]]
name = "duniter-dbex"
version = "0.1.0"
......@@ -855,6 +876,18 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fastrand"
version = "1.4.0"
......@@ -1185,6 +1218,22 @@ version = "0.2.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa7087f49d294270db4e1928fc110c976cd4b9e5a16348e0a1df09afa99e6c98"
[[package]]
name = "libsqlite3-sys"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3a245984b1b06c291f46e27ebda9f369a94a1ab8461d0e845e23f9ced01f5db"
dependencies = [
"pkg-config",
"vcpkg",
]
[[package]]
name = "linked-hash-map"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a"
[[package]]
name = "lock_api"
version = "0.3.4"
......@@ -1218,6 +1267,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d0925aed5b12ed59857f438d25a910cf051dbcd4107907be1e7abf6c44ec903"
[[package]]
name = "lru-cache"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
dependencies = [
"linked-hash-map",
]
[[package]]
name = "maplit"
version = "1.0.2"
......@@ -1624,6 +1682,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33"
[[package]]
name = "plotters"
version = "0.2.15"
......@@ -1850,6 +1914,22 @@ dependencies = [
"winapi",
]
[[package]]
name = "rusqlite"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c78c3275d9d6eb684d2db4b2388546b32fdae0586c20a82f3905d21ea78b9ef"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"libsqlite3-sys",
"lru-cache",
"memchr",
"serde_json",
"smallvec",
]
[[package]]
name = "rust-argon2"
version = "0.8.2"
......@@ -2266,6 +2346,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e33648dd74328e622c7be51f3b40a303c63f93e6fa5f08778b6203a4c25c20f"
[[package]]
name = "vcpkg"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c"
[[package]]
name = "vec-arena"
version = "1.0.0"
......
......@@ -32,6 +32,7 @@ members = [
"rust-bins/duniter-dbex",
"rust-bins/xtask",
"rust-libs/dubp-wot",
"rust-libs/duniter-dal",
"rust-libs/duniter-dbs",
"rust-libs/tools/kv_typed",
"rust-libs/tools/kv_typed_code_gen"
......
[package]
name = "duniter-dal"
version = "0.1.0"
authors = ["elois <elois@duniter.org>"]
description = "Duniter Data Access Layer"
repository = "https://git.duniter.org/nodes/typescript/duniter"
keywords = ["duniter"]
license = "AGPL-3.0"
edition = "2018"
[lib]
path = "src/lib.rs"
[dependencies]
bincode = "1.2.1"
crossbeam-utils = "0.7.2"
dubp-common = { version = "0.25.2", features = ["crypto_scrypt"] }
dubp-wot = { path = "../dubp-wot" }
duniter-dbs = { path = "../duniter-dbs" }
flate2 = "1.0.16"
log = "0.4.8"
mockall = { version = "0.8.0", optional = true }
rusqlite = { version = "0.25.2", features = ["serde_json"] }
serde = { version = "1.0.105", features = ["derive"] }
serde_json = "1.0.53"
thiserror = "1.0.20"
[dev-dependencies]
once_cell = "1.4.0"
tempdir = "0.3.7"
unwrap = "1.2.1"
[features]
default = ["mock"]
#default = ["test_real"]
mock = ["duniter-dbs/mock", "mockall"]
test_real = []
use crate::*;
use duniter_dbs::kv_typed::backend::sled;
pub trait GenBackendConf<B: Backend> {
fn gen_backend_conf(db_name: &'static str, data_path_opt: Option<&Path>) -> B::Conf;
}
impl GenBackendConf<Mem> for Mem {
#[inline(always)]
fn gen_backend_conf(_db_name: &'static str, _data_path_opt: Option<&Path>) -> MemConf {
MemConf::default()
}
}
impl GenBackendConf<Sled> for Sled {
#[inline(always)]
fn gen_backend_conf(db_name: &'static str, data_path_opt: Option<&Path>) -> sled::Config {
if let Some(data_path) = data_path_opt {
sled::Config::default().path(data_path.join(format!("{}_sled", db_name)))
} else {
sled::Config::default().temporary(true)
}
}
}
impl GenBackendConf<LevelDb> for LevelDb {
#[inline(always)]
fn gen_backend_conf(_db_name: &'static str, data_path_opt: Option<&Path>) -> LevelDbConf {
if let Some(data_path) = data_path_opt {
LevelDbConf {
db_path: data_path.join("leveldb"),
..Default::default()
}
} else {
panic!("LevelDb backend not supported memory mode")
}
}
}
// Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! Duniter databases
use crate::*;
/// Blockchain database
pub type BcDb<B> = BcV1Db<B>;
/// Blockchain database on read-only mode
pub type BcDbRo<B> = BcV1DbRo<B>;
#[derive(Clone)]
pub struct DatabasesRo<B: Backend> {
pub(crate) bc_db: BcDbRo<B>,
pub(crate) peers_db: Arc<rusqlite::Connection>,
pub(crate) txs_mp_db: Arc<rusqlite::Connection>,
pub(crate) wot_mp_db: Arc<rusqlite::Connection>,
}
impl<B: Backend> Debug for DatabasesRo<B> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "DatabasesRo {{ ... }}")
}
}
#[derive(Clone)]
pub struct Databases<B: Backend> {
pub(crate) bc_db: BcDb<B>,
pub(crate) peers_db: Arc<rusqlite::Connection>,
pub(crate) txs_mp_db: Arc<rusqlite::Connection>,
pub(crate) wot_mp_db: Arc<rusqlite::Connection>,
ro: DatabasesRo<B>,
}
impl<B: Backend> Debug for Databases<B> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Databases {{ ... }}")
}
}
impl<B: Backend> Databases<B> {
pub(crate) fn to_ro(&self) -> &DatabasesRo<B> {
&self.ro
}
}
impl<B: Backend + GenBackendConf<B>> Databases<B> {
pub(crate) fn open(data_path_opt: Option<&Path>) -> DalResult<Databases<B>> {
let bc_db_conf = B::gen_backend_conf(BC_DB_NAME, data_path_opt);
let bc_db = BcV1Db::<B>::open(bc_db_conf)?;
let ro = DatabasesRo {
bc_db: bc_db.get_ro_handler(),
peers_db: Arc::new(open_sqlite_db(data_path_opt, PEERS_DB)?),
txs_mp_db: Arc::new(open_sqlite_db(data_path_opt, TX_MP_DB)?),
wot_mp_db: Arc::new(open_sqlite_db(data_path_opt, WOT_MP_SQLITE_DB)?),
};
Ok(Databases {
bc_db,
peers_db: ro.peers_db.clone(),
txs_mp_db: ro.txs_mp_db.clone(),
wot_mp_db: ro.wot_mp_db.clone(),
ro,
})
}
}
// TMP - Sqlite will be removed on the future !
#[inline(always)]
fn open_sqlite_db(
data_path_opt: Option<&Path>,
db_file_name: &str,
) -> DalResult<rusqlite::Connection> {
Ok(if let Some(data_path) = data_path_opt.as_deref() {
let mut sqlite_db_path = data_path.to_owned();
sqlite_db_path.pop();
sqlite_db_path.push(db_file_name);
rusqlite::Connection::open(sqlite_db_path)?
} else {
rusqlite::Connection::open(":memory:")?
})
}
/// Vector of string deserialized from JSON string
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct JsonVecStr(pub Vec<String>);
impl rusqlite::types::FromSql for JsonVecStr {
fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult<Self> {
let json_value = serde_json::Value::column_result(value)?;
if let serde_json::Value::Array(array) = json_value {
let mut vec_string = Vec::with_capacity(array.len());
for json_value in array {
if let serde_json::Value::String(string) = json_value {
vec_string.push(string);
} else {
return Err(rusqlite::types::FromSqlError::InvalidType);
}
}
Ok(JsonVecStr(vec_string))
} else {
Err(rusqlite::types::FromSqlError::InvalidType)
}
}
}
impl rusqlite::types::ToSql for JsonVecStr {
fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {
let json_string = serde_json::to_string(&self.0)
.map_err(|e| rusqlite::Error::ToSqlConversionFailure(Box::new(e)))?;
Ok(rusqlite::types::ToSqlOutput::Owned(
rusqlite::types::Value::Text(json_string),
))
}
}
// Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! Duniter DAL entities
pub mod peer;
use crate::*;
/// DUBP Block in database
pub type BlockDb = BlockDbV1;
// Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! Define peer entity
use crate::*;
/// DUNP Peer card in database
#[allow(missing_docs)]
#[derive(Debug, Default, Deserialize, Serialize)]
pub struct PeerCardDb {
pub version: i64,
pub currency: String,
pub status: String,
pub status_ts: i64,
pub hash: String,
pub first_down: Option<i64>,
pub last_try: Option<i64>,
pub last_contact: i64,
pub pubkey: String,
pub block: String,
pub signature: String,
pub endpoints: JsonVecStr,
pub raw: String,
pub non_wot: bool,
}
impl<'stmt> TryFrom<&rusqlite::Row<'stmt>> for PeerCardDb {
type Error = rusqlite::Error;
fn try_from(row: &rusqlite::Row<'stmt>) -> Result<Self, Self::Error> {
Ok(PeerCardDb {
version: row.get(0)?,
currency: row.get(1)?,
status: row.get(2)?,
status_ts: row.get(3)?,
hash: row.get(4)?,
first_down: row.get(5)?,
last_try: row.get(6)?,
last_contact: row.get(7)?,
pubkey: row.get(8)?,
block: row.get(9)?,
signature: row.get(10)?,
endpoints: row.get(11)?,
raw: row.get(12)?,
non_wot: row.get(13)?,
})
}
}
// Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! Define DAL errors
use crate::*;
/// Duniter DAL Result
pub type DalResult<T> = Result<T, DalError>;
#[derive(Debug, Error)]
/// Dal error
pub enum DalError {
/// Database error
#[error("{0}")]
DbError(ErrorDb),
/// Fail to create folder
#[error("Fail to create folder: {0}")]
FailToCreateFolder(std::io::Error),
/// Fail to open wot file
#[error("Fail to open wot file: {0}")]
FailToOpenWotFile(bincode::Error),
/// Fail to save wot file
#[error("Fail to save wot file: {0}")]
FailToSaveWotFile(bincode::Error),
/// Fail to get currency parameters from genesis block
#[error("Fail to get genesis parameters: {0}")]
GenesisParamsError(String),
/// No blockchain
#[error("No blockchain")]
NoBlockchain,
/// Sqlite error
#[error("Sqlite error: {0}")]
SqLiteError(rusqlite::Error),
/// Wrong backend
#[error("Database already exist with {exist_backend} bakend, please use them.")]
WrongBackend {
/// Existing bakend
exist_backend: &'static str,
},
}
impl From<KvError> for DalError {
fn from(e: KvError) -> Self {
Self::DbError(ErrorDb::DbError(format!("{}", e)))
}
}
impl From<ErrorDb> for DalError {
fn from(e: ErrorDb) -> Self {
Self::DbError(e)
}
}
impl From<rusqlite::Error> for DalError {
fn from(e: rusqlite::Error) -> Self {
Self::SqLiteError(e)
}
}
// Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! Duniter file data access layer
use crate::*;
#[derive(Clone, Debug)]
/// Duniter Data Access Layer on read-only mode that read data from filesystem
pub struct FileDalRo<B: Backend> {
pub(crate) databases: DatabasesRo<B>,
pub(crate) currency_params: CurrencyParameters,
pub(crate) wot: Wot,
}
#[derive(Clone, Debug)]
/// Duniter Data Access Layer that store data on filesystem
pub struct FileDal<B: Backend> {
pub(crate) databases: Databases<B>,
pub(crate) currency_params: CurrencyParameters,
pub(crate) wot: Wot,
}
impl<B: Backend + GenBackendConf<B>> FileDal<B> {
#[inline(always)]
/// Initialise file DAL
pub fn init<P: AsRef<Path>>(profile_path: P) -> DalResult<Self> {
let data_path = profile_path.as_ref().join(DATA_DIR);
verify_data_path(&data_path, B::NAME)?;
let databases = Databases::open(Some(data_path.as_path()))?;
let currency_params = get_currency_params(&databases.bc_db, data_path.as_path())?;
let wot = Wot::open(Some(data_path.as_path()), currency_params.sig_stock)
.map_err(DalError::FailToOpenWotFile)?;
Ok(FileDal {
currency_params,
databases,
wot,
})
}
}
impl<B: Backend> ToDalRo for FileDal<B> {
type DalRo = FileDalRo<B>;
fn to_dal_ro(&self) -> Self::DalRo {
FileDalRo {
databases: self.databases.to_ro().clone(),
currency_params: self.currency_params,
wot: self.wot.clone(),
}
}
}
fn get_currency_params<B: Backend>(
bc_db: &BcDb<B>,
data_path: &Path,
) -> DalResult<CurrencyParameters> {
Ok(if let Some(currency_params) = get_genesis_params(&bc_db)? {
currency_params
} else {
let mut file = fs::File::open(data_path.join(GENESIS_PARAMS_FILE))
.map_err(|e| DalError::GenesisParamsError(format!("{}", e)))?;
let mut file_contents = String::new();
file.read_to_string(&mut file_contents)
.map_err(|e| DalError::GenesisParamsError(format!("{}", e)))?;
serde_json::from_str::<CurrencyNameAndGenesisBlockParams>(&file_contents)
.map_err(|e| DalError::GenesisParamsError(format!("{}", e)))?
.into()
})
}
fn get_genesis_params<B: Backend>(bc_db: &BcDb<B>) -> DalResult<Option<CurrencyParameters>> {
let genesis_opt = bc_db
.main_blocks()
.get(&duniter_dbs::BlockNumberKeyV1(BlockNumber(0)))?;
Ok(if let Some(genesis) = genesis_opt {
Some(CurrencyParameters::from((
&CurrencyName(genesis.currency),
dubp_common::currency_params::BlockV10Parameters::from_str(&genesis.parameters)
.map_err(|e| DalError::GenesisParamsError(format!("{}", e)))?,
)))
} else {
None
})