diff --git a/Cargo.lock b/Cargo.lock index f8273605b557cec3dea9a2621d8d01ca846ee9a2..c396643e53f8d567a7e3c5ce006b1b52f99789ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1031,6 +1031,7 @@ dependencies = [ "rkv 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "rustbreak 2.0.0-rc3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "sled 0.31.0 (git+https://github.com/librelois/sled?branch=librelois_impl_transactional_for_vec_tree)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1407,6 +1408,15 @@ name = "fragile" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "fuchsia-zircon" version = "0.3.3" @@ -2598,6 +2608,21 @@ name = "slab" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "sled" +version = "0.31.0" +source = "git+https://github.com/librelois/sled?branch=librelois_impl_transactional_for_vec_tree#18cc66fb194aa381d41eb5c8d15f6365fc3fce69" +dependencies = [ + "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "smallvec" version = "1.1.0" @@ -3214,6 +3239,7 @@ dependencies = [ "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" "checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" "checksum fragile 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f8140122fa0d5dcb9fc8627cfce2b37cc1500f752636d46ea28bc26785c2f9" +"checksum fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b6f16056ecbb57525ff698bb955162d0cd03bee84e6241c27ff75c08d8ca5987" @@ -3348,6 +3374,7 @@ dependencies = [ "checksum shrinkwraprs 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e63e6744142336dfb606fe2b068afa2e1cca1ee6a5d8377277a92945d81fa331" "checksum signal-hook-registry 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +"checksum sled 0.31.0 (git+https://github.com/librelois/sled?branch=librelois_impl_transactional_for_vec_tree)" = "<none>" "checksum smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44e59e0c9fa00817912ae6e4e6e3c4fe04455e75699d06eedc7d85917ed8e8f4" "checksum socket2 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "e8b74de517221a2cb01a53349cf54182acdc31a074727d3079068448c0676d85" "checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3" diff --git a/lib/tools/common-tools/src/traits/mod.rs b/lib/tools/common-tools/src/traits/mod.rs index 812a3a13a101ed3b67e8671788eb16d32bc3d671..d2da8fced109086aa560c2a6f74169eefae45e7a 100644 --- a/lib/tools/common-tools/src/traits/mod.rs +++ b/lib/tools/common-tools/src/traits/mod.rs @@ -21,3 +21,12 @@ pub mod merge; /// Allows to mark the real structure in order to differentiate it from the mocked structure, /// is essential in some special cases pub trait NotMock {} + +/// Create value from bytes +pub trait FromBytes: Sized { + /// Error when try to instantiate value from bytes + type Error: std::fmt::Display; + + /// Instantiate value from bytes + fn from_bytes(bytes: &[u8]) -> Result<Self, Self::Error>; +} diff --git a/lib/tools/dbs-tools/Cargo.toml b/lib/tools/dbs-tools/Cargo.toml index b239cf367317c45ddfc6d3f3e2882f1ac4b6cf34..655acf55356e90f66d87dc6566deedd11bfa349b 100644 --- a/lib/tools/dbs-tools/Cargo.toml +++ b/lib/tools/dbs-tools/Cargo.toml @@ -17,6 +17,8 @@ log = "0.4.*" rkv = "0.10.2" rustbreak = {version = "2.0.0-rc3", features = ["bin_enc"]} serde = { version = "1.0.*", features = ["derive"] } +sled = { git = "https://github.com/librelois/sled", branch="librelois_impl_transactional_for_vec_tree" } +#sled = { path = "/home/elois/dev/rust/sled" } [dev-dependencies] tempfile = "3.1.0" diff --git a/lib/tools/dbs-tools/src/kv_db.rs b/lib/tools/dbs-tools/src/kv_db.rs new file mode 100644 index 0000000000000000000000000000000000000000..ff5f4706b844d3fc7ddce719d69233f39698aae0 --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db.rs @@ -0,0 +1,154 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Define Key-Value database + +mod collections; +mod errors; +mod ro; +mod rw; + +pub use collections::{ + readable::{MultiTxCollectionReadable, SingleTxCollectionReadable}, + ro::collection::single::SingleCollectionRo, + ro::collection::{CollectionRo, TxCollectionRo}, + ro::{CollectionsRo, TxCollectionsRo}, + rw::collection::single::SingleCollectionRw, + rw::collection::{CollectionRw, TxCollectionRw}, + rw::{CollectionsRw, TxCollectionsRw}, + KvDbCollectionType, +}; +pub use errors::{KvDbError, KvDbTxAbortError, KvDbTxError}; +pub use ro::KvDbRo; +pub use rw::KvDbRw; + +use failure::Fail; +use sled::Transactional; +use std::path::Path; + +/// Key-value database schema +pub trait KvDbSchema { + /// Get collections types + fn get_collections_types() -> Vec<KvDbCollectionType>; +} + +type Result<T> = std::result::Result<T, KvDbError>; + +/// Key-value database +pub trait KvDb: KvDvPrivateTrait { + /// Database schema + type Schema: KvDbSchema; + + /// Open Key-Value database + fn open<P: AsRef<Path>>(path: P, cache_capacity: u64) -> Result<Self> { + let db = sled::Config::default() + .cache_capacity(cache_capacity) + .flush_every_ms(None) + .path(path) + .read_only(Self::read_only()) + .open() + .map_err(KvDbError::OpenDb)?; + + let mut collections = Vec::new(); + + for (collection_id, _collection_type) in + Self::Schema::get_collections_types().iter().enumerate() + { + let tree = db + .open_tree(collection_id.to_be_bytes()) + .map_err(KvDbError::OpenDb)?; + collections.push(CollectionRw::Single(SingleCollectionRw(tree))); + } + + Ok(<Self as KvDvPrivateTrait>::new(KvDbInner { + db, + collections: CollectionsRw { collections }, + })) + } + /// Read datas in a transaction + fn read<D, F>(&self, f: F) -> Result<D> + where + F: Fn(TxCollectionsRo) -> std::result::Result<D, KvDbTxError<()>>, + { + ro::read_res_from_sled_tx_res(self.collections().trees().transaction(|tx_trees| { + ro::read_res_to_sled_tx_res(f(Self::tx_trees_to_ro_collections(tx_trees))) + })) + } + #[doc(hidden)] + fn tx_trees_to_ro_collections(tx_trees: &[sled::TransactionalTree]) -> TxCollectionsRo { + TxCollectionsRo { + collections: tx_trees + .iter() + .zip(Self::Schema::get_collections_types()) + .map(TxCollectionRo::from_type_and_tree) + .collect(), + } + } +} + +#[doc(hidden)] +#[derive(Clone, Debug)] +pub struct KvDbInner { + db: sled::Db, + collections: CollectionsRw, +} + +#[doc(hidden)] +pub trait KvDvPrivateTrait: Sized { + fn collections(&self) -> &CollectionsRw { + &self.inner().collections + } + fn db(&self) -> &sled::Db { + &self.inner().db + } + fn inner(&self) -> &KvDbInner; + fn new(inner: KvDbInner) -> Self; + fn read_only() -> bool { + false + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use sled::Config; + use std::thread; + use tempfile::tempdir; + + #[test] + fn test_tmp_sled() { + //let tmp_dir = tempdir().expect("fail to create tmpdir."); + let db = sled::Config::default() + .path("test_sled_db".to_owned()) + .open() + .expect("fail to open sled DB."); + + db.insert(b"k1", b"v1").expect("db error"); + assert_eq!(&db.get(b"k1").expect("db error").expect("no value"), b"v1"); + + let _ = thread::spawn(move || { + let db2 = sled::Config::default() + .path("test_sled_db".to_owned()) + .open() + .expect("fail to open sled DB."); + + assert_eq!(&db2.get(b"k1").expect("db error").expect("no value"), b"v1"); + }); + + db.insert(b"k2", b"v2").expect("db error"); + assert_eq!(&db.get(b"k2").expect("db error").expect("no value"), b"v2"); + } +} diff --git a/lib/tools/dbs-tools/src/kv_db/collections.rs b/lib/tools/dbs-tools/src/kv_db/collections.rs new file mode 100644 index 0000000000000000000000000000000000000000..be45ee25bfd8a8e03a677171d33224319243e292 --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/collections.rs @@ -0,0 +1,54 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Define Key-Value database: read-write collections + +pub mod readable; +pub mod ro; +pub mod rw; +pub mod writable; + +use super::{KvDbError, KvDbTxAbortError, KvDbTxError}; +use durs_common_tools::traits::FromBytes; +use std::ops::Deref; + +#[derive(Clone, Copy, Debug)] +/// Database colelction type +pub enum KvDbCollectionType { + /// Mono valued collection + Single, + /// multi valued collection + Multi, +} + +#[doc(hidden)] +pub trait Collection { + fn tree(&self) -> &sled::Tree; +} + +#[doc(hidden)] +pub trait MultiCollection { + fn tree(&self) -> &sled::Tree; +} + +#[doc(hidden)] +pub trait TxCollection { + fn tree(&self) -> &sled::TransactionalTree; +} + +#[doc(hidden)] +pub trait MultiTxCollection { + fn tree(&self) -> &sled::TransactionalTree; +} diff --git a/lib/tools/dbs-tools/src/kv_db/collections/readable.rs b/lib/tools/dbs-tools/src/kv_db/collections/readable.rs new file mode 100644 index 0000000000000000000000000000000000000000..0873a0940e95c9a1284d9c4355e970ebf0e175d4 --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/collections/readable.rs @@ -0,0 +1,86 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Key-Value database: define readable operations on collections + +use super::{MultiTxCollection, TxCollection}; +use crate::kv_db::{KvDbError, KvDbTxAbortError, KvDbTxError}; +use durs_common_tools::traits::FromBytes; +use serde::de::DeserializeOwned; +use std::collections::BTreeSet; +use std::ops::Deref; + +pub trait SingleTxCollectionReadable: TxCollection { + fn get<AbortType, K: AsRef<[u8]>, V: FromBytes>( + &self, + key: &K, + ) -> Result<Option<V>, KvDbTxError<AbortType>> { + self.tree() + .get(key) + .map_err(sled::ConflictableTransactionError::from) + .map_err(KvDbTxError::TxError)? + .map(|v| V::from_bytes(v.as_ref())) + .transpose() + .map_err(|e| KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e)))) + } + fn get_and_deser<AbortType, K: AsRef<[u8]>, V: serde::de::DeserializeOwned>( + &self, + key: &K, + ) -> Result<Option<V>, KvDbTxError<AbortType>> { + self.tree() + .get(key) + .map_err(sled::ConflictableTransactionError::from) + .map_err(KvDbTxError::TxError)? + .map(|v| bincode::deserialize(v.as_ref())) + .transpose() + .map_err(|e| KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e.deref())))) + } +} + +impl<T> SingleTxCollectionReadable for T where T: TxCollection {} + +pub trait MultiTxCollectionReadable: MultiTxCollection { + /// Checks whether the pair (k, v) is contained in the multi-valued collection. + /// 3 cases: + /// + /// - Returns Some(true) if the key and the value are present and the value is well associated with the searched key. + /// - Returns Some(false) if the key is present but the searched value does not exist in the values associated with this key. + /// - Returns None if the key is not present. + /// + fn contains<AbortType, K: AsRef<[u8]>, V: DeserializeOwned + Ord + Sized>( + &self, + key: &K, + value: V, + ) -> Result<Option<bool>, KvDbTxError<AbortType>> { + Ok(self + .get_and_deser_all(key)? + .map(|set: BTreeSet<V>| set.contains(&value))) + } + #[inline] + fn get_and_deser_all<AbortType, K: AsRef<[u8]>, V: DeserializeOwned + Ord + Sized>( + &self, + key: &K, + ) -> Result<Option<BTreeSet<V>>, KvDbTxError<AbortType>> { + self.tree() + .get(key) + .map_err(sled::ConflictableTransactionError::from) + .map_err(KvDbTxError::TxError)? + .map(|v| bincode::deserialize::<BTreeSet<V>>(v.as_ref())) + .transpose() + .map_err(|e| KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e)))) + } +} + +impl<T> MultiTxCollectionReadable for T where T: MultiTxCollection {} diff --git a/lib/tools/dbs-tools/src/kv_db/collections/ro.rs b/lib/tools/dbs-tools/src/kv_db/collections/ro.rs new file mode 100644 index 0000000000000000000000000000000000000000..b26099a5495ef7aad91862c13da56546d3b93a8c --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/collections/ro.rs @@ -0,0 +1,55 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Key-Value database: define read-only collections + +pub mod collection; + +use collection::{ + single::{SingleCollectionRo, SingleTxCollectionRo}, + CollectionRo, TxCollectionRo, +}; + +use super::KvDbCollectionType; + +/// Key-value database collections +#[derive(Clone, Debug)] +pub struct CollectionsRo { + pub(crate) collections: Vec<CollectionRo>, +} + +impl CollectionsRo { + pub fn get(&self, collection_id: usize) -> &CollectionRo { + &self.collections[collection_id] + } + pub(crate) fn trees(&self) -> Vec<sled::Tree> { + self.collections + .iter() + .map(CollectionRo::tree) + .cloned() + .collect() + } +} + +/// Key-value database collections accessor in transaction +pub struct TxCollectionsRo<'a> { + pub(crate) collections: Vec<TxCollectionRo<'a>>, +} + +impl<'a> TxCollectionsRo<'a> { + pub fn get(&'a self, collection_id: usize) -> &TxCollectionRo<'a> { + &self.collections[collection_id] + } +} diff --git a/lib/tools/dbs-tools/src/kv_db/collections/ro/collection.rs b/lib/tools/dbs-tools/src/kv_db/collections/ro/collection.rs new file mode 100644 index 0000000000000000000000000000000000000000..24bcd352a3bdb4eb1685838358beab09576502b1 --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/collections/ro/collection.rs @@ -0,0 +1,53 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Define Key-Value database read-only collection + +pub mod single; + +use single::{SingleCollectionRo, SingleTxCollectionRo}; + +use super::KvDbCollectionType; + +#[derive(Clone, Debug)] +pub enum CollectionRo { + /// Mono valued collection + Single(SingleCollectionRo), +} + +impl CollectionRo { + pub(crate) fn tree(&self) -> &sled::Tree { + match self { + Self::Single(collection) => &collection.0, + } + } +} + +#[derive(Clone)] +pub enum TxCollectionRo<'a> { + /// Mono valued transactional collection + Single(SingleTxCollectionRo<'a>), +} + +impl<'a> TxCollectionRo<'a> { + pub(crate) fn from_type_and_tree( + (tree, r#type): (&'a sled::TransactionalTree, KvDbCollectionType), + ) -> Self { + match r#type { + KvDbCollectionType::Single => TxCollectionRo::Single(SingleTxCollectionRo(tree)), + KvDbCollectionType::Multi => unimplemented!(), + } + } +} diff --git a/lib/tools/dbs-tools/src/kv_db/collections/ro/collection/single.rs b/lib/tools/dbs-tools/src/kv_db/collections/ro/collection/single.rs new file mode 100644 index 0000000000000000000000000000000000000000..9482fd5d87f1b79eb479bc37b0b109d6202c6968 --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/collections/ro/collection/single.rs @@ -0,0 +1,39 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Define Key-Value database read-only single collection + +use super::KvDbCollectionType; +use crate::kv_db::collections::{Collection, TxCollection}; + +#[derive(Clone, Debug)] +/// Single collection +pub struct SingleCollectionRo(pub(crate) sled::Tree); + +impl Collection for SingleCollectionRo { + fn tree(&self) -> &sled::Tree { + &self.0 + } +} + +/// Single transactional collection +#[derive(Clone)] +pub struct SingleTxCollectionRo<'a>(pub(crate) &'a sled::TransactionalTree); + +impl<'a> TxCollection for SingleTxCollectionRo<'a> { + fn tree(&self) -> &sled::TransactionalTree { + self.0 + } +} diff --git a/lib/tools/dbs-tools/src/kv_db/collections/rw.rs b/lib/tools/dbs-tools/src/kv_db/collections/rw.rs new file mode 100644 index 0000000000000000000000000000000000000000..3296cfd55dede1300734d6be96724ab0f27e219c --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/collections/rw.rs @@ -0,0 +1,55 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Key-Value database: define read-write collections + +pub mod collection; + +use collection::{ + single::{SingleCollectionRw, SingleTxCollectionRw}, + CollectionRw, TxCollectionRw, +}; + +use super::KvDbCollectionType; + +/// Key-value database collections +#[derive(Clone, Debug)] +pub struct CollectionsRw { + pub(crate) collections: Vec<CollectionRw>, +} + +impl CollectionsRw { + pub fn get(&self, collection_id: usize) -> &CollectionRw { + &self.collections[collection_id] + } + pub(crate) fn trees(&self) -> Vec<sled::Tree> { + self.collections + .iter() + .map(CollectionRw::tree) + .cloned() + .collect() + } +} + +/// Key-value database collections accessor in transaction +pub struct TxCollectionsRw<'a> { + pub(crate) collections: Vec<TxCollectionRw<'a>>, +} + +impl<'a> TxCollectionsRw<'a> { + pub fn get(&'a self, collection_id: usize) -> &TxCollectionRw<'a> { + &self.collections[collection_id] + } +} diff --git a/lib/tools/dbs-tools/src/kv_db/collections/rw/collection.rs b/lib/tools/dbs-tools/src/kv_db/collections/rw/collection.rs new file mode 100644 index 0000000000000000000000000000000000000000..ebe19c44153852f91a379cbb289bed3519fc7308 --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/collections/rw/collection.rs @@ -0,0 +1,53 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Define Key-Value database read-write collection + +pub mod single; + +use single::{SingleCollectionRw, SingleTxCollectionRw}; + +use super::KvDbCollectionType; + +#[derive(Clone, Debug)] +pub enum CollectionRw { + /// Mono valued collection + Single(SingleCollectionRw), +} + +impl CollectionRw { + pub(crate) fn tree(&self) -> &sled::Tree { + match self { + Self::Single(collection) => &collection.0, + } + } +} + +#[derive(Clone)] +pub enum TxCollectionRw<'a> { + /// Mono valued transactional collection + Single(SingleTxCollectionRw<'a>), +} + +impl<'a> TxCollectionRw<'a> { + pub(crate) fn from_type_and_tree( + (tree, r#type): (&'a sled::TransactionalTree, KvDbCollectionType), + ) -> Self { + match r#type { + KvDbCollectionType::Single => TxCollectionRw::Single(SingleTxCollectionRw(tree)), + KvDbCollectionType::Multi => unimplemented!(), + } + } +} diff --git a/lib/tools/dbs-tools/src/kv_db/collections/rw/collection/single.rs b/lib/tools/dbs-tools/src/kv_db/collections/rw/collection/single.rs new file mode 100644 index 0000000000000000000000000000000000000000..446c58d0e6262cf3089c6fd8067610d23ecde45a --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/collections/rw/collection/single.rs @@ -0,0 +1,44 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Define Key-Value database read-write single collection + +use super::KvDbCollectionType; +use crate::kv_db::collections::writable::CollectionWritable; +use crate::kv_db::collections::{Collection, TxCollection}; + +#[derive(Clone, Debug)] +/// Single collection +pub struct SingleCollectionRw(pub(crate) sled::Tree); + +impl Collection for SingleCollectionRw { + fn tree(&self) -> &sled::Tree { + &self.0 + } +} + +impl CollectionWritable for SingleCollectionRw {} + +/// Single transactional collection +#[derive(Clone)] +pub struct SingleTxCollectionRw<'a>(pub(crate) &'a sled::TransactionalTree); + +impl<'a> TxCollection for SingleTxCollectionRw<'a> { + fn tree(&self) -> &sled::TransactionalTree { + self.0 + } +} + +impl CollectionWritable for SingleTxCollectionRw<'_> {} diff --git a/lib/tools/dbs-tools/src/kv_db/collections/writable.rs b/lib/tools/dbs-tools/src/kv_db/collections/writable.rs new file mode 100644 index 0000000000000000000000000000000000000000..751c71053b6170d89694b4dcc917f1429f219979 --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/collections/writable.rs @@ -0,0 +1,129 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Key-Value database: define writable operations on collections + +use super::{MultiTxCollection, TxCollection}; +use crate::kv_db::collections::readable::MultiTxCollectionReadable; +use crate::kv_db::{KvDbError, KvDbTxAbortError, KvDbTxError}; +use durs_common_tools::traits::FromBytes; +use serde::de::DeserializeOwned; +use serde::Serialize; +use std::collections::BTreeSet; +use std::ops::Deref; + +#[doc(hidden)] +pub trait CollectionWritable {} + +pub trait SingleTxCollectionWritable: CollectionWritable + TxCollection { + fn remove<AbortType, K: AsRef<[u8]>, V: DeserializeOwned>( + &self, + key: &K, + ) -> Result<Option<V>, KvDbTxError<AbortType>> { + self.tree() + .remove(key.as_ref()) + .map_err(sled::ConflictableTransactionError::from) + .map_err(KvDbTxError::TxError)? + .map(|v| bincode::deserialize(v.as_ref())) + .transpose() + .map_err(|e| KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e)))) + } + fn insert<AbortType, K: AsRef<[u8]>, V: Serialize>( + &self, + key: &K, + value: &V, + ) -> Result<(), KvDbTxError<AbortType>> { + let bytes = bincode::serialize(value).map_err(|e| { + KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e.deref()))) + })?; + self.tree() + .insert(key.as_ref(), &bytes[..]) + .map_err(sled::ConflictableTransactionError::from) + .map_err(KvDbTxError::TxError)?; + Ok(()) + } + fn insert_ref<AbortType, K: AsRef<[u8]>, V: AsRef<[u8]>>( + &self, + key: &K, + value: &V, + ) -> Result<(), KvDbTxError<AbortType>> { + self.tree() + .insert(key.as_ref(), value.as_ref()) + .map_err(sled::ConflictableTransactionError::from) + .map_err(KvDbTxError::TxError)?; + Ok(()) + } + fn insert_and_get_old_value<AbortType, K: AsRef<[u8]>, V: Serialize + DeserializeOwned>( + &self, + key: &K, + value: &V, + ) -> Result<Option<V>, KvDbTxError<AbortType>> { + let bytes = bincode::serialize(value).map_err(|e| { + KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e.deref()))) + })?; + self.tree() + .insert(key.as_ref(), bytes) + .map_err(sled::ConflictableTransactionError::from) + .map_err(KvDbTxError::TxError)? + .map(|v| bincode::deserialize(v.as_ref())) + .transpose() + .map_err(|e| KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e)))) + } +} + +impl<T> SingleTxCollectionWritable for T where T: CollectionWritable + TxCollection {} + +pub trait MultiTxCollectionWritable: CollectionWritable + MultiTxCollectionReadable { + /// Returns true if the value was present + fn remove<AbortType, K: AsRef<[u8]>, V: DeserializeOwned + Ord + Sized>( + &self, + key: &K, + value: &V, + ) -> Result<bool, KvDbTxError<AbortType>> { + Ok(self + .get_and_deser_all(key)? + .map(|mut set: BTreeSet<V>| set.remove(value)) + .is_some()) + } + /// Returns true if the key was present + fn remove_all<AbortType, K: AsRef<[u8]>, V: DeserializeOwned>( + &self, + key: &K, + ) -> Result<bool, KvDbTxError<AbortType>> { + Ok(self + .tree() + .remove(key.as_ref()) + .map_err(sled::ConflictableTransactionError::from)? + .is_some()) + } + fn insert<AbortType, K: AsRef<[u8]>, V: DeserializeOwned + Ord + Serialize + Sized>( + &self, + key: &K, + value: V, + ) -> Result<(), KvDbTxError<AbortType>> { + let mut set: BTreeSet<V> = self.get_and_deser_all(key)?.unwrap_or_default(); + set.insert(value); + let bytes = bincode::serialize(&set).map_err(|e| { + KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e.deref()))) + })?; + self.tree() + .insert(key.as_ref(), &bytes[..]) + .map_err(sled::ConflictableTransactionError::from) + .map_err(KvDbTxError::TxError)?; + Ok(()) + } +} + +impl<T> MultiTxCollectionWritable for T where T: CollectionWritable + MultiTxCollectionReadable {} diff --git a/lib/tools/dbs-tools/src/kv_db/errors.rs b/lib/tools/dbs-tools/src/kv_db/errors.rs new file mode 100644 index 0000000000000000000000000000000000000000..973b6001a5365c95bf9715886590716187375eb9 --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/errors.rs @@ -0,0 +1,62 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Define Key-Value database errors + +use failure::Fail; + +#[derive(Clone, Debug, PartialEq)] +pub enum KvDbTxError<AbortType> { + /// Abort error + Abort(KvDbTxAbortError<AbortType>), + /// Transaction error + TxError(sled::ConflictableTransactionError<KvDbTxAbortError<AbortType>>), +} + +impl<AbortType> From<sled::ConflictableTransactionError<KvDbTxAbortError<AbortType>>> + for KvDbTxError<AbortType> +{ + fn from(error: sled::ConflictableTransactionError<KvDbTxAbortError<AbortType>>) -> Self { + KvDbTxError::TxError(error) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum KvDbTxAbortError<AbortType> { + /// Abort error + Abort(AbortType), + /// Serde error + SerdeError(String), +} + +#[derive(Clone, Debug, Fail, PartialEq)] +/// Key-Value database error +pub enum KvDbError { + /// Fail to open db + #[fail(display = "Fail to open DB: {}.", _0)] + OpenDb(sled::Error), + /// Read error + #[fail(display = "Read error: {}.", _0)] + ReadError(sled::Error), + /// Save error + #[fail(display = "Save error: {}.", _0)] + SaveError(sled::Error), + /// Serde error + #[fail(display = "Serde error: {}.", _0)] + SerdeError(String), + /// Write error + #[fail(display = "Write error: {}.", _0)] + WriteError(sled::Error), +} diff --git a/lib/tools/dbs-tools/src/kv_db/ro.rs b/lib/tools/dbs-tools/src/kv_db/ro.rs new file mode 100644 index 0000000000000000000000000000000000000000..c046cfab2f8197b264ea41c025d46fdd3274487c --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/ro.rs @@ -0,0 +1,75 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Define Key-Value database read only handler + +use super::{ + KvDb, KvDbError, KvDbInner, KvDbSchema, KvDbTxAbortError, KvDbTxError, KvDvPrivateTrait, Result, +}; +use std::collections::BTreeMap; +use std::path::Path; + +/// Key-Value database read only +#[derive(Clone, Debug)] +pub struct KvDbRo<S: KvDbSchema> { + pub(crate) inner: KvDbInner, + pub(crate) phantom: std::marker::PhantomData<S>, +} + +impl<S: KvDbSchema> KvDvPrivateTrait for KvDbRo<S> { + fn inner(&self) -> &KvDbInner { + &self.inner + } + fn new(inner: KvDbInner) -> Self { + KvDbRo { + inner, + phantom: std::marker::PhantomData, + } + } + fn read_only() -> bool { + true + } +} + +impl<S: KvDbSchema> KvDb for KvDbRo<S> { + type Schema = S; +} + +#[inline] +pub(super) fn read_res_to_sled_tx_res<D>( + res: std::result::Result<D, KvDbTxError<()>>, +) -> sled::ConflictableTransactionResult<D, KvDbTxAbortError<()>> { + match res { + Ok(datas) => Ok(datas), + Err(KvDbTxError::TxError(e)) => Err(e), + Err(KvDbTxError::Abort(reason)) => Err(sled::ConflictableTransactionError::Abort(reason)), + } +} + +#[inline] +pub(super) fn read_res_from_sled_tx_res<D>( + sled_tx_res: sled::TransactionResult<D, KvDbTxAbortError<()>>, +) -> Result<D> { + match sled_tx_res { + Ok(datas) => Ok(datas), + Err(sled::TransactionError::Storage(e)) => Err(KvDbError::ReadError(e)), + Err(sled::TransactionError::Abort(KvDbTxAbortError::SerdeError(e))) => { + Err(KvDbError::SerdeError(e)) + } + Err(sled::TransactionError::Abort(KvDbTxAbortError::Abort(()))) => { + durs_common_tools::fatal_error!("Unexpected abort in read-only transaction.") + } + } +} diff --git a/lib/tools/dbs-tools/src/kv_db/rw.rs b/lib/tools/dbs-tools/src/kv_db/rw.rs new file mode 100644 index 0000000000000000000000000000000000000000..612df1fde7d378f723fdcd2d3a6fd4754c2fb493 --- /dev/null +++ b/lib/tools/dbs-tools/src/kv_db/rw.rs @@ -0,0 +1,111 @@ +// Copyright (C) 2017-2019 The AXIOM TEAM Association. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. + +//! Define Key-Value database read write handler + +use super::{ + KvDb, KvDbError, KvDbInner, KvDbRo, KvDbSchema, KvDbTxAbortError, KvDbTxError, + KvDvPrivateTrait, Result, TxCollectionRw, TxCollectionsRw, +}; +use sled::Transactional; +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::path::Path; + +/// Key-Value database +#[derive(Clone, Debug)] +pub struct KvDbRw<S: KvDbSchema> { + inner: KvDbInner, + phantom: std::marker::PhantomData<S>, +} + +impl<S: KvDbSchema> KvDb for KvDbRw<S> { + type Schema = S; +} + +/// Write transaction result +pub enum WriteResult<AbortType, Datas> { + Commit(Datas), + Abort(AbortType), +} + +impl<S: KvDbSchema> KvDbRw<S> { + /// Get read only handler to database + pub fn get_read_only_handler(&self) -> KvDbRo<S> { + KvDbRo { + inner: self.inner.clone(), + phantom: std::marker::PhantomData, + } + } + /// Save database content on disk + pub fn save(&self) -> Result<usize> { + self.db().flush().map_err(KvDbError::SaveError) + } + fn tx_trees_to_rw_collections(tx_trees: &[sled::TransactionalTree]) -> TxCollectionsRw { + TxCollectionsRw { + collections: tx_trees + .iter() + .zip(S::get_collections_types()) + .map(TxCollectionRw::from_type_and_tree) + .collect(), + } + } + + fn write<AbortType: Debug, D, F>( + &self, + f: F, + ) -> Result<WriteResult<KvDbTxAbortError<AbortType>, D>> + where + F: Fn(TxCollectionsRw) -> std::result::Result<D, KvDbTxError<AbortType>>, + { + write_res_from_sled_tx_res(self.collections().trees().transaction(|tx_trees| { + write_res_to_sled_tx_res(f(Self::tx_trees_to_rw_collections(tx_trees))) + })) + } +} + +impl<S: KvDbSchema> KvDvPrivateTrait for KvDbRw<S> { + fn inner(&self) -> &KvDbInner { + &self.inner + } + fn new(inner: KvDbInner) -> Self { + KvDbRw { + inner, + phantom: std::marker::PhantomData, + } + } +} + +#[inline] +fn write_res_to_sled_tx_res<AbortType: Debug, D>( + res: std::result::Result<D, KvDbTxError<AbortType>>, +) -> sled::ConflictableTransactionResult<D, KvDbTxAbortError<AbortType>> { + match res { + Ok(datas) => Ok(datas), + Err(KvDbTxError::TxError(e)) => Err(e), + Err(KvDbTxError::Abort(reason)) => Err(sled::ConflictableTransactionError::Abort(reason)), + } +} + +#[inline] +fn write_res_from_sled_tx_res<AbortType, D>( + sled_tx_res: sled::TransactionResult<D, AbortType>, +) -> Result<WriteResult<AbortType, D>> { + match sled_tx_res { + Ok(datas) => Ok(WriteResult::Commit(datas)), + Err(sled::TransactionError::Storage(e)) => Err(KvDbError::WriteError(e)), + Err(sled::TransactionError::Abort(reason)) => Ok(WriteResult::Abort(reason)), + } +} diff --git a/lib/tools/dbs-tools/src/lib.rs b/lib/tools/dbs-tools/src/lib.rs index 28e0af247733e1b3f8ace8eb9dd28dd34b176407..6911c2072995e7bf424e4a7d57067e323b679810 100644 --- a/lib/tools/dbs-tools/src/lib.rs +++ b/lib/tools/dbs-tools/src/lib.rs @@ -25,17 +25,18 @@ trivial_numeric_casts, unsafe_code, unstable_features, - unused_import_braces, - unused_qualifications + unused_import_braces )] mod errors; mod free_struct_db; +mod kv_db; /// module a supprimer pub mod kv_db_old; pub use errors::DbError; pub use free_struct_db::{open_free_struct_file_db, open_free_struct_memory_db, BinFreeStructDb}; +pub use kv_db::{KvDb, KvDbCollectionType, KvDbError, KvDbRo, KvDbRw, KvDbSchema}; use serde::de::DeserializeOwned; use serde::Serialize;