wip

Cargo.lock

@@ -1031,6 +1031,7 @@ dependencies = [
  "rkv 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustbreak 2.0.0-rc3 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
+ "sled 0.31.0 (git+https://github.com/librelois/sled?branch=librelois_impl_transactional_for_vec_tree)",
  "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1407,6 +1408,15 @@ name = "fragile"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "fs2"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "fuchsia-zircon"
 version = "0.3.3"
@@ -2598,6 +2608,21 @@ name = "slab"
 version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "sled"
+version = "0.31.0"
+source = "git+https://github.com/librelois/sled?branch=librelois_impl_transactional_for_vec_tree#18cc66fb194aa381d41eb5c8d15f6365fc3fce69"
+dependencies = [
+ "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parking_lot 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "smallvec"
 version = "1.1.0"
@@ -3214,6 +3239,7 @@ dependencies = [
 "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
 "checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 "checksum fragile 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f8140122fa0d5dcb9fc8627cfce2b37cc1500f752636d46ea28bc26785c2f9"
+"checksum fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
 "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
 "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
 "checksum futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b6f16056ecbb57525ff698bb955162d0cd03bee84e6241c27ff75c08d8ca5987"
@@ -3348,6 +3374,7 @@ dependencies = [
 "checksum shrinkwraprs 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e63e6744142336dfb606fe2b068afa2e1cca1ee6a5d8377277a92945d81fa331"
 "checksum signal-hook-registry 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41"
 "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
+"checksum sled 0.31.0 (git+https://github.com/librelois/sled?branch=librelois_impl_transactional_for_vec_tree)" = "<none>"
 "checksum smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44e59e0c9fa00817912ae6e4e6e3c4fe04455e75699d06eedc7d85917ed8e8f4"
 "checksum socket2 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "e8b74de517221a2cb01a53349cf54182acdc31a074727d3079068448c0676d85"
 "checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3"

lib/tools/common-tools/src/traits/mod.rs

@@ -21,3 +21,12 @@ pub mod merge;
 /// Allows to mark the real structure in order to differentiate it from the mocked structure,
 /// is essential in some special cases
 pub trait NotMock {}
+
+/// Create value from bytes
+pub trait FromBytes: Sized {
+    /// Error when try to instantiate value from bytes
+    type Error: std::fmt::Display;
+
+    /// Instantiate value from bytes
+    fn from_bytes(bytes: &[u8]) -> Result<Self, Self::Error>;
+}

lib/tools/dbs-tools/Cargo.toml

@@ -17,6 +17,8 @@ log = "0.4.*"
 rkv = "0.10.2"
 rustbreak = {version = "2.0.0-rc3", features = ["bin_enc"]}
 serde = { version = "1.0.*", features = ["derive"] }
+sled = { git = "https://github.com/librelois/sled", branch="librelois_impl_transactional_for_vec_tree" }
+#sled = { path = "/home/elois/dev/rust/sled" }
 
 [dev-dependencies]
 tempfile = "3.1.0"

lib/tools/dbs-tools/src/kv_db.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Define Key-Value database
+
+mod collections;
+mod errors;
+mod ro;
+mod rw;
+
+pub use collections::{
+    readable::{MultiTxCollectionReadable, SingleTxCollectionReadable},
+    ro::collection::single::SingleCollectionRo,
+    ro::collection::{CollectionRo, TxCollectionRo},
+    ro::{CollectionsRo, TxCollectionsRo},
+    rw::collection::single::SingleCollectionRw,
+    rw::collection::{CollectionRw, TxCollectionRw},
+    rw::{CollectionsRw, TxCollectionsRw},
+    KvDbCollectionType,
+};
+pub use errors::{KvDbError, KvDbTxAbortError, KvDbTxError};
+pub use ro::KvDbRo;
+pub use rw::KvDbRw;
+
+use failure::Fail;
+use sled::Transactional;
+use std::path::Path;
+
+/// Key-value database schema
+pub trait KvDbSchema {
+    /// Get collections types
+    fn get_collections_types() -> Vec<KvDbCollectionType>;
+}
+
+type Result<T> = std::result::Result<T, KvDbError>;
+
+/// Key-value database
+pub trait KvDb: KvDvPrivateTrait {
+    /// Database schema
+    type Schema: KvDbSchema;
+
+    /// Open Key-Value database
+    fn open<P: AsRef<Path>>(path: P, cache_capacity: u64) -> Result<Self> {
+        let db = sled::Config::default()
+            .cache_capacity(cache_capacity)
+            .flush_every_ms(None)
+            .path(path)
+            .read_only(Self::read_only())
+            .open()
+            .map_err(KvDbError::OpenDb)?;
+
+        let mut collections = Vec::new();
+
+        for (collection_id, _collection_type) in
+            Self::Schema::get_collections_types().iter().enumerate()
+        {
+            let tree = db
+                .open_tree(collection_id.to_be_bytes())
+                .map_err(KvDbError::OpenDb)?;
+            collections.push(CollectionRw::Single(SingleCollectionRw(tree)));
+        }
+
+        Ok(<Self as KvDvPrivateTrait>::new(KvDbInner {
+            db,
+            collections: CollectionsRw { collections },
+        }))
+    }
+    /// Read datas in a transaction
+    fn read<D, F>(&self, f: F) -> Result<D>
+    where
+        F: Fn(TxCollectionsRo) -> std::result::Result<D, KvDbTxError<()>>,
+    {
+        ro::read_res_from_sled_tx_res(self.collections().trees().transaction(|tx_trees| {
+            ro::read_res_to_sled_tx_res(f(Self::tx_trees_to_ro_collections(tx_trees)))
+        }))
+    }
+    #[doc(hidden)]
+    fn tx_trees_to_ro_collections(tx_trees: &[sled::TransactionalTree]) -> TxCollectionsRo {
+        TxCollectionsRo {
+            collections: tx_trees
+                .iter()
+                .zip(Self::Schema::get_collections_types())
+                .map(TxCollectionRo::from_type_and_tree)
+                .collect(),
+        }
+    }
+}
+
+#[doc(hidden)]
+#[derive(Clone, Debug)]
+pub struct KvDbInner {
+    db: sled::Db,
+    collections: CollectionsRw,
+}
+
+#[doc(hidden)]
+pub trait KvDvPrivateTrait: Sized {
+    fn collections(&self) -> &CollectionsRw {
+        &self.inner().collections
+    }
+    fn db(&self) -> &sled::Db {
+        &self.inner().db
+    }
+    fn inner(&self) -> &KvDbInner;
+    fn new(inner: KvDbInner) -> Self;
+    fn read_only() -> bool {
+        false
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use sled::Config;
+    use std::thread;
+    use tempfile::tempdir;
+
+    #[test]
+    fn test_tmp_sled() {
+        //let tmp_dir = tempdir().expect("fail to create tmpdir.");
+        let db = sled::Config::default()
+            .path("test_sled_db".to_owned())
+            .open()
+            .expect("fail to open sled DB.");
+
+        db.insert(b"k1", b"v1").expect("db error");
+        assert_eq!(&db.get(b"k1").expect("db error").expect("no value"), b"v1");
+
+        let _ = thread::spawn(move || {
+            let db2 = sled::Config::default()
+                .path("test_sled_db".to_owned())
+                .open()
+                .expect("fail to open sled DB.");
+
+            assert_eq!(&db2.get(b"k1").expect("db error").expect("no value"), b"v1");
+        });
+
+        db.insert(b"k2", b"v2").expect("db error");
+        assert_eq!(&db.get(b"k2").expect("db error").expect("no value"), b"v2");
+    }
+}

lib/tools/dbs-tools/src/kv_db/collections.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Define Key-Value database: read-write collections
+
+pub mod readable;
+pub mod ro;
+pub mod rw;
+pub mod writable;
+
+use super::{KvDbError, KvDbTxAbortError, KvDbTxError};
+use durs_common_tools::traits::FromBytes;
+use std::ops::Deref;
+
+#[derive(Clone, Copy, Debug)]
+/// Database colelction type
+pub enum KvDbCollectionType {
+    /// Mono valued collection
+    Single,
+    /// multi valued collection
+    Multi,
+}
+
+#[doc(hidden)]
+pub trait Collection {
+    fn tree(&self) -> &sled::Tree;
+}
+
+#[doc(hidden)]
+pub trait MultiCollection {
+    fn tree(&self) -> &sled::Tree;
+}
+
+#[doc(hidden)]
+pub trait TxCollection {
+    fn tree(&self) -> &sled::TransactionalTree;
+}
+
+#[doc(hidden)]
+pub trait MultiTxCollection {
+    fn tree(&self) -> &sled::TransactionalTree;
+}

lib/tools/dbs-tools/src/kv_db/collections/readable.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Key-Value database: define readable operations on collections
+
+use super::{MultiTxCollection, TxCollection};
+use crate::kv_db::{KvDbError, KvDbTxAbortError, KvDbTxError};
+use durs_common_tools::traits::FromBytes;
+use serde::de::DeserializeOwned;
+use std::collections::BTreeSet;
+use std::ops::Deref;
+
+pub trait SingleTxCollectionReadable: TxCollection {
+    fn get<AbortType, K: AsRef<[u8]>, V: FromBytes>(
+        &self,
+        key: &K,
+    ) -> Result<Option<V>, KvDbTxError<AbortType>> {
+        self.tree()
+            .get(key)
+            .map_err(sled::ConflictableTransactionError::from)
+            .map_err(KvDbTxError::TxError)?
+            .map(|v| V::from_bytes(v.as_ref()))
+            .transpose()
+            .map_err(|e| KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e))))
+    }
+    fn get_and_deser<AbortType, K: AsRef<[u8]>, V: serde::de::DeserializeOwned>(
+        &self,
+        key: &K,
+    ) -> Result<Option<V>, KvDbTxError<AbortType>> {
+        self.tree()
+            .get(key)
+            .map_err(sled::ConflictableTransactionError::from)
+            .map_err(KvDbTxError::TxError)?
+            .map(|v| bincode::deserialize(v.as_ref()))
+            .transpose()
+            .map_err(|e| KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e.deref()))))
+    }
+}
+
+impl<T> SingleTxCollectionReadable for T where T: TxCollection {}
+
+pub trait MultiTxCollectionReadable: MultiTxCollection {
+    /// Checks whether the pair (k, v) is contained in the multi-valued collection.
+    /// 3 cases:
+    ///
+    /// - Returns Some(true) if the key and the value are present and the value is well associated with the searched key.
+    /// - Returns Some(false) if the key is present but the searched value does not exist in the values associated with this key.
+    /// - Returns None if the key is not present.
+    ///
+    fn contains<AbortType, K: AsRef<[u8]>, V: DeserializeOwned + Ord + Sized>(
+        &self,
+        key: &K,
+        value: V,
+    ) -> Result<Option<bool>, KvDbTxError<AbortType>> {
+        Ok(self
+            .get_and_deser_all(key)?
+            .map(|set: BTreeSet<V>| set.contains(&value)))
+    }
+    #[inline]
+    fn get_and_deser_all<AbortType, K: AsRef<[u8]>, V: DeserializeOwned + Ord + Sized>(
+        &self,
+        key: &K,
+    ) -> Result<Option<BTreeSet<V>>, KvDbTxError<AbortType>> {
+        self.tree()
+            .get(key)
+            .map_err(sled::ConflictableTransactionError::from)
+            .map_err(KvDbTxError::TxError)?
+            .map(|v| bincode::deserialize::<BTreeSet<V>>(v.as_ref()))
+            .transpose()
+            .map_err(|e| KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e))))
+    }
+}
+
+impl<T> MultiTxCollectionReadable for T where T: MultiTxCollection {}

lib/tools/dbs-tools/src/kv_db/collections/ro.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Key-Value database: define read-only collections
+
+pub mod collection;
+
+use collection::{
+    single::{SingleCollectionRo, SingleTxCollectionRo},
+    CollectionRo, TxCollectionRo,
+};
+
+use super::KvDbCollectionType;
+
+/// Key-value database collections
+#[derive(Clone, Debug)]
+pub struct CollectionsRo {
+    pub(crate) collections: Vec<CollectionRo>,
+}
+
+impl CollectionsRo {
+    pub fn get(&self, collection_id: usize) -> &CollectionRo {
+        &self.collections[collection_id]
+    }
+    pub(crate) fn trees(&self) -> Vec<sled::Tree> {
+        self.collections
+            .iter()
+            .map(CollectionRo::tree)
+            .cloned()
+            .collect()
+    }
+}
+
+/// Key-value database collections accessor in transaction
+pub struct TxCollectionsRo<'a> {
+    pub(crate) collections: Vec<TxCollectionRo<'a>>,
+}
+
+impl<'a> TxCollectionsRo<'a> {
+    pub fn get(&'a self, collection_id: usize) -> &TxCollectionRo<'a> {
+        &self.collections[collection_id]
+    }
+}

lib/tools/dbs-tools/src/kv_db/collections/ro/collection.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Define Key-Value database read-only collection
+
+pub mod single;
+
+use single::{SingleCollectionRo, SingleTxCollectionRo};
+
+use super::KvDbCollectionType;
+
+#[derive(Clone, Debug)]
+pub enum CollectionRo {
+    /// Mono valued collection
+    Single(SingleCollectionRo),
+}
+
+impl CollectionRo {
+    pub(crate) fn tree(&self) -> &sled::Tree {
+        match self {
+            Self::Single(collection) => &collection.0,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub enum TxCollectionRo<'a> {
+    /// Mono valued transactional collection
+    Single(SingleTxCollectionRo<'a>),
+}
+
+impl<'a> TxCollectionRo<'a> {
+    pub(crate) fn from_type_and_tree(
+        (tree, r#type): (&'a sled::TransactionalTree, KvDbCollectionType),
+    ) -> Self {
+        match r#type {
+            KvDbCollectionType::Single => TxCollectionRo::Single(SingleTxCollectionRo(tree)),
+            KvDbCollectionType::Multi => unimplemented!(),
+        }
+    }
+}

lib/tools/dbs-tools/src/kv_db/collections/ro/collection/single.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Define Key-Value database read-only single collection
+
+use super::KvDbCollectionType;
+use crate::kv_db::collections::{Collection, TxCollection};
+
+#[derive(Clone, Debug)]
+/// Single collection
+pub struct SingleCollectionRo(pub(crate) sled::Tree);
+
+impl Collection for SingleCollectionRo {
+    fn tree(&self) -> &sled::Tree {
+        &self.0
+    }
+}
+
+/// Single transactional collection
+#[derive(Clone)]
+pub struct SingleTxCollectionRo<'a>(pub(crate) &'a sled::TransactionalTree);
+
+impl<'a> TxCollection for SingleTxCollectionRo<'a> {
+    fn tree(&self) -> &sled::TransactionalTree {
+        self.0
+    }
+}

lib/tools/dbs-tools/src/kv_db/collections/rw.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Key-Value database: define read-write collections
+
+pub mod collection;
+
+use collection::{
+    single::{SingleCollectionRw, SingleTxCollectionRw},
+    CollectionRw, TxCollectionRw,
+};
+
+use super::KvDbCollectionType;
+
+/// Key-value database collections
+#[derive(Clone, Debug)]
+pub struct CollectionsRw {
+    pub(crate) collections: Vec<CollectionRw>,
+}
+
+impl CollectionsRw {
+    pub fn get(&self, collection_id: usize) -> &CollectionRw {
+        &self.collections[collection_id]
+    }
+    pub(crate) fn trees(&self) -> Vec<sled::Tree> {
+        self.collections
+            .iter()
+            .map(CollectionRw::tree)
+            .cloned()
+            .collect()
+    }
+}
+
+/// Key-value database collections accessor in transaction
+pub struct TxCollectionsRw<'a> {
+    pub(crate) collections: Vec<TxCollectionRw<'a>>,
+}
+
+impl<'a> TxCollectionsRw<'a> {
+    pub fn get(&'a self, collection_id: usize) -> &TxCollectionRw<'a> {
+        &self.collections[collection_id]
+    }
+}

lib/tools/dbs-tools/src/kv_db/collections/rw/collection.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Define Key-Value database read-write collection
+
+pub mod single;
+
+use single::{SingleCollectionRw, SingleTxCollectionRw};
+
+use super::KvDbCollectionType;
+
+#[derive(Clone, Debug)]
+pub enum CollectionRw {
+    /// Mono valued collection
+    Single(SingleCollectionRw),
+}
+
+impl CollectionRw {
+    pub(crate) fn tree(&self) -> &sled::Tree {
+        match self {
+            Self::Single(collection) => &collection.0,
+        }
+    }
+}
+
+#[derive(Clone)]
+pub enum TxCollectionRw<'a> {
+    /// Mono valued transactional collection
+    Single(SingleTxCollectionRw<'a>),
+}
+
+impl<'a> TxCollectionRw<'a> {
+    pub(crate) fn from_type_and_tree(
+        (tree, r#type): (&'a sled::TransactionalTree, KvDbCollectionType),
+    ) -> Self {
+        match r#type {
+            KvDbCollectionType::Single => TxCollectionRw::Single(SingleTxCollectionRw(tree)),
+            KvDbCollectionType::Multi => unimplemented!(),
+        }
+    }
+}

lib/tools/dbs-tools/src/kv_db/collections/rw/collection/single.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Define Key-Value database read-write single collection
+
+use super::KvDbCollectionType;
+use crate::kv_db::collections::writable::CollectionWritable;
+use crate::kv_db::collections::{Collection, TxCollection};
+
+#[derive(Clone, Debug)]
+/// Single collection
+pub struct SingleCollectionRw(pub(crate) sled::Tree);
+
+impl Collection for SingleCollectionRw {
+    fn tree(&self) -> &sled::Tree {
+        &self.0
+    }
+}
+
+impl CollectionWritable for SingleCollectionRw {}
+
+/// Single transactional collection
+#[derive(Clone)]
+pub struct SingleTxCollectionRw<'a>(pub(crate) &'a sled::TransactionalTree);
+
+impl<'a> TxCollection for SingleTxCollectionRw<'a> {
+    fn tree(&self) -> &sled::TransactionalTree {
+        self.0
+    }
+}
+
+impl CollectionWritable for SingleTxCollectionRw<'_> {}

lib/tools/dbs-tools/src/kv_db/collections/writable.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Key-Value database: define writable operations on collections
+
+use super::{MultiTxCollection, TxCollection};
+use crate::kv_db::collections::readable::MultiTxCollectionReadable;
+use crate::kv_db::{KvDbError, KvDbTxAbortError, KvDbTxError};
+use durs_common_tools::traits::FromBytes;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+use std::collections::BTreeSet;
+use std::ops::Deref;
+
+#[doc(hidden)]
+pub trait CollectionWritable {}
+
+pub trait SingleTxCollectionWritable: CollectionWritable + TxCollection {
+    fn remove<AbortType, K: AsRef<[u8]>, V: DeserializeOwned>(
+        &self,
+        key: &K,
+    ) -> Result<Option<V>, KvDbTxError<AbortType>> {
+        self.tree()
+            .remove(key.as_ref())
+            .map_err(sled::ConflictableTransactionError::from)
+            .map_err(KvDbTxError::TxError)?
+            .map(|v| bincode::deserialize(v.as_ref()))
+            .transpose()
+            .map_err(|e| KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e))))
+    }
+    fn insert<AbortType, K: AsRef<[u8]>, V: Serialize>(
+        &self,
+        key: &K,
+        value: &V,
+    ) -> Result<(), KvDbTxError<AbortType>> {
+        let bytes = bincode::serialize(value).map_err(|e| {
+            KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e.deref())))
+        })?;
+        self.tree()
+            .insert(key.as_ref(), &bytes[..])
+            .map_err(sled::ConflictableTransactionError::from)
+            .map_err(KvDbTxError::TxError)?;
+        Ok(())
+    }
+    fn insert_ref<AbortType, K: AsRef<[u8]>, V: AsRef<[u8]>>(
+        &self,
+        key: &K,
+        value: &V,
+    ) -> Result<(), KvDbTxError<AbortType>> {
+        self.tree()
+            .insert(key.as_ref(), value.as_ref())
+            .map_err(sled::ConflictableTransactionError::from)
+            .map_err(KvDbTxError::TxError)?;
+        Ok(())
+    }
+    fn insert_and_get_old_value<AbortType, K: AsRef<[u8]>, V: Serialize + DeserializeOwned>(
+        &self,
+        key: &K,
+        value: &V,
+    ) -> Result<Option<V>, KvDbTxError<AbortType>> {
+        let bytes = bincode::serialize(value).map_err(|e| {
+            KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e.deref())))
+        })?;
+        self.tree()
+            .insert(key.as_ref(), bytes)
+            .map_err(sled::ConflictableTransactionError::from)
+            .map_err(KvDbTxError::TxError)?
+            .map(|v| bincode::deserialize(v.as_ref()))
+            .transpose()
+            .map_err(|e| KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e))))
+    }
+}
+
+impl<T> SingleTxCollectionWritable for T where T: CollectionWritable + TxCollection {}
+
+pub trait MultiTxCollectionWritable: CollectionWritable + MultiTxCollectionReadable {
+    /// Returns true if the value was present
+    fn remove<AbortType, K: AsRef<[u8]>, V: DeserializeOwned + Ord + Sized>(
+        &self,
+        key: &K,
+        value: &V,
+    ) -> Result<bool, KvDbTxError<AbortType>> {
+        Ok(self
+            .get_and_deser_all(key)?
+            .map(|mut set: BTreeSet<V>| set.remove(value))
+            .is_some())
+    }
+    /// Returns true if the key was present
+    fn remove_all<AbortType, K: AsRef<[u8]>, V: DeserializeOwned>(
+        &self,
+        key: &K,
+    ) -> Result<bool, KvDbTxError<AbortType>> {
+        Ok(self
+            .tree()
+            .remove(key.as_ref())
+            .map_err(sled::ConflictableTransactionError::from)?
+            .is_some())
+    }
+    fn insert<AbortType, K: AsRef<[u8]>, V: DeserializeOwned + Ord + Serialize + Sized>(
+        &self,
+        key: &K,
+        value: V,
+    ) -> Result<(), KvDbTxError<AbortType>> {
+        let mut set: BTreeSet<V> = self.get_and_deser_all(key)?.unwrap_or_default();
+        set.insert(value);
+        let bytes = bincode::serialize(&set).map_err(|e| {
+            KvDbTxError::Abort(KvDbTxAbortError::SerdeError(format!("{}", e.deref())))
+        })?;
+        self.tree()
+            .insert(key.as_ref(), &bytes[..])
+            .map_err(sled::ConflictableTransactionError::from)
+            .map_err(KvDbTxError::TxError)?;
+        Ok(())
+    }
+}
+
+impl<T> MultiTxCollectionWritable for T where T: CollectionWritable + MultiTxCollectionReadable {}

lib/tools/dbs-tools/src/kv_db/errors.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Define Key-Value database errors
+
+use failure::Fail;
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum KvDbTxError<AbortType> {
+    /// Abort error
+    Abort(KvDbTxAbortError<AbortType>),
+    /// Transaction error
+    TxError(sled::ConflictableTransactionError<KvDbTxAbortError<AbortType>>),
+}
+
+impl<AbortType> From<sled::ConflictableTransactionError<KvDbTxAbortError<AbortType>>>
+    for KvDbTxError<AbortType>
+{
+    fn from(error: sled::ConflictableTransactionError<KvDbTxAbortError<AbortType>>) -> Self {
+        KvDbTxError::TxError(error)
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum KvDbTxAbortError<AbortType> {
+    /// Abort error
+    Abort(AbortType),
+    /// Serde error
+    SerdeError(String),
+}
+
+#[derive(Clone, Debug, Fail, PartialEq)]
+/// Key-Value database error
+pub enum KvDbError {
+    /// Fail to open db
+    #[fail(display = "Fail to open DB: {}.", _0)]
+    OpenDb(sled::Error),
+    /// Read error
+    #[fail(display = "Read error: {}.", _0)]
+    ReadError(sled::Error),
+    /// Save error
+    #[fail(display = "Save error: {}.", _0)]
+    SaveError(sled::Error),
+    /// Serde error
+    #[fail(display = "Serde error: {}.", _0)]
+    SerdeError(String),
+    /// Write error
+    #[fail(display = "Write error: {}.", _0)]
+    WriteError(sled::Error),
+}

lib/tools/dbs-tools/src/kv_db/ro.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Define Key-Value database read only handler
+
+use super::{
+    KvDb, KvDbError, KvDbInner, KvDbSchema, KvDbTxAbortError, KvDbTxError, KvDvPrivateTrait, Result,
+};
+use std::collections::BTreeMap;
+use std::path::Path;
+
+/// Key-Value database read only
+#[derive(Clone, Debug)]
+pub struct KvDbRo<S: KvDbSchema> {
+    pub(crate) inner: KvDbInner,
+    pub(crate) phantom: std::marker::PhantomData<S>,
+}
+
+impl<S: KvDbSchema> KvDvPrivateTrait for KvDbRo<S> {
+    fn inner(&self) -> &KvDbInner {
+        &self.inner
+    }
+    fn new(inner: KvDbInner) -> Self {
+        KvDbRo {
+            inner,
+            phantom: std::marker::PhantomData,
+        }
+    }
+    fn read_only() -> bool {
+        true
+    }
+}
+
+impl<S: KvDbSchema> KvDb for KvDbRo<S> {
+    type Schema = S;
+}
+
+#[inline]
+pub(super) fn read_res_to_sled_tx_res<D>(
+    res: std::result::Result<D, KvDbTxError<()>>,
+) -> sled::ConflictableTransactionResult<D, KvDbTxAbortError<()>> {
+    match res {
+        Ok(datas) => Ok(datas),
+        Err(KvDbTxError::TxError(e)) => Err(e),
+        Err(KvDbTxError::Abort(reason)) => Err(sled::ConflictableTransactionError::Abort(reason)),
+    }
+}
+
+#[inline]
+pub(super) fn read_res_from_sled_tx_res<D>(
+    sled_tx_res: sled::TransactionResult<D, KvDbTxAbortError<()>>,
+) -> Result<D> {
+    match sled_tx_res {
+        Ok(datas) => Ok(datas),
+        Err(sled::TransactionError::Storage(e)) => Err(KvDbError::ReadError(e)),
+        Err(sled::TransactionError::Abort(KvDbTxAbortError::SerdeError(e))) => {
+            Err(KvDbError::SerdeError(e))
+        }
+        Err(sled::TransactionError::Abort(KvDbTxAbortError::Abort(()))) => {
+            durs_common_tools::fatal_error!("Unexpected abort in read-only transaction.")
+        }
+    }
+}

lib/tools/dbs-tools/src/kv_db/rw.rs

+//  Copyright (C) 2017-2019  The AXIOM TEAM Association.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+//! Define Key-Value database read write handler
+
+use super::{
+    KvDb, KvDbError, KvDbInner, KvDbRo, KvDbSchema, KvDbTxAbortError, KvDbTxError,
+    KvDvPrivateTrait, Result, TxCollectionRw, TxCollectionsRw,
+};
+use sled::Transactional;
+use std::collections::BTreeMap;
+use std::fmt::Debug;
+use std::path::Path;
+
+/// Key-Value database
+#[derive(Clone, Debug)]
+pub struct KvDbRw<S: KvDbSchema> {
+    inner: KvDbInner,
+    phantom: std::marker::PhantomData<S>,
+}
+
+impl<S: KvDbSchema> KvDb for KvDbRw<S> {
+    type Schema = S;
+}
+
+/// Write transaction result
+pub enum WriteResult<AbortType, Datas> {
+    Commit(Datas),
+    Abort(AbortType),
+}
+
+impl<S: KvDbSchema> KvDbRw<S> {
+    /// Get read only handler to database
+    pub fn get_read_only_handler(&self) -> KvDbRo<S> {
+        KvDbRo {
+            inner: self.inner.clone(),
+            phantom: std::marker::PhantomData,
+        }
+    }
+    /// Save database content on disk
+    pub fn save(&self) -> Result<usize> {
+        self.db().flush().map_err(KvDbError::SaveError)
+    }
+    fn tx_trees_to_rw_collections(tx_trees: &[sled::TransactionalTree]) -> TxCollectionsRw {
+        TxCollectionsRw {
+            collections: tx_trees
+                .iter()
+                .zip(S::get_collections_types())
+                .map(TxCollectionRw::from_type_and_tree)
+                .collect(),
+        }
+    }
+
+    fn write<AbortType: Debug, D, F>(
+        &self,
+        f: F,
+    ) -> Result<WriteResult<KvDbTxAbortError<AbortType>, D>>
+    where
+        F: Fn(TxCollectionsRw) -> std::result::Result<D, KvDbTxError<AbortType>>,
+    {
+        write_res_from_sled_tx_res(self.collections().trees().transaction(|tx_trees| {
+            write_res_to_sled_tx_res(f(Self::tx_trees_to_rw_collections(tx_trees)))
+        }))
+    }
+}
+
+impl<S: KvDbSchema> KvDvPrivateTrait for KvDbRw<S> {
+    fn inner(&self) -> &KvDbInner {
+        &self.inner
+    }
+    fn new(inner: KvDbInner) -> Self {
+        KvDbRw {
+            inner,
+            phantom: std::marker::PhantomData,
+        }
+    }
+}
+
+#[inline]
+fn write_res_to_sled_tx_res<AbortType: Debug, D>(
+    res: std::result::Result<D, KvDbTxError<AbortType>>,
+) -> sled::ConflictableTransactionResult<D, KvDbTxAbortError<AbortType>> {
+    match res {
+        Ok(datas) => Ok(datas),
+        Err(KvDbTxError::TxError(e)) => Err(e),
+        Err(KvDbTxError::Abort(reason)) => Err(sled::ConflictableTransactionError::Abort(reason)),
+    }
+}
+
+#[inline]
+fn write_res_from_sled_tx_res<AbortType, D>(
+    sled_tx_res: sled::TransactionResult<D, AbortType>,
+) -> Result<WriteResult<AbortType, D>> {
+    match sled_tx_res {
+        Ok(datas) => Ok(WriteResult::Commit(datas)),
+        Err(sled::TransactionError::Storage(e)) => Err(KvDbError::WriteError(e)),
+        Err(sled::TransactionError::Abort(reason)) => Ok(WriteResult::Abort(reason)),
+    }
+}

lib/tools/dbs-tools/src/lib.rs

@@ -25,17 +25,18 @@
     trivial_numeric_casts,
     unsafe_code,
     unstable_features,
-    unused_import_braces,
-    unused_qualifications
+    unused_import_braces
 )]
 
 mod errors;
 mod free_struct_db;
+mod kv_db;
 /// module a supprimer
 pub mod kv_db_old;
 
 pub use errors::DbError;
 pub use free_struct_db::{open_free_struct_file_db, open_free_struct_memory_db, BinFreeStructDb};
+pub use kv_db::{KvDb, KvDbCollectionType, KvDbError, KvDbRo, KvDbRw, KvDbSchema};
 
 use serde::de::DeserializeOwned;
 use serde::Serialize;