From 116bdbe459f3eaabd14fe0611527379689385d6f Mon Sep 17 00:00:00 2001
From: librelois <c@elo.tf>
Date: Fri, 14 May 2021 19:37:01 +0200
Subject: [PATCH] perf(dbs): serialize integers as varint
---
Cargo.lock | 10 ++++++++--
bc-reader/Cargo.toml | 2 +-
dbs/Cargo.toml | 4 ++--
dbs/src/keys/blockstamp.rs | 2 +-
dbs/src/keys/pubkey_and_sig.rs | 2 +-
dbs/src/keys/source_key.rs | 2 +-
dbs/src/keys/uid.rs | 8 +++-----
dbs/src/keys/wallet_conditions.rs | 12 +++++++-----
dbs/src/lib.rs | 6 ++++++
dbs/src/values/block_db.rs | 8 ++++++--
dbs/src/values/block_meta.rs | 14 ++++++++------
dbs/src/values/dunp_head.rs | 8 ++++++--
dbs/src/values/idty_db.rs | 6 ++++--
dbs/src/values/peer_card.rs | 8 ++++++--
dbs/src/values/tx_db.rs | 8 ++++++--
dbs/src/values/txs.rs | 8 ++++++--
dbs/src/values/utxo.rs | 6 ++++--
dbs/src/values/wallet_script_with_sa.rs | 6 ++++--
tools/kv_typed/Cargo.toml | 4 ++--
19 files changed, 82 insertions(+), 42 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 2af0f7e..7eb46ff 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -27,6 +27,12 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
+[[package]]
+name = "arrayvec"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a2f58b0bb10c380af2b26e57212856b8c9a59e0925b4c20f4a174a49734eaf7"
+
[[package]]
name = "async-attributes"
version = "1.1.2"
@@ -243,7 +249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9ff35b701f3914bdb8fad3368d822c766ef2858b2583198e41639b936f09d3f"
dependencies = [
"arrayref",
- "arrayvec",
+ "arrayvec 0.5.2",
"cc",
"cfg-if 0.1.10",
"constant_time_eq",
@@ -812,7 +818,7 @@ name = "duniter-dbs"
version = "0.1.0"
dependencies = [
"anyhow",
- "arrayvec",
+ "arrayvec 0.7.0",
"bincode",
"byteorder",
"chrono",
diff --git a/bc-reader/Cargo.toml b/bc-reader/Cargo.toml
index d2e0957..bc421b9 100644
--- a/bc-reader/Cargo.toml
+++ b/bc-reader/Cargo.toml
@@ -18,4 +18,4 @@ dubp = { version = "0.53.1", features = ["duniter"] }
resiter = "0.4.0"
[dev-dependencies]
-smallvec = { version = "1.4.0", features = ["serde", "write"] }
+smallvec = { version = "1.6", features = ["serde", "write"] }
diff --git a/dbs/Cargo.toml b/dbs/Cargo.toml
index 6dfe970..332848d 100644
--- a/dbs/Cargo.toml
+++ b/dbs/Cargo.toml
@@ -12,7 +12,7 @@ edition = "2018"
path = "src/lib.rs"
[dependencies]
-arrayvec = "0.5.1"
+arrayvec = "0.7"
bincode = "1.2.1"
byteorder = "1.3.4"
chrono = { version = "0.4.15", optional = true }
@@ -25,7 +25,7 @@ paste = "1.0.2"
rand = "0.8.3"
serde = { version = "1.0.105", features = ["derive"] }
serde_json = "1.0.53"
-smallvec = { version = "1.4.0", features = ["serde", "write"] }
+smallvec = { version = "1.6", features = ["serde", "write"] }
thiserror = "1.0.20"
uninit = "0.4.0"
zerocopy = "0.3.0"
diff --git a/dbs/src/keys/blockstamp.rs b/dbs/src/keys/blockstamp.rs
index 6b4a8df..aa2e23c 100644
--- a/dbs/src/keys/blockstamp.rs
+++ b/dbs/src/keys/blockstamp.rs
@@ -28,7 +28,7 @@ impl kv_typed::prelude::FromBytes for BlockstampKeyV1 {
type Err = CorruptedBytes;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- let blockstamp_strs: ArrayVec<[&str; 2]> = std::str::from_utf8(bytes)
+ let blockstamp_strs: ArrayVec<&str, 2> = std::str::from_utf8(bytes)
.map_err(|e| CorruptedBytes(e.to_string()))?
.split('-')
.collect();
diff --git a/dbs/src/keys/pubkey_and_sig.rs b/dbs/src/keys/pubkey_and_sig.rs
index 47a14c9..9c6a010 100644
--- a/dbs/src/keys/pubkey_and_sig.rs
+++ b/dbs/src/keys/pubkey_and_sig.rs
@@ -42,7 +42,7 @@ impl kv_typed::prelude::FromBytes for PubKeyAndSigV1 {
if raw_str == "ALL" {
Ok(PubKeyAndSigV1::all())
} else {
- let array_str: ArrayVec<[&str; 2]> = raw_str.split(':').collect();
+ let array_str: ArrayVec<&str, 2> = raw_str.split(':').collect();
let pubkey =
PublicKey::from_base58(array_str[0]).map_err(|e| CorruptedBytes(e.to_string()))?;
let sig =
diff --git a/dbs/src/keys/source_key.rs b/dbs/src/keys/source_key.rs
index accb5c9..fbdb1d6 100644
--- a/dbs/src/keys/source_key.rs
+++ b/dbs/src/keys/source_key.rs
@@ -47,7 +47,7 @@ impl kv_typed::prelude::FromBytes for SourceKeyV1 {
type Err = CorruptedBytes;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- let strs: ArrayVec<[&str; 3]> = std::str::from_utf8(bytes)
+ let strs: ArrayVec<&str, 3> = std::str::from_utf8(bytes)
.map_err(|e| CorruptedBytes(e.to_string()))?
.split('-')
.collect();
diff --git a/dbs/src/keys/uid.rs b/dbs/src/keys/uid.rs
index a5e4bd7..968d7e8 100644
--- a/dbs/src/keys/uid.rs
+++ b/dbs/src/keys/uid.rs
@@ -18,7 +18,7 @@ use crate::*;
const USERNAME_MAX_LEN: usize = 100;
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd)]
-pub struct UidKeyV1(pub ArrayString<[u8; USERNAME_MAX_LEN]>);
+pub struct UidKeyV1(pub ArrayString<USERNAME_MAX_LEN>);
impl AsBytes for UidKeyV1 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
@@ -32,7 +32,7 @@ impl kv_typed::prelude::FromBytes for UidKeyV1 {
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
let uid_str = std::str::from_utf8(bytes).map_err(|e| CorruptedBytes(e.to_string()))?;
Ok(Self(
- ArrayString::<[u8; USERNAME_MAX_LEN]>::from_str(uid_str)
+ ArrayString::<USERNAME_MAX_LEN>::from_str(uid_str)
.map_err(|e| CorruptedBytes(e.to_string()))?,
))
}
@@ -42,9 +42,7 @@ impl FromStr for UidKeyV1 {
type Err = arrayvec::CapacityError;
fn from_str(source: &str) -> std::result::Result<Self, Self::Err> {
- Ok(UidKeyV1(ArrayString::<[u8; USERNAME_MAX_LEN]>::from_str(
- source,
- )?))
+ Ok(UidKeyV1(ArrayString::<USERNAME_MAX_LEN>::from_str(source)?))
}
}
diff --git a/dbs/src/keys/wallet_conditions.rs b/dbs/src/keys/wallet_conditions.rs
index 56af52a..d7a8cb7 100644
--- a/dbs/src/keys/wallet_conditions.rs
+++ b/dbs/src/keys/wallet_conditions.rs
@@ -18,7 +18,7 @@ use crate::*;
const CONDITIONS_MAX_LEN: usize = 256;
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd)]
-pub struct WalletConditionsV1(pub ArrayString<[u8; CONDITIONS_MAX_LEN]>);
+pub struct WalletConditionsV1(pub ArrayString<CONDITIONS_MAX_LEN>);
impl AsBytes for WalletConditionsV1 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
@@ -32,7 +32,7 @@ impl kv_typed::prelude::FromBytes for WalletConditionsV1 {
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
let uid_str = std::str::from_utf8(bytes).map_err(|e| CorruptedBytes(e.to_string()))?;
Ok(Self(
- ArrayString::<[u8; CONDITIONS_MAX_LEN]>::from_str(uid_str)
+ ArrayString::<CONDITIONS_MAX_LEN>::from_str(uid_str)
.map_err(|e| CorruptedBytes(e.to_string()))?,
))
}
@@ -43,7 +43,7 @@ impl FromStr for WalletConditionsV1 {
fn from_str(source: &str) -> std::result::Result<Self, Self::Err> {
Ok(WalletConditionsV1(
- ArrayString::<[u8; CONDITIONS_MAX_LEN]>::from_str(source)?,
+ ArrayString::<CONDITIONS_MAX_LEN>::from_str(source)?,
))
}
}
@@ -79,7 +79,9 @@ impl WalletConditionsV2 {
impl AsBytes for WalletConditionsV2 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
let mut buffer = SmallVec::<[u8; 256]>::new();
- bincode::serialize_into(&mut buffer, &self.0).unwrap_or_else(|_| unreachable!());
+ bincode_db()
+ .serialize_into(&mut buffer, &self.0)
+ .unwrap_or_else(|_| unreachable!());
f(buffer.as_ref())
}
}
@@ -88,7 +90,7 @@ impl kv_typed::prelude::FromBytes for WalletConditionsV2 {
type Err = bincode::Error;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- Ok(Self(bincode::deserialize(bytes)?))
+ Ok(Self(bincode_db().deserialize(bytes)?))
}
}
diff --git a/dbs/src/lib.rs b/dbs/src/lib.rs
index 63d8a82..863221c 100644
--- a/dbs/src/lib.rs
+++ b/dbs/src/lib.rs
@@ -89,6 +89,7 @@ pub use values::wallet_script_with_sa::WalletScriptWithSourceAmountV1Db;
// Crate imports
pub(crate) use arrayvec::{ArrayString, ArrayVec};
+pub(crate) use bincode::Options;
#[cfg(feature = "explorer")]
use chrono::NaiveDateTime;
pub(crate) use dubp::common::crypto::bases::b58::ToBase58 as _;
@@ -142,3 +143,8 @@ impl SharedDbs<Mem> {
})
}
}
+
+// Bincode db configuration
+pub fn bincode_db() -> impl bincode::Options {
+ bincode::options()
+}
diff --git a/dbs/src/values/block_db.rs b/dbs/src/values/block_db.rs
index c288cda..2c6c508 100644
--- a/dbs/src/values/block_db.rs
+++ b/dbs/src/values/block_db.rs
@@ -122,7 +122,9 @@ pub struct BlockDbV2(pub dubp::block::DubpBlockV10);
impl AsBytes for BlockDbV2 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
- let bytes = bincode::serialize(self).unwrap_or_else(|_| unreachable!());
+ let bytes = bincode_db()
+ .serialize(self)
+ .unwrap_or_else(|_| unreachable!());
f(bytes.as_ref())
}
}
@@ -131,7 +133,9 @@ impl kv_typed::prelude::FromBytes for BlockDbV2 {
type Err = CorruptedBytes;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- bincode::deserialize(&bytes).map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
+ bincode_db()
+ .deserialize(&bytes)
+ .map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
}
}
diff --git a/dbs/src/values/block_meta.rs b/dbs/src/values/block_meta.rs
index 949a644..92eaee0 100644
--- a/dbs/src/values/block_meta.rs
+++ b/dbs/src/values/block_meta.rs
@@ -17,7 +17,7 @@ use dubp::block::DubpBlockV10;
use crate::*;
-const BLOCK_META_SERIALIZED_SIZE: usize = 323;
+const BLOCK_META_MAX_SERIALIZED_SIZE: usize = 323;
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Serialize)]
pub struct BlockMetaV2 {
@@ -52,8 +52,10 @@ impl BlockMetaV2 {
impl AsBytes for BlockMetaV2 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
- let mut buffer = [0u8; BLOCK_META_SERIALIZED_SIZE];
- bincode::serialize_into(&mut buffer[..], self).unwrap_or_else(|_| unreachable!());
+ let mut buffer = ArrayVec::<u8, BLOCK_META_MAX_SERIALIZED_SIZE>::new();
+ bincode_db()
+ .serialize_into(&mut buffer, self)
+ .unwrap_or_else(|_| unreachable!());
f(buffer.as_ref())
}
}
@@ -62,7 +64,7 @@ impl kv_typed::prelude::FromBytes for BlockMetaV2 {
type Err = bincode::Error;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- bincode::deserialize(bytes)
+ bincode_db().deserialize(bytes)
}
}
@@ -91,11 +93,11 @@ mod tests {
#[test]
fn block_meta_v2_as_bytes() {
assert_eq!(
- unwrap!(bincode::serialized_size(&BlockMetaV2 {
+ unwrap!(bincode_db().serialized_size(&BlockMetaV2 {
dividend: Some(SourceAmount::new(42, 0)),
..Default::default()
})),
- BLOCK_META_SERIALIZED_SIZE as u64
+ 255
);
let bloc_meta = BlockMetaV2::default();
diff --git a/dbs/src/values/dunp_head.rs b/dbs/src/values/dunp_head.rs
index bf42b21..a9b1515 100644
--- a/dbs/src/values/dunp_head.rs
+++ b/dbs/src/values/dunp_head.rs
@@ -94,7 +94,9 @@ mod tests {
impl AsBytes for DunpHeadDbV1 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
- let bytes = bincode::serialize(self).unwrap_or_else(|_| unreachable!());
+ let bytes = bincode_db()
+ .serialize(self)
+ .unwrap_or_else(|_| unreachable!());
f(bytes.as_ref())
}
}
@@ -103,7 +105,9 @@ impl kv_typed::prelude::FromBytes for DunpHeadDbV1 {
type Err = CorruptedBytes;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- bincode::deserialize(&bytes).map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
+ bincode_db()
+ .deserialize(&bytes)
+ .map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
}
}
diff --git a/dbs/src/values/idty_db.rs b/dbs/src/values/idty_db.rs
index 504f799..688e1f9 100644
--- a/dbs/src/values/idty_db.rs
+++ b/dbs/src/values/idty_db.rs
@@ -23,7 +23,9 @@ pub struct IdtyDbV2 {
impl AsBytes for IdtyDbV2 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
- f(&bincode::serialize(&self).unwrap_or_else(|_| unreachable!()))
+ f(&bincode_db()
+ .serialize(&self)
+ .unwrap_or_else(|_| unreachable!()))
}
}
@@ -31,7 +33,7 @@ impl kv_typed::prelude::FromBytes for IdtyDbV2 {
type Err = bincode::Error;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- bincode::deserialize(bytes)
+ bincode_db().deserialize(bytes)
}
}
diff --git a/dbs/src/values/peer_card.rs b/dbs/src/values/peer_card.rs
index 15dfaec..39979f5 100644
--- a/dbs/src/values/peer_card.rs
+++ b/dbs/src/values/peer_card.rs
@@ -53,7 +53,9 @@ impl PeerCardDbV1 {
impl AsBytes for PeerCardDbV1 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
- let bytes = bincode::serialize(self).unwrap_or_else(|_| unreachable!());
+ let bytes = bincode_db()
+ .serialize(self)
+ .unwrap_or_else(|_| unreachable!());
f(bytes.as_ref())
}
}
@@ -62,7 +64,9 @@ impl kv_typed::prelude::FromBytes for PeerCardDbV1 {
type Err = CorruptedBytes;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- bincode::deserialize(&bytes).map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
+ bincode_db()
+ .deserialize(&bytes)
+ .map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
}
}
diff --git a/dbs/src/values/tx_db.rs b/dbs/src/values/tx_db.rs
index 59700b7..beae057 100644
--- a/dbs/src/values/tx_db.rs
+++ b/dbs/src/values/tx_db.rs
@@ -24,7 +24,9 @@ pub struct PendingTxDbV2 {
impl AsBytes for PendingTxDbV2 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
- let bytes = bincode::serialize(self).unwrap_or_else(|_| unreachable!());
+ let bytes = bincode_db()
+ .serialize(self)
+ .unwrap_or_else(|_| unreachable!());
f(bytes.as_ref())
}
}
@@ -33,7 +35,9 @@ impl kv_typed::prelude::FromBytes for PendingTxDbV2 {
type Err = CorruptedBytes;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- bincode::deserialize(&bytes).map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
+ bincode_db()
+ .deserialize(&bytes)
+ .map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
}
}
diff --git a/dbs/src/values/txs.rs b/dbs/src/values/txs.rs
index f150673..530506e 100644
--- a/dbs/src/values/txs.rs
+++ b/dbs/src/values/txs.rs
@@ -21,7 +21,9 @@ pub struct BlockTxsDbV2(pub SmallVec<[TransactionDocumentV10; 8]>);
impl AsBytes for BlockTxsDbV2 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
- let bytes = bincode::serialize(self).unwrap_or_else(|_| unreachable!());
+ let bytes = bincode_db()
+ .serialize(self)
+ .unwrap_or_else(|_| unreachable!());
f(bytes.as_ref())
}
}
@@ -30,7 +32,9 @@ impl kv_typed::prelude::FromBytes for BlockTxsDbV2 {
type Err = CorruptedBytes;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- bincode::deserialize(&bytes).map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
+ bincode_db()
+ .deserialize(&bytes)
+ .map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
}
}
diff --git a/dbs/src/values/utxo.rs b/dbs/src/values/utxo.rs
index dc5b60e..2c8bb4b 100644
--- a/dbs/src/values/utxo.rs
+++ b/dbs/src/values/utxo.rs
@@ -113,7 +113,9 @@ pub struct BlockUtxosV2Db(pub HashMap<UtxoIdV10, WalletScriptWithSourceAmountV1D
impl AsBytes for BlockUtxosV2Db {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
- f(&bincode::serialize(&self).unwrap_or_else(|_| unreachable!()))
+ f(&bincode_db()
+ .serialize(&self)
+ .unwrap_or_else(|_| unreachable!()))
}
}
@@ -121,7 +123,7 @@ impl kv_typed::prelude::FromBytes for BlockUtxosV2Db {
type Err = bincode::Error;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- bincode::deserialize(bytes)
+ bincode_db().deserialize(bytes)
}
}
diff --git a/dbs/src/values/wallet_script_with_sa.rs b/dbs/src/values/wallet_script_with_sa.rs
index 5d40dd9..251a964 100644
--- a/dbs/src/values/wallet_script_with_sa.rs
+++ b/dbs/src/values/wallet_script_with_sa.rs
@@ -23,7 +23,9 @@ pub struct WalletScriptWithSourceAmountV1Db {
impl AsBytes for WalletScriptWithSourceAmountV1Db {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
- f(&bincode::serialize(&self).unwrap_or_else(|_| unreachable!()))
+ f(&bincode_db()
+ .serialize(&self)
+ .unwrap_or_else(|_| unreachable!()))
}
}
@@ -31,7 +33,7 @@ impl kv_typed::prelude::FromBytes for WalletScriptWithSourceAmountV1Db {
type Err = bincode::Error;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
- bincode::deserialize(bytes)
+ bincode_db().deserialize(bytes)
}
}
diff --git a/tools/kv_typed/Cargo.toml b/tools/kv_typed/Cargo.toml
index 8dc44bb..0b8a4a1 100644
--- a/tools/kv_typed/Cargo.toml
+++ b/tools/kv_typed/Cargo.toml
@@ -23,7 +23,7 @@ rayon = { version = "1.3.1", optional = true }
regex = { version = "1.3.9", optional = true }
serde_json = { version = "1.0.53", optional = true }
sled = { version = "0.34.6", optional = true }
-smallvec = { version = "1.4.0", features = ["serde", "write"] }
+smallvec = { version = "1.6", features = ["serde", "write"] }
thiserror = "1.0.20"
uninit = "0.4.0"
zerocopy = "0.3.0"
@@ -36,7 +36,7 @@ required-features = ["leveldb_backend", "sled_backend"]
[dev-dependencies]
async-std = { version = "1.6.3", features = ["attributes"] }
maybe-async = "0.2.0"
-smallvec = { version = "1.4.0", features = ["serde", "write"] }
+smallvec = { version = "1.6", features = ["serde", "write"] }
tempfile = "3.2.0"
unwrap = "1.2.1"
--
GitLab