[feat] bc: indexing utxos

Cargo.lock

@@ -1124,6 +1124,7 @@ dependencies = [
  "duniter-dbs",
  "fast-threadpool",
  "log",
+ "maplit",
  "resiter",
  "serde_json",
 ]

rust-libs/duniter-dbs-write-ops/Cargo.toml

@@ -22,6 +22,7 @@ resiter = "0.4.0"
 [dev-dependencies]
 anyhow = "1.0.34"
 duniter-dbs = { path = "../duniter-dbs", features = ["mem"] }
+maplit = "1.0.2"
 serde_json = "1.0.53"
 
 [features]

rust-libs/duniter-dbs-write-ops/src/bc.rs

@@ -55,6 +55,8 @@ pub fn apply_block<B: Backend>(
         bc_db.uds_write(),
         bc_db.uds_reval_write(),
         bc_db.uids_index_write(),
+        bc_db.utxos_write(),
+        bc_db.consumed_utxos_write(),
     )
         .write(
             |(
@@ -64,6 +66,8 @@ pub fn apply_block<B: Backend>(
                 mut uds,
                 mut uds_reval,
                 mut uids_index,
+                mut utxos,
+                mut consumed_utxos,
             )| {
                 blocks_meta.upsert(U32BE(block.number().0), block_meta);
                 identities::update_identities::<B>(&block, &mut identities)?;
@@ -81,14 +85,35 @@ pub fn apply_block<B: Backend>(
                         &mut uds_reval,
                     )?;
                 }
-                txs::apply_txs::<B>(block.transactions(), &mut txs_hashs, &mut uds)?;
+                txs::apply_txs::<B>(
+                    block.number(),
+                    block.transactions(),
+                    &mut txs_hashs,
+                    &mut uds,
+                    &mut utxos,
+                    &mut consumed_utxos,
+                )?;
                 Ok(())
             },
         )?;
 
+    if block_meta.number > ROLL_BACK_MAX {
+        prune_bc_db(bc_db, BlockNumber(block_meta.number))?;
+    }
+
     Ok(block_meta)
 }
 
+fn prune_bc_db<B: Backend>(
+    bc_db: &duniter_dbs::databases::bc_v2::BcV2Db<B>,
+    current_block_number: BlockNumber,
+) -> KvResult<()> {
+    bc_db
+        .consumed_utxos_write()
+        .remove(U32BE(current_block_number.0 - ROLL_BACK_MAX))?;
+    Ok(())
+}
+
 pub fn revert_block<B: Backend>(
     bc_db: &duniter_dbs::databases::bc_v2::BcV2Db<B>,
     block: &DubpBlockV10,
@@ -100,6 +125,8 @@ pub fn revert_block<B: Backend>(
         bc_db.uds_write(),
         bc_db.uds_reval_write(),
         bc_db.uids_index_write(),
+        bc_db.utxos_write(),
+        bc_db.consumed_utxos_write(),
     )
         .write(
             |(
@@ -109,8 +136,17 @@ pub fn revert_block<B: Backend>(
                 mut uds,
                 mut uds_reval,
                 mut uids_index,
+                mut utxos,
+                mut consumed_utxos,
             )| {
-                txs::revert_txs::<B>(block.transactions(), &mut txs_hashs, &mut uds)?;
+                txs::revert_txs::<B>(
+                    block.number(),
+                    block.transactions(),
+                    &mut txs_hashs,
+                    &mut uds,
+                    &mut utxos,
+                    &mut consumed_utxos,
+                )?;
                 if block.dividend().is_some() {
                     uds::revert_uds::<B>(
                         block.number(),
@@ -133,3 +169,177 @@ pub fn revert_block<B: Backend>(
             },
         )
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use dubp::{
+        crypto::keys::PublicKey as _, documents::transaction::TransactionDocumentV10Stringified,
+        documents_parser::prelude::FromStringObject,
+    };
+    use duniter_dbs::{
+        databases::bc_v2::*, BlockUtxosV2Db, UtxoIdDbV2, WalletScriptWithSourceAmountV1Db,
+    };
+    use maplit::hashmap;
+
+    #[test]
+    fn test_bc_apply_block() -> anyhow::Result<()> {
+        let bc_db = BcV2Db::<Mem>::open(MemConf::default())?;
+
+        let s1 = WalletScriptV10::single_sig(PublicKey::from_base58(
+            "D9D2zaJoWYWveii1JRYLVK3J4Z7ZH3QczoKrnQeiM6mx",
+        )?);
+        let s2 = WalletScriptV10::single_sig(PublicKey::from_base58(
+            "4fHMTFBMo5sTQEc5p1CNWz28S4mnnqdUBmECq1zt4n2m",
+        )?);
+
+        let b0 = DubpBlockV10::from_string_object(&DubpBlockV10Stringified {
+            version: 10,
+            median_time: 5_243,
+            dividend: Some(1000),
+            joiners: vec!["D9D2zaJoWYWveii1JRYLVK3J4Z7ZH3QczoKrnQeiM6mx:FFeyrvYio9uYwY5aMcDGswZPNjGLrl8THn9l3EPKSNySD3SDSHjCljSfFEwb87sroyzJQoVzPwER0sW/cbZMDg==:0-E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855:0-E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855:elois".to_owned()],
+            inner_hash: Some("0000000A65A12DB95B3153BCD05DB4D5C30CC7F0B1292D9FFBC3DE67F72F6040".to_owned()),
+            signature: "7B0hvcfajE2G8nBLp0vLVaQcQdQIyli21Gu8F2l+nimKHRe+fUNi+MWd1e/u29BYZa+RZ1yxhbHIbFzytg7fAA==".to_owned(),
+            hash: Some("0000000000000000000000000000000000000000000000000000000000000000".to_owned()),
+            ..Default::default()
+        })?;
+
+        apply_block(&bc_db, &b0)?;
+
+        assert_eq!(bc_db.blocks_meta().count()?, 1);
+        assert_eq!(bc_db.uds().count()?, 1);
+        assert_eq!(bc_db.utxos().count()?, 0);
+        assert_eq!(bc_db.consumed_utxos().count()?, 0);
+
+        let b1 = DubpBlockV10::from_string_object(&DubpBlockV10Stringified {
+            number: 1,
+            version: 10,
+            median_time: 5_245,
+            transactions: vec![TransactionDocumentV10Stringified {
+                currency: "test".to_owned(),
+                blockstamp: "0-0000000000000000000000000000000000000000000000000000000000000000".to_owned(),
+                locktime: 0,
+                issuers: vec!["D9D2zaJoWYWveii1JRYLVK3J4Z7ZH3QczoKrnQeiM6mx".to_owned()],
+                inputs: vec!["1000:0:D:D9D2zaJoWYWveii1JRYLVK3J4Z7ZH3QczoKrnQeiM6mx:0".to_owned()],
+                unlocks: vec![],
+                outputs: vec![
+                    "600:0:SIG(4fHMTFBMo5sTQEc5p1CNWz28S4mnnqdUBmECq1zt4n2m)".to_owned(),
+                    "400:0:SIG(D9D2zaJoWYWveii1JRYLVK3J4Z7ZH3QczoKrnQeiM6mx)".to_owned(),
+                ],
+                comment: "".to_owned(),
+                signatures: vec![],
+                hash: Some("0000000000000000000000000000000000000000000000000000000000000000".to_owned()),
+            }],
+            inner_hash: Some("0000000A65A12DB95B3153BCD05DB4D5C30CC7F0B1292D9FFBC3DE67F72F6040".to_owned()),
+            signature: "7B0hvcfajE2G8nBLp0vLVaQcQdQIyli21Gu8F2l+nimKHRe+fUNi+MWd1e/u29BYZa+RZ1yxhbHIbFzytg7fAA==".to_owned(),
+            hash: Some("0000000000000000000000000000000000000000000000000000000000000000".to_owned()),
+            ..Default::default()
+        })?;
+
+        apply_block(&bc_db, &b1)?;
+
+        assert_eq!(bc_db.blocks_meta().count()?, 2);
+        assert_eq!(bc_db.uds().count()?, 0);
+        assert_eq!(bc_db.utxos().count()?, 2);
+        assert_eq!(
+            bc_db
+                .utxos()
+                .iter(.., |it| it.collect::<KvResult<Vec<_>>>())?,
+            vec![
+                (
+                    UtxoIdDbV2(Hash::default(), 0),
+                    WalletScriptWithSourceAmountV1Db {
+                        wallet_script: s2.clone(),
+                        source_amount: SourceAmount::with_base0(600)
+                    }
+                ),
+                (
+                    UtxoIdDbV2(Hash::default(), 1),
+                    WalletScriptWithSourceAmountV1Db {
+                        wallet_script: s1.clone(),
+                        source_amount: SourceAmount::with_base0(400)
+                    }
+                )
+            ]
+        );
+        assert_eq!(bc_db.consumed_utxos().count()?, 0);
+
+        let b2 = DubpBlockV10::from_string_object(&DubpBlockV10Stringified {
+            number: 2,
+            version: 10,
+            median_time: 5_247,
+            transactions: vec![TransactionDocumentV10Stringified {
+                currency: "test".to_owned(),
+                blockstamp: "0-0000000000000000000000000000000000000000000000000000000000000000".to_owned(),
+                locktime: 0,
+                issuers: vec!["D9D2zaJoWYWveii1JRYLVK3J4Z7ZH3QczoKrnQeiM6mx".to_owned()],
+                inputs: vec!["400:0:T:0000000000000000000000000000000000000000000000000000000000000000:1".to_owned()],
+                unlocks: vec![],
+                outputs: vec![
+                    "300:0:SIG(D9D2zaJoWYWveii1JRYLVK3J4Z7ZH3QczoKrnQeiM6mx)".to_owned(),
+                    "100:0:SIG(4fHMTFBMo5sTQEc5p1CNWz28S4mnnqdUBmECq1zt4n2m)".to_owned(),
+                ],
+                comment: "".to_owned(),
+                signatures: vec![],
+                hash: Some("0101010101010101010101010101010101010101010101010101010101010101".to_owned()),
+            }],
+            inner_hash: Some("0000000A65A12DB95B3153BCD05DB4D5C30CC7F0B1292D9FFBC3DE67F72F6040".to_owned()),
+            signature: "7B0hvcfajE2G8nBLp0vLVaQcQdQIyli21Gu8F2l+nimKHRe+fUNi+MWd1e/u29BYZa+RZ1yxhbHIbFzytg7fAA==".to_owned(),
+            hash: Some("0000000000000000000000000000000000000000000000000000000000000000".to_owned()),
+            ..Default::default()
+        })?;
+
+        apply_block(&bc_db, &b2)?;
+
+        assert_eq!(bc_db.blocks_meta().count()?, 3);
+        assert_eq!(bc_db.uds().count()?, 0);
+        assert_eq!(bc_db.utxos().count()?, 3);
+        assert_eq!(bc_db.consumed_utxos().count()?, 1);
+
+        assert_eq!(
+            bc_db
+                .consumed_utxos()
+                .iter(.., |it| it.collect::<KvResult<Vec<_>>>())?,
+            vec![(
+                U32BE(2),
+                BlockUtxosV2Db(
+                    hashmap![UtxoIdV10 { tx_hash: Hash::default(), output_index: 1 } => WalletScriptWithSourceAmountV1Db {
+                        wallet_script: s1.clone(),
+                        source_amount: SourceAmount::with_base0(400)
+                    }]
+                )
+            )]
+        );
+
+        assert_eq!(
+            bc_db
+                .utxos()
+                .iter(.., |it| it.collect::<KvResult<Vec<_>>>())?,
+            vec![
+                (
+                    UtxoIdDbV2(Hash::default(), 0),
+                    WalletScriptWithSourceAmountV1Db {
+                        wallet_script: s2.clone(),
+                        source_amount: SourceAmount::with_base0(600)
+                    }
+                ),
+                (
+                    UtxoIdDbV2(Hash([1; 32]), 0),
+                    WalletScriptWithSourceAmountV1Db {
+                        wallet_script: s1,
+                        source_amount: SourceAmount::with_base0(300)
+                    }
+                ),
+                (
+                    UtxoIdDbV2(Hash([1; 32]), 1),
+                    WalletScriptWithSourceAmountV1Db {
+                        wallet_script: s2,
+                        source_amount: SourceAmount::with_base0(100)
+                    }
+                )
+            ]
+        );
+
+        Ok(())
+    }
+}

rust-libs/duniter-dbs-write-ops/src/bc/txs.rs

@@ -13,40 +13,105 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+use std::collections::HashMap;
+
 use crate::*;
+use dubp::documents::transaction::TransactionOutputV10;
 use duniter_dbs::{
-    databases::bc_v2::{TxsHashsEvent, UdsEvent},
-    UdIdV2,
+    databases::bc_v2::{ConsumedUtxosEvent, TxsHashsEvent, UdsEvent, UtxosEvent},
+    BlockUtxosV2Db, UdIdV2, UtxoIdDbV2, WalletScriptWithSourceAmountV1Db,
 };
 
 pub(crate) fn apply_txs<B: Backend>(
+    block_number: BlockNumber,
     block_txs: &[TransactionDocumentV10],
     txs_hashs: &mut TxColRw<B::Col, TxsHashsEvent>,
     uds: &mut TxColRw<B::Col, UdsEvent>,
+    utxos: &mut TxColRw<B::Col, UtxosEvent>,
+    consumed_utxos: &mut TxColRw<B::Col, ConsumedUtxosEvent>,
 ) -> KvResult<()> {
+    if !block_txs.is_empty() {
+        let mut block_consumed_utxos = HashMap::with_capacity(block_txs.len() * 3);
         for tx in block_txs {
-        txs_hashs.upsert(HashKeyV2(tx.get_hash()), ());
+            let tx_hash = tx.get_hash();
+            txs_hashs.upsert(HashKeyV2(tx_hash), ());
             for input in tx.get_inputs() {
-            if let SourceIdV10::Ud(UdSourceIdV10 {
+                match input.id {
+                    SourceIdV10::Ud(UdSourceIdV10 {
                         issuer,
                         block_number,
-            }) = input.id
-            {
+                    }) => {
                         uds.remove(UdIdV2(issuer, block_number));
                     }
+                    SourceIdV10::Utxo(utxo_id) => {
+                        let utxo_id_db = UtxoIdDbV2(utxo_id.tx_hash, utxo_id.output_index as u32);
+                        if let Some(wallet_script_with_sa) = utxos.get(&utxo_id_db)? {
+                            utxos.remove(utxo_id_db);
+                            block_consumed_utxos.insert(utxo_id, wallet_script_with_sa);
+                        } else {
+                            return Err(KvError::Custom(
+                                format!("db corrupted: not found utxo {:?}", utxo_id_db).into(),
+                            ));
+                        }
+                    }
+                }
+            }
+            for (output_index, TransactionOutputV10 { amount, conditions }) in
+                tx.get_outputs().iter().enumerate()
+            {
+                let utxo_id = UtxoIdDbV2(tx_hash, output_index as u32);
+                let wallet_script_with_sa = WalletScriptWithSourceAmountV1Db {
+                    wallet_script: conditions.script.clone(),
+                    source_amount: *amount,
+                };
+                utxos.upsert(utxo_id, wallet_script_with_sa);
+            }
+        }
+        if !block_consumed_utxos.is_empty() {
+            consumed_utxos.upsert(U32BE(block_number.0), BlockUtxosV2Db(block_consumed_utxos));
         }
     }
     Ok(())
 }
 
 pub(crate) fn revert_txs<B: Backend>(
+    block_number: BlockNumber,
     block_txs: &[TransactionDocumentV10],
     txs_hashs: &mut TxColRw<B::Col, TxsHashsEvent>,
     uds: &mut TxColRw<B::Col, UdsEvent>,
+    utxos: &mut TxColRw<B::Col, UtxosEvent>,
+    consumed_utxos: &mut TxColRw<B::Col, ConsumedUtxosEvent>,
 ) -> KvResult<()> {
     for tx in block_txs {
-        txs_hashs.remove(HashKeyV2(tx.get_hash()));
+        let tx_hash = tx.get_hash();
+        txs_hashs.remove(HashKeyV2(tx_hash));
         for input in tx.get_inputs() {
+            match input.id {
+                SourceIdV10::Ud(UdSourceIdV10 {
+                    issuer,
+                    block_number,
+                }) => {
+                    uds.upsert(UdIdV2(issuer, block_number), ());
+                }
+                SourceIdV10::Utxo(utxo_id) => {
+                    let utxo_id_db = UtxoIdDbV2(utxo_id.tx_hash, utxo_id.output_index as u32);
+                    if let Some(block_utxos) = consumed_utxos.get(&U32BE(block_number.0))? {
+                        if let Some(wallet_script_with_sa) = block_utxos.0.get(&utxo_id) {
+                            utxos.upsert(utxo_id_db, wallet_script_with_sa.clone());
+                        } else {
+                            return Err(KvError::Custom(
+                                format!("db corrupted: not found consumed utxos {}", utxo_id)
+                                    .into(),
+                            ));
+                        }
+                    } else {
+                        return Err(KvError::Custom(
+                            format!("db corrupted: not found consumed utxos {:?}", utxo_id_db)
+                                .into(),
+                        ));
+                    }
+                }
+            }
             if let SourceIdV10::Ud(UdSourceIdV10 {
                 issuer,
                 block_number,
@@ -55,6 +120,11 @@ pub(crate) fn revert_txs<B: Backend>(
                 uds.upsert(UdIdV2(issuer, block_number), ());
             }
         }
+        for output_index in 0..tx.get_outputs().len() {
+            let utxo_id = UtxoIdDbV2(tx_hash, output_index as u32);
+            utxos.remove(utxo_id);
+        }
     }
+    consumed_utxos.remove(U32BE(block_number.0));
     Ok(())
 }

rust-libs/duniter-dbs-write-ops/src/lib.rs

@@ -49,3 +49,5 @@ use resiter::filter_map::FilterMap;
 use resiter::flatten::Flatten;
 use resiter::map::Map;
 use std::ops::Deref;
+
+const ROLL_BACK_MAX: u32 = 100;

rust-libs/duniter-dbs/src/databases/bc_v2.rs

@@ -24,5 +24,7 @@ db_schema!(
         ["uds", Uds, UdIdV2, ()],
         ["uds_reval", UdsReval, U32BE, SourceAmountValV2],
         ["uids_index", UidsIndex, String, PubKeyValV2],
+        ["utxos", Utxos, UtxoIdDbV2, WalletScriptWithSourceAmountV1Db],
+        ["consumed_utxos", ConsumedUtxos, U32BE, BlockUtxosV2Db],
     ]
 );

rust-libs/duniter-dbs/src/lib.rs

@@ -83,9 +83,10 @@ pub use values::sindex_db::{SIndexDBV1, SourceKeyArrayDbV1};
 pub use values::source_amount::SourceAmountValV2;
 pub use values::tx_db::{PendingTxDbV2, TxDbV2};
 pub use values::ud_entry_db::{ConsumedUdDbV1, UdAmountDbV1, UdEntryDbV1};
-pub use values::utxo::UtxoValV2;
+pub use values::utxo::{BlockUtxosV2Db, UtxoValV2};
 pub use values::wallet_db::WalletDbV1;
 pub use values::wallet_script_array::WalletScriptArrayV2;
+pub use values::wallet_script_with_sa::WalletScriptWithSourceAmountV1Db;
 
 // Crate imports
 pub(crate) use arrayvec::{ArrayString, ArrayVec};

rust-libs/duniter-dbs/src/values.rs

@@ -33,3 +33,4 @@ pub mod ud_entry_db;
 pub mod utxo;
 pub mod wallet_db;
 pub mod wallet_script_array;
+pub mod wallet_script_with_sa;

rust-libs/duniter-dbs/src/values/utxo.rs

@@ -14,7 +14,7 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 use crate::*;
-use std::ops::Deref;
+use std::{collections::HashMap, ops::Deref};
 
 #[derive(
     Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, zerocopy::AsBytes, zerocopy::FromBytes,
@@ -108,6 +108,39 @@ impl ExplorableValue for UtxoValV2 {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
+pub struct BlockUtxosV2Db(pub HashMap<UtxoIdV10, WalletScriptWithSourceAmountV1Db>);
+
+impl ValueAsBytes for BlockUtxosV2Db {
+    fn as_bytes<T, F: FnMut(&[u8]) -> KvResult<T>>(&self, mut f: F) -> KvResult<T> {
+        f(&bincode::serialize(&self).map_err(|e| KvError::DeserError(format!("{}", e)))?)
+    }
+}
+
+impl kv_typed::prelude::FromBytes for BlockUtxosV2Db {
+    type Err = StringErr;
+
+    fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
+        Ok(bincode::deserialize(bytes).map_err(|e| StringErr(format!("{}", e)))?)
+    }
+}
+
+impl ToDumpString for BlockUtxosV2Db {
+    fn to_dump_string(&self) -> String {
+        todo!()
+    }
+}
+
+#[cfg(feature = "explorer")]
+impl ExplorableValue for BlockUtxosV2Db {
+    fn from_explorer_str(_: &str) -> std::result::Result<Self, StringErr> {
+        unimplemented!()
+    }
+    fn to_explorer_json(&self) -> KvResult<serde_json::Value> {
+        Ok(serde_json::to_value(self).map_err(|e| KvError::DeserError(e.to_string()))?)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

rust-libs/duniter-dbs/src/values/wallet_script_with_sa.rs

+//  Copyright (C) 2020 Éloïs SANCHEZ.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+use crate::*;
+
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
+pub struct WalletScriptWithSourceAmountV1Db {
+    pub wallet_script: WalletScriptV10,
+    pub source_amount: SourceAmount,
+}
+
+impl ValueAsBytes for WalletScriptWithSourceAmountV1Db {
+    fn as_bytes<T, F: FnMut(&[u8]) -> KvResult<T>>(&self, mut f: F) -> KvResult<T> {
+        f(&bincode::serialize(&self).map_err(|e| KvError::DeserError(format!("{}", e)))?)
+    }
+}
+
+impl kv_typed::prelude::FromBytes for WalletScriptWithSourceAmountV1Db {
+    type Err = StringErr;
+
+    fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
+        Ok(bincode::deserialize(bytes).map_err(|e| StringErr(format!("{}", e)))?)
+    }
+}
+
+impl ToDumpString for WalletScriptWithSourceAmountV1Db {
+    fn to_dump_string(&self) -> String {
+        todo!()
+    }
+}
+
+#[cfg(feature = "explorer")]
+impl ExplorableValue for WalletScriptWithSourceAmountV1Db {
+    fn from_explorer_str(_: &str) -> std::result::Result<Self, StringErr> {
+        unimplemented!()
+    }
+    fn to_explorer_json(&self) -> KvResult<serde_json::Value> {
+        Ok(serde_json::to_value(self).map_err(|e| KvError::DeserError(e.to_string()))?)
+    }
+}