Draft: (paused) distance result precomputation

Outputs the result of the distance computation for all identities in a json file in the form:

{"block":"0x432a43af2b2aef4ab33ac9a4323b1f084b86388bc46744f7ce5936a5bbcd07f6","results":{"4":4,"1":4,"3":4,"2":4},"referees":4}

This file can be computed regularly and served directly through nginx. This would allow the clients to have a recent estimate of the distance result before requesting evaluation.

distance-oracle/src/bin/distance_precompute.rs

+use api::IdtyIndex;
+use distance_oracle::api;
+use fnv::{FnvHashMap, FnvHashSet};
+use log::debug;
+use std::{io::Write, path::PathBuf};
+
+// this script is mainly copied from distance oracle code
+// it adds values not necessary in distance computation but convenient to have
+// it allows to have a recent estimate of the distance computation
+// intended use:
+// - compute distance result for all identities with received certifications
+// - put the result in a json file to be served publicly
+
+use clap::Parser;
+
+#[derive(Debug, clap::Parser)]
+struct Cli {
+    #[clap(short = 'd', long, default_value = "/tmp/duniter/chains/gdev/distance")]
+    evaluation_result_dir: String,
+    #[clap(short = 'u', long, default_value = "ws://127.0.0.1:9944")]
+    rpc_url: String,
+    /// Log level (off, error, warn, info, debug, trace)
+    #[clap(short = 'l', long, default_value = "info")]
+    log: log::LevelFilter,
+}
+
+/// computation result as it will be serialized in json
+#[derive(serde::Serialize)]
+struct PrecomputationResult {
+    height: u32,
+    block: sp_core::H256,
+    referees_count: u32,
+    member_count: u32,
+    min_certs_for_referee: u32,
+    results: FnvHashMap<IdtyIndex, u32>,
+}
+
+#[tokio::main]
+async fn main() {
+    let cli = Cli::parse();
+
+    simple_logger::SimpleLogger::new()
+        .with_level(cli.log)
+        .init()
+        .unwrap();
+
+    let client = &distance_oracle::api::client(cli.rpc_url.clone()).await;
+    let block_hash = api::parent_hash(client).await; // get hash of recent block
+    let block_height = client.blocks().at(block_hash).await.unwrap().number();
+    let max_depth = api::max_referee_distance(client).await; // get param
+
+    // get certs and member iterators
+    let mut certs_iter = api::cert_iter(client, block_hash).await;
+    let mut members_iter = api::member_iter(client, block_hash).await;
+
+    // initialize hashmaps
+    // member idty -> issued certs count (will only retain referees)
+    let mut members = FnvHashMap::<IdtyIndex, u32>::default();
+
+    // idty -> received certs (collects once from iterator and is passed to distance function)
+    let mut received_certs = FnvHashMap::<IdtyIndex, Vec<IdtyIndex>>::default();
+
+    // collect all members
+    while let Some(member_idty) = members_iter
+        .next()
+        .await
+        .expect("Cannot fetch next members")
+    {
+        members.insert(member_idty, 0);
+    }
+    let member_count = members.len();
+
+    // compute certification threshold as a function of the total member count
+    let min_certs_for_referee = (member_count as f32).powf(1. / (max_depth as f32)).ceil() as u32;
+
+    // collect certifications and updates members map at the same time
+    while let Some((receiver, issuers)) = certs_iter
+        .next()
+        .await
+        .expect("Cannot fetch next certification")
+    {
+        if (issuers.len() as u32) < min_certs_for_referee {
+            // This member is not referee (not enough received certs)
+            members.remove(&receiver);
+        }
+        for (issuer, _removable_on) in issuers.iter() {
+            if let Some(issued_certs) = members.get_mut(issuer) {
+                // update number of issued certs
+                *issued_certs += 1;
+            }
+        }
+        // collect certifications
+        received_certs.insert(
+            receiver,
+            issuers
+                .into_iter()
+                .map(|(issuer, _removable_on)| issuer)
+                .collect(),
+        );
+    }
+
+    // Only retain referees
+    members.retain(|_idty, issued_certs| *issued_certs >= min_certs_for_referee);
+    let referees = members;
+    let referees_count = referees.len();
+
+    // initialize map of distance results
+    let mut results = FnvHashMap::<IdtyIndex, u32>::default();
+
+    // compute all distances (optimization: parallel)
+    received_certs.keys().for_each(|idty| {
+        results.insert(
+            *idty,
+            distance_rule(&received_certs, &referees, max_depth, *idty) as u32,
+        );
+    });
+
+    // put results in a struct for serialization
+    let precomputation_result = PrecomputationResult {
+        height: block_height,
+        block: block_hash,
+        min_certs_for_referee,
+        referees_count: referees_count as u32,
+        member_count: member_count as u32,
+        results,
+    };
+
+    // ---- SAVE
+    const FILENAME: &str = "latest_distance.json";
+    let evaluation_result_path = Into::<PathBuf>::into(cli.evaluation_result_dir).join(FILENAME);
+    debug!("Saving distance evaluation result to file `{evaluation_result_path:?}`");
+    let mut evaluation_result_file = std::fs::OpenOptions::new()
+        .write(true)
+        .create(true)
+        .truncate(true)
+        .open(&evaluation_result_path)
+        .unwrap_or_else(|e| {
+            panic!(
+                "Cannot open distance evaluation result file `{evaluation_result_path:?}`: {e:?}"
+            )
+        });
+    evaluation_result_file
+        .write_all(
+            &serde_json::to_vec( &precomputation_result).expect("Cannot serialize result"),
+        )
+        .unwrap_or_else(|e| {
+            panic!(
+                "Cannot write distance evaluation result to file `{evaluation_result_path:?}`: {e:?}"
+            )
+        });
+}
+
+// alternative function which returns number of reached referees instead of percentage
+fn distance_rule(
+    received_certs: &FnvHashMap<IdtyIndex, Vec<IdtyIndex>>,
+    referees: &FnvHashMap<IdtyIndex, u32>,
+    depth: u32,
+    idty: IdtyIndex,
+) -> usize {
+    let mut accessible_referees =
+        FnvHashSet::<IdtyIndex>::with_capacity_and_hasher(referees.len(), Default::default());
+    let mut known_idties =
+        FnvHashMap::<IdtyIndex, u32>::with_capacity_and_hasher(referees.len(), Default::default());
+    distance_oracle::distance_rule_recursive(
+        received_certs,
+        referees,
+        idty,
+        &mut accessible_referees,
+        &mut known_idties,
+        depth,
+    );
+    accessible_referees.len()
+}