From 35585c696339f679c790c1e52bad6bfa531f86d9 Mon Sep 17 00:00:00 2001
From: Hugo Trentesaux <hugo@trentesaux.fr>
Date: Fri, 25 Oct 2024 11:36:46 +0200
Subject: [PATCH] wip more reading
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
not easy 😢
---
Cargo.lock | 2 +
Cargo.toml | 8 ++-
node/Cargo.toml | 29 ++++++---
node/src/cli.rs | 2 +-
node/src/command.rs | 7 +--
node/src/endpoint_gossip/mod.rs | 107 +++++++++++++++++++++++++-------
node/src/service.rs | 61 +++++++++---------
7 files changed, 148 insertions(+), 68 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 0d42a0448..57cf6dec0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2181,6 +2181,7 @@ dependencies = [
name = "duniter"
version = "0.8.0"
dependencies = [
+ "array-bytes",
"async-io",
"bs58 0.5.1",
"clap",
@@ -2206,6 +2207,7 @@ dependencies = [
"pallet-transaction-payment-rpc",
"pallet-transaction-payment-rpc-runtime-api",
"pallet-treasury",
+ "parity-scale-codec",
"sc-basic-authorship",
"sc-chain-spec",
"sc-cli",
diff --git a/Cargo.toml b/Cargo.toml
index 8dafdaef0..0a61728d9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -64,13 +64,15 @@ bs58 = { version = "0.5.1", default-features = false }
placeholder = { version = "1.1.4", default-features = false }
clap = { version = "4.5.3" }
clap_complete = { version = "4.5.1" }
-reqwest = { version = "0.12.0", default-features = false, features = ["rustls-tls"] }
+reqwest = { version = "0.12.0", default-features = false, features = [
+ "rustls-tls",
+] }
glob = { version = "0.3.1", default-features = false }
convert_case = { version = "0.6.0", default-features = false }
subweight-core = { version = "3.3.1", default-features = false }
version_check = { version = "0.9.4", default-features = false }
codec = { package = "parity-scale-codec", version = "3.6.9", default-features = false }
-enum-as-inner = { version = "=0.5.1", default-features = false } #https://github.com/bluejekyll/trust-dns/issues/1946
+enum-as-inner = { version = "=0.5.1", default-features = false } #https://github.com/bluejekyll/trust-dns/issues/1946
futures = { version = "0.3.30", default-features = false }
tera = { version = "1", default-features = false }
hex = { version = "0.4.3", default-features = false }
@@ -101,6 +103,7 @@ simple_logger = { version = "4.3.3", default-features = false }
bincode = { version = "1.3.3", default-features = false }
dubp-wot = { version = "0.11.1", default-features = false }
flate2 = { version = "1.0.28", default-features = false }
+array-bytes = { version = "6.2.2", default-features = false }
# Subxt
subxt = { git = 'https://github.com/duniter/subxt', branch = 'subxt-v0.37.0-duniter-substrate-v1.16.0', default-features = false }
@@ -274,4 +277,3 @@ zeroize = { opt-level = 3 }
lto = "thin"
# Substrate runtime requires unwinding.
panic = "unwind"
-
diff --git a/node/Cargo.toml b/node/Cargo.toml
index 784c75437..dcfedb5d5 100644
--- a/node/Cargo.toml
+++ b/node/Cargo.toml
@@ -48,7 +48,7 @@ runtime-benchmarks = [
"sp-runtime/runtime-benchmarks",
]
try-runtime = [
- "runtime-benchmarks",
+ "runtime-benchmarks",
"common-runtime/try-runtime",
"dc-distance?/try-runtime",
"distance-oracle?/try-runtime",
@@ -125,6 +125,8 @@ num-format = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
serde_yaml = { workspace = true }
+codec = { workspace = true }
+array-bytes = { workspace = true }
# Local
common-runtime = { workspace = true }
@@ -184,7 +186,7 @@ sp-transaction-storage-proof = { workspace = true, default-features = true }
substrate-frame-rpc-system = { workspace = true, default-features = true }
[build-dependencies]
-substrate-build-script-utils = { workspace = true, default-features = true}
+substrate-build-script-utils = { workspace = true, default-features = true }
# Dependencies for specific targets
[target.'cfg(any(target_arch="x86_64", target_arch="aarch64"))'.dependencies]
@@ -194,9 +196,20 @@ sp-trie = { workspace = true, default-features = true }
[package.metadata.deb]
maintainer-scripts = "../resources/debian"
-systemd-units = [ { unit-name = "duniter-mirror", enable = false },
- { unit-name = "duniter-smith", enable = false },
- { unit-name = "distance-oracle", enable = false },
- ]
-assets = [ ["../resources/debian/env_file", "/etc/duniter/env_file", "0640"],
- ["../target/release/duniter", "/usr/bin/duniter2", "755"]]
+systemd-units = [
+ { unit-name = "duniter-mirror", enable = false },
+ { unit-name = "duniter-smith", enable = false },
+ { unit-name = "distance-oracle", enable = false },
+]
+assets = [
+ [
+ "../resources/debian/env_file",
+ "/etc/duniter/env_file",
+ "0640",
+ ],
+ [
+ "../target/release/duniter",
+ "/usr/bin/duniter2",
+ "755",
+ ],
+]
diff --git a/node/src/cli.rs b/node/src/cli.rs
index 647845b5f..6b63b941d 100644
--- a/node/src/cli.rs
+++ b/node/src/cli.rs
@@ -35,7 +35,7 @@ pub struct Cli {
}
/// add options specific to duniter client
-#[derive(Debug, Clone, clap::Parser)]
+#[derive(Debug, Default, Clone, clap::Parser)]
pub struct DuniterConfigExtension {
/// Public RPC endpoint to gossip on the network and make available in the apps.
#[arg(long)]
diff --git a/node/src/command.rs b/node/src/command.rs
index 9bac52396..9b424b059 100644
--- a/node/src/command.rs
+++ b/node/src/command.rs
@@ -22,7 +22,7 @@ pub mod utils;
use crate::{
chain_spec,
- cli::{Cli, Subcommand},
+ cli::{Cli, DuniterConfigExtension, Subcommand},
service,
service::{runtime_executor::Executor, RuntimeType},
};
@@ -364,8 +364,7 @@ pub fn run() -> sc_cli::Result<()> {
}
None => {
let runner = cli.create_runner(&cli.run)?;
- // TODO add endpoint config here
- let config_extension: DuniterConfigExtension = ();
+ let duniter_options: DuniterConfigExtension = cli.duniter_options;
runner.run_node_until_exit(|mut config| async move {
// Force offchain worker and offchain indexing if we have the role Authority
if config.role.is_authority() {
@@ -378,7 +377,7 @@ pub fn run() -> sc_cli::Result<()> {
service::runtime_executor::runtime::RuntimeApi,
Executor,
sc_network::Litep2pNetworkBackend,
- >(config, cli.sealing, config_extension)
+ >(config, cli.sealing, duniter_options)
.map_err(sc_cli::Error::Service)
}
})
diff --git a/node/src/endpoint_gossip/mod.rs b/node/src/endpoint_gossip/mod.rs
index 682463363..5a684e2f1 100644
--- a/node/src/endpoint_gossip/mod.rs
+++ b/node/src/endpoint_gossip/mod.rs
@@ -1,12 +1,22 @@
+use codec::{Decode, DecodeAll, Encode};
+use futures::prelude::*;
use sc_chain_spec::ChainSpec;
-use sc_network::types::ProtocolName;
+use sc_network::{types::ProtocolName, NotificationService};
// endpoint gossip is a single protocol used to gossip different kinds of endpoints like
// - rpc endpoints
// - squid indexer endpoints
// ...
-pub(crate) const NAME: &str = "/endpoint_gossip/1";
+pub const NAME: &str = "/endpoint_gossip/1";
+
+#[derive(Debug, Encode, Decode)]
+enum EndpointMsg {
+ /// RPC endpoint of Duniter
+ RpcEndpoint(String),
+ /// GraphQL endpoint of Squid
+ SquidEndpoint(String),
+}
/// Name of the notifications protocol used by endpoint gossip.
// Prefix by fork id to avoid gossiping endpoints of wrong network
@@ -22,18 +32,17 @@ pub fn protocol_name<Hash: AsRef<[u8]>>(
format!("{}{}", chain_prefix, NAME).into()
}
-/// Returns the configuration value to put in
-/// [`sc_network::config::FullNetworkConfiguration`].
-pub fn peers_set_config<B: BlockT, N: NetworkBackend<B, <B as BlockT>::Hash>>(
+// returns config from protocol name
+pub fn set_config(
protocol_name: ProtocolName,
- metrics: sc_network::service::NotificationMetrics,
- peer_store_handle: Arc<dyn sc_network::peer_store::PeerStoreProvider>,
-) -> (N::NotificationProtocolConfig, Box<dyn NotificationService>) {
- use communication::grandpa_protocol_name;
- N::notification_config(
+) -> (
+ sc_network::config::NonDefaultSetConfig,
+ Box<dyn NotificationService>,
+) {
+ sc_network::config::NonDefaultSetConfig::new(
protocol_name,
vec![],
- // Inspired by GRANDPA notifications which are ~256kiB
+ // Copied from GRANDPA notifications which are ~256kiB
1024 * 1024,
None,
sc_network::config::SetConfig {
@@ -42,19 +51,69 @@ pub fn peers_set_config<B: BlockT, N: NetworkBackend<B, <B as BlockT>::Hash>>(
reserved_nodes: Vec::new(),
non_reserved_mode: sc_network::config::NonReservedPeerMode::Deny,
},
- metrics,
- peer_store_handle,
)
}
-// /// Create notification protocol configuration and an associated `NotificationService`
-// /// for the protocol.
-// fn notification_config(
-// protocol_name: ProtocolName,
-// fallback_names: Vec<ProtocolName>,
-// max_notification_size: u64,
-// handshake: Option<NotificationHandshake>,
-// set_config: SetConfig,
-// metrics: NotificationMetrics,
-// peerstore_handle: Arc<dyn PeerStoreProvider>,
-// ) -> (Self::NotificationProtocolConfig, Box<dyn NotificationService>);
+// copied from grandpa config
+// /substrate/client/consensus/grandpa/src/lib.rs:212
+struct EndpointGossipConfig {
+ /// The expected duration for a message to be gossiped across the network.
+ gossip_duration: std::time::Duration,
+ /// Period (in blocks) to share endpoints again
+ pub share_period: u32,
+ /// Some local identifier of the endpoint.
+ pub name: Option<String>,
+ /// Chain specific endpoint protocol name
+ protocol_name: ProtocolName,
+}
+
+struct EndpointGossipParams<Network> {
+ config: EndpointGossipConfig,
+ network: Network,
+ notification_service: Box<dyn NotificationService>,
+ offchain_endpoint_tester: (),
+}
+
+fn run_gossip_worker<Network>(params: EndpointGossipParams<Network>) {
+ let topic = "XXX";
+
+ // TODO network has no gossip engine, it is created in network bridge
+ // to gossip a message
+ params
+ .network
+ .gossip_engine
+ .lock()
+ .gossip_message(topic, notification.message.clone(), false);
+
+ // to process incoming messages
+ let stream = gossip_engine
+ .clone()
+ .lock()
+ .messages_for(topic)
+ .filter_map(|notification| {
+ let decoded = EndpointMsg::decode_all(&mut ¬ification.message[..]);
+ if let Err(ref e) = decoded {
+ println!(
+ "Skipping malformed commit message {:?}: {}",
+ notification, e
+ );
+ }
+ future::ready(decoded.map(move |d| (notification, d)).ok())
+ })
+ .filter_map(move |(notification, msg)| {
+ future::ready(match msg {
+ EndpointMsg::RpcEndpoint(msg) => {
+ println!("rpc {msg}");
+ Some(())
+ }
+ EndpointMsg::SquidEndpoint(msg) => {
+ println!("rpc {msg}");
+ Some(())
+ }
+ _ => {
+ println!("Skipping unknown message type");
+ None
+ }
+ })
+ });
+}
diff --git a/node/src/service.rs b/node/src/service.rs
index d6b23bc4e..131c41aa5 100644
--- a/node/src/service.rs
+++ b/node/src/service.rs
@@ -127,6 +127,7 @@ pub fn new_chain_ops(
} = new_partial::<runtime_executor::runtime::RuntimeApi, runtime_executor::Executor>(
config,
manual_consensus,
+ Default::default(),
)?;
Ok((
Arc::new(Client::Client(client)),
@@ -147,7 +148,7 @@ type FullGrandpaBlockImport<RuntimeApi, Executor> = sc_consensus_grandpa::Grandp
pub fn new_partial<RuntimeApi, Executor>(
config: &Configuration,
consensus_manual: bool,
- config_extension: DuniterConfigExtension,
+ duniter_config: crate::cli::DuniterConfigExtension,
) -> Result<
sc_service::PartialComponents<
FullClient<RuntimeApi, Executor>,
@@ -304,6 +305,7 @@ pub fn new_full<
>(
config: Configuration,
sealing: crate::cli::Sealing,
+ duniter_options: crate::cli::DuniterConfigExtension,
) -> Result<TaskManager, ServiceError>
where
RuntimeApi: sp_api::ConstructRuntimeApi<Block, FullClient<RuntimeApi, Executor>>
@@ -323,7 +325,11 @@ where
select_chain,
transaction_pool,
other: (block_import, babe_link, babe_worker_handle, grandpa_link, mut telemetry),
- } = new_partial::<RuntimeApi, Executor>(&config, sealing.is_manual_consensus())?;
+ } = new_partial::<RuntimeApi, Executor>(
+ &config,
+ sealing.is_manual_consensus(),
+ duniter_options,
+ )?;
// genesis hash used in protocol names
let genesis_hash = client
@@ -352,42 +358,21 @@ where
);
net_config.add_notification_protocol(grandpa_protocol_config);
- // endpoint gossip
- // only active for non-authority (to save workload)
- // if !role.is_authority() {
- // configure protocol
+ // endpoint gossip network config
let endpoint_gossip_protocol_name =
- endpoint_gossip::protocol_name(&genesis_hash, &config.chain_spec);
+ crate::endpoint_gossip::protocol_name(&genesis_hash, &config.chain_spec);
let (endpoint_gossip_protocol_config, endpoint_gossip_notification_service) =
- endpoint_gossip::peers_set_config::<_, N>(
- endpoint_gossip_protocol_name.clone(),
- metrics.clone(),
- peer_store_handle,
- );
+ crate::endpoint_gossip::set_config(endpoint_gossip_protocol_name.clone());
net_config.add_notification_protocol(endpoint_gossip_protocol_config);
- let endpoint_gossip_config = ();
-
- let endpoint_gossip_params = EndpointGossipParams {
- config: endpoint_gossip_config,
- network,
- notification_service: endpoint_gossip_notification_service,
- // offchain storage
- offchain_endpoint_tester: (),
- };
- // spawn task
- task_manager.spawn_handle().spawn(
- "endpoint-gossip",
- None,
- run_gossip_worker(endpoint_gossip_params)?,
- );
- // }
+ // warp sync network provider
let warp_sync = Arc::new(sc_consensus_grandpa::warp_proof::NetworkProvider::new(
backend.clone(),
grandpa_link.shared_authority_set().clone(),
Vec::default(),
));
+ // build network service from params
let (network, system_rpc_tx, tx_handler_controller, network_starter, sync_service) =
sc_service::build_network(sc_service::BuildNetworkParams {
config: &config,
@@ -402,6 +387,7 @@ where
metrics,
})?;
+ // aliases
let role = config.role;
let force_authoring = config.force_authoring;
let backoff_authoring_blocks: Option<()> = None;
@@ -671,6 +657,7 @@ where
None
};
+ // init grandpa config
let grandpa_config = sc_consensus_grandpa::Config {
gossip_duration: Duration::from_millis(333),
justification_generation_period: GRANDPA_JUSTIFICATION_PERIOD,
@@ -713,6 +700,24 @@ where
);
}
+ // start endpoint gossip task
+ // only active for non-authority (to save workload)
+ // if !role.is_authority() {
+ let endpoint_gossip_params = crate::endpoint_gossip::EndpointGossipParams {
+ config: endpoint_gossip_config,
+ network,
+ notification_service: endpoint_gossip_notification_service,
+ // offchain storage
+ offchain_endpoint_tester: (),
+ };
+ // spawn endpoint gossip task
+ task_manager.spawn_handle().spawn(
+ "endpoint-gossip",
+ None,
+ crate::endpoint_gossip::run_gossip_worker(endpoint_gossip_params)?,
+ );
+ // }
+
network_starter.start_network();
log::info!("***** Duniter has fully started *****");
--
GitLab