From 5d8ba754739157d7697f6e6396f75c8a6b961fde Mon Sep 17 00:00:00 2001
From: Hugo Trentesaux <hugo.trentesaux@lilo.org>
Date: Sun, 28 May 2023 01:08:13 +0200
Subject: [PATCH] prepare gtest runtime (nodes/rust/duniter-v2s!171)
* fix gtest runtime type name
* add embeded client spec and genesis in binary
(this re-buids chainspecs at each run)
* refac chainspecs generation
* add what is needed to build gtest chainspecs
---
Cargo.lock | 1 +
Cargo.toml | 1 +
docs/dev/launch-a-live-network.md | 2 +-
node/specs/gtest_client-specs.json | 20 +++
.../specs/gtest_genesis.json | 0
node/src/chain_spec/gen_genesis_data.rs | 3 +
node/src/chain_spec/gtest.rs | 130 +++++++++++-------
node/src/chain_spec/gtest_genesis.rs | 29 +---
node/src/command.rs | 74 +++++++---
node/src/service.rs | 5 +-
10 files changed, 165 insertions(+), 100 deletions(-)
create mode 100644 node/specs/gtest_client-specs.json
rename resources/gtest.json => node/specs/gtest_genesis.json (100%)
diff --git a/Cargo.lock b/Cargo.lock
index 7c5fc58d6..f6684b603 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1598,6 +1598,7 @@ dependencies = [
"sc-finality-grandpa",
"sc-keystore",
"sc-network",
+ "sc-network-common",
"sc-rpc-api",
"sc-service",
"sc-telemetry",
diff --git a/Cargo.toml b/Cargo.toml
index 42515b104..0896f98b8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -99,6 +99,7 @@ sc-executor = { git = "https://github.com/duniter/substrate", branch = "duniter-
sc-finality-grandpa = { git = "https://github.com/duniter/substrate", branch = "duniter-substrate-v0.9.32" }
sc-keystore = { git = "https://github.com/duniter/substrate", branch = "duniter-substrate-v0.9.32" }
sc-network = { git = "https://github.com/duniter/substrate", branch = "duniter-substrate-v0.9.32" }
+sc-network-common = { git = "https://github.com/duniter/substrate", branch = "duniter-substrate-v0.9.32" }
sc-rpc-api = { git = "https://github.com/duniter/substrate", branch = "duniter-substrate-v0.9.32" }
sc-service = { git = "https://github.com/duniter/substrate", branch = "duniter-substrate-v0.9.32", default-features = false }
sc-telemetry = { git = "https://github.com/duniter/substrate", branch = "duniter-substrate-v0.9.32" }
diff --git a/docs/dev/launch-a-live-network.md b/docs/dev/launch-a-live-network.md
index cc87050d8..fca4accb5 100644
--- a/docs/dev/launch-a-live-network.md
+++ b/docs/dev/launch-a-live-network.md
@@ -67,7 +67,7 @@ An example of genesis configuration file: `resources/gdev.json`. Paste your sess
### Generate raw spec
```docker
-docker run -v $HOME/dev/duniter-v2s/resources:/var/lib/duniter/resources -e DUNITER_GENESIS_CONFIG=/var/lib/duniter/resources/gdev.json --rm duniter/duniter-v2s:TAG -- build-spec -lerror --chain=gdev-gl --raw > name-raw.json
+docker run -v $HOME/dev/duniter-v2s/resources:/var/lib/duniter/resources -e DUNITER_GENESIS_CONFIG=/var/lib/duniter/resources/gdev.json --rm duniter/duniter-v2s:TAG -- build-spec -lerror --chain=gdev_live --raw > name-raw.json
```
```bash
diff --git a/node/specs/gtest_client-specs.json b/node/specs/gtest_client-specs.json
new file mode 100644
index 000000000..b14c21695
--- /dev/null
+++ b/node/specs/gtest_client-specs.json
@@ -0,0 +1,20 @@
+{
+ "name": "ÄžTest",
+ "id": "gtest",
+ "chainType": "Live",
+ "bootNodes": [
+ "/dns/gdev.p2p.legal/tcp/30334/p2p/12D3KooW9v5WsP38qU1kmafvA4CDw2vzYnFoWtdUqwonZtJK597r",
+ "/dns/gdev.elo.tf/tcp/30334/p2p/12D3KooWH4gAzvcRRCH8bFGZYomCGn3tosXDGtnL5yVVMkpGRL7B",
+ "/dns/gdev.librelois.fr/tcp/30334/p2p/12D3KooWMb8hmxdtBQhGLB8caokbAZQ3YokN9erMe6oAEsPDrwYf"
+ ],
+ "telemetryEndpoints": [
+ [
+ "/dns/telemetry.polkadot.io/tcp/443/x-parity-wss/%2Fsubmit%2F",
+ 0
+ ]
+ ],
+ "properties": {
+ "tokenDecimals": 2,
+ "tokenSymbol": "ÄžT"
+ }
+}
\ No newline at end of file
diff --git a/resources/gtest.json b/node/specs/gtest_genesis.json
similarity index 100%
rename from resources/gtest.json
rename to node/specs/gtest_genesis.json
diff --git a/node/src/chain_spec/gen_genesis_data.rs b/node/src/chain_spec/gen_genesis_data.rs
index 5e0736824..b26ebe3ad 100644
--- a/node/src/chain_spec/gen_genesis_data.rs
+++ b/node/src/chain_spec/gen_genesis_data.rs
@@ -98,6 +98,9 @@ impl From<Cert> for (String, Option<u32>) {
}
}
+/// generate genesis data from a json file
+/// takes DUNITER_GENESIS_CONFIG env var if present or duniter-gen-conf.json by default
+// this function is targeting dev chainspecs, do not use in production network
pub fn generate_genesis_data<CS, P, SK, F>(
f: F,
maybe_force_authority: Option<Vec<u8>>,
diff --git a/node/src/chain_spec/gtest.rs b/node/src/chain_spec/gtest.rs
index d45890721..023aef9f3 100644
--- a/node/src/chain_spec/gtest.rs
+++ b/node/src/chain_spec/gtest.rs
@@ -18,13 +18,18 @@ use super::*;
use common_runtime::constants::*;
use common_runtime::entities::IdtyData;
use common_runtime::*;
+use gtest_genesis::{build_genesis, GenesisJson};
use gtest_runtime::{
opaque::SessionKeys, AccountConfig, AccountId, AuthorityMembersConfig, BabeConfig, CertConfig,
GenesisConfig, IdentityConfig, ImOnlineId, MembershipConfig, SessionConfig, SmithCertConfig,
SmithMembershipConfig, SudoConfig, SystemConfig, TechnicalCommitteeConfig,
UniversalDividendConfig, WASM_BINARY,
};
+use jsonrpsee::core::JsonValue;
+use sc_network_common::config::MultiaddrWithPeerId; // in the future available in sc_network::config
use sc_service::ChainType;
+use sc_telemetry::TelemetryEndpoints;
+use serde::Deserialize;
use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId;
use sp_consensus_babe::AuthorityId as BabeId;
use sp_core::{blake2_256, sr25519, Encode, H256};
@@ -32,6 +37,7 @@ use sp_finality_grandpa::AuthorityId as GrandpaId;
use sp_membership::MembershipData;
use std::collections::BTreeMap;
+pub type ChainSpec = sc_service::GenericChainSpec<GenesisConfig>;
pub type AuthorityKeys = (
AccountId,
GrandpaId,
@@ -39,14 +45,6 @@ pub type AuthorityKeys = (
ImOnlineId,
AuthorityDiscoveryId,
);
-
-pub type ChainSpec = sc_service::GenericChainSpec<GenesisConfig>;
-
-const TOKEN_DECIMALS: usize = 2;
-const TOKEN_SYMBOL: &str = "ÄžT";
-// The URL for the telemetry server.
-// const STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/";
-
/// Generate an authority keys.
pub fn get_authority_keys_from_seed(s: &str) -> AuthorityKeys {
(
@@ -57,7 +55,6 @@ pub fn get_authority_keys_from_seed(s: &str) -> AuthorityKeys {
get_from_seed::<AuthorityDiscoveryId>(s),
)
}
-
/// Generate session keys
fn get_session_keys_from_seed(s: &str) -> SessionKeys {
let authority_keys = get_authority_keys_from_seed(s);
@@ -68,7 +65,6 @@ fn get_session_keys_from_seed(s: &str) -> SessionKeys {
authority_keys.4,
)
}
-
/// make session keys struct
fn session_keys(
grandpa: GrandpaId,
@@ -84,14 +80,50 @@ fn session_keys(
}
}
+// === client specifications ===
+
+/// emulate client specifications to get them from json
+#[derive(Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+#[serde(deny_unknown_fields)]
+pub struct ClientSpec {
+ name: String,
+ id: String,
+ chain_type: ChainType,
+ boot_nodes: Vec<MultiaddrWithPeerId>,
+ telemetry_endpoints: Option<TelemetryEndpoints>,
+ // protocol_id: Option<String>,
+ // #[serde(default = "Default::default", skip_serializing_if = "Option::is_none")]
+ // fork_id: Option<String>,
+ properties: Option<serde_json::Map<std::string::String, JsonValue>>,
+ // #[serde(default)]
+ // code_substitutes: BTreeMap<String, Bytes>,
+}
+
+// === development chainspecs ===
+
/// generate development chainspec with Alice validator
-pub fn development_chain_spec() -> Result<ChainSpec, String> {
+// there is some code duplication because we can not use ClientSpec
+pub fn development_chainspecs() -> Result<ChainSpec, String> {
let wasm_binary = WASM_BINARY.ok_or_else(|| "wasm not available".to_string())?;
// custom genesis when DUNITER_GTEST_GENESIS is set
if let Ok(genesis_json_path) = std::env::var("DUNITER_GTEST_GENESIS") {
// log
log::info!("loading genesis from {genesis_json_path}");
+ // open json genesis file
+ let file = std::fs::File::open(&genesis_json_path)
+ .map_err(|e| format!("Error opening gen conf file `{}`: {}", genesis_json_path, e))?;
+ // memory map the file to avoid loading it in memory
+ let bytes = unsafe {
+ memmap2::Mmap::map(&file).map_err(|e| {
+ format!("Error mmaping gen conf file `{}`: {}", genesis_json_path, e)
+ })?
+ };
+ // parse the json file
+ let genesis_data: GenesisJson = serde_json::from_slice(&bytes)
+ .map_err(|e| format!("Error parsing gen conf file: {}", e))?;
+
// return chainspecs
Ok(ChainSpec::from_genesis(
// Name
@@ -102,9 +134,9 @@ pub fn development_chain_spec() -> Result<ChainSpec, String> {
sc_service::ChainType::Development,
// genesis config constructor
move || {
- super::gtest_genesis::build_genesis(
- // path of json genesis
- &genesis_json_path,
+ build_genesis(
+ // genesis data built from json
+ genesis_data.clone(),
// wasm binary
wasm_binary,
// replace authority by Alice
@@ -123,8 +155,8 @@ pub fn development_chain_spec() -> Result<ChainSpec, String> {
// Properties
Some(
serde_json::json!({
- "tokenDecimals": TOKEN_DECIMALS,
- "tokenSymbol": TOKEN_SYMBOL,
+ "tokenDecimals": 2,
+ "tokenSymbol": "ÄžT",
})
.as_object()
.expect("must be a map")
@@ -146,7 +178,7 @@ pub fn development_chain_spec() -> Result<ChainSpec, String> {
ChainType::Development,
// constructor
move || {
- gen_genesis_for_local_chain(
+ generate_genesis(
wasm_binary,
// Initial authorities len
1,
@@ -156,7 +188,6 @@ pub fn development_chain_spec() -> Result<ChainSpec, String> {
4,
// Sudo account
get_account_id_from_seed::<sr25519::Public>("Alice"),
- true,
)
},
// Bootnodes
@@ -170,8 +201,8 @@ pub fn development_chain_spec() -> Result<ChainSpec, String> {
// Properties
Some(
serde_json::json!({
- "tokenDecimals": TOKEN_DECIMALS,
- "tokenSymbol": TOKEN_SYMBOL,
+ "tokenDecimals": 2,
+ "tokenSymbol": "ÄžT",
})
.as_object()
.expect("must be a map")
@@ -183,63 +214,58 @@ pub fn development_chain_spec() -> Result<ChainSpec, String> {
}
}
-pub fn local_testnet_config(
- initial_authorities_len: usize,
- initial_smiths_len: usize,
- initial_identities_len: usize,
+// === live chainspecs ===
+
+/// live chainspecs
+// one smith must have session keys
+pub fn live_chainspecs(
+ client_spec: ClientSpec,
+ genesis_data: GenesisJson,
) -> Result<ChainSpec, String> {
let wasm_binary = WASM_BINARY.ok_or_else(|| "wasm not available".to_string())?;
+ // return chainspecs
Ok(ChainSpec::from_genesis(
// Name
- "ÄžTest Local Testnet",
+ client_spec.name.as_str(),
// ID
- "gtest_local",
- ChainType::Local,
+ client_spec.id.as_str(),
+ // chain type
+ client_spec.chain_type,
+ // genesis config constructor
move || {
- gen_genesis_for_local_chain(
+ build_genesis(
+ // genesis data
+ genesis_data.clone(),
+ // wasm binary
wasm_binary,
- // Initial authorities len
- initial_authorities_len,
- // Initial smiths len,
- initial_smiths_len,
- // Initial identities len
- initial_identities_len,
- // Sudo account
- get_account_id_from_seed::<sr25519::Public>("Alice"),
- true,
+ // do not replace session keys
+ None,
)
+ .expect("genesis building failed")
},
// Bootnodes
- vec![],
- // Telemetry
- None,
+ client_spec.boot_nodes,
+ // Telemetry (by default, enable telemetry, can be disabled with argument)
+ client_spec.telemetry_endpoints,
// Protocol ID
None,
- //Fork ID
+ // Fork ID
None,
// Properties
- Some(
- serde_json::json!({
- "tokenDecimals": TOKEN_DECIMALS,
- "tokenSymbol": TOKEN_SYMBOL,
- })
- .as_object()
- .expect("must be a map")
- .clone(),
- ),
+ client_spec.properties,
// Extensions
None,
))
}
-fn gen_genesis_for_local_chain(
+/// generate a genesis with given number of smith and identities
+fn generate_genesis(
wasm_binary: &[u8],
initial_authorities_len: usize,
initial_smiths_len: usize,
initial_identities_len: usize,
root_key: AccountId,
- _enable_println: bool,
) -> GenesisConfig {
assert!(initial_identities_len <= 6);
assert!(initial_smiths_len <= initial_identities_len);
diff --git a/node/src/chain_spec/gtest_genesis.rs b/node/src/chain_spec/gtest_genesis.rs
index 9aa6f737f..004a1cc6b 100644
--- a/node/src/chain_spec/gtest_genesis.rs
+++ b/node/src/chain_spec/gtest_genesis.rs
@@ -38,7 +38,8 @@ static SMITH_MIN_CERT: u32 = parameters::SmithWotMinCertForMembership::get();
// define structure of json
#[derive(Clone, Deserialize)]
-struct GenesisJson {
+#[serde(deny_unknown_fields)]
+pub struct GenesisJson {
identities: HashMap<String, Identity>,
smiths: HashMap<String, Smith>,
first_ud: u64,
@@ -96,8 +97,8 @@ fn validate_idty_name(idty_name: &str) -> bool {
/// ============================================================================================ ///
/// build genesis from json file
pub fn build_genesis(
- // path of genesis config
- genesis_config_path: &str,
+ // genesis data build from json
+ genesis_data: GenesisJson,
// wasm binary
wasm_binary: &[u8],
// useful to enforce Alice authority when developing
@@ -120,28 +121,6 @@ pub fn build_genesis(
};
log::info!("genesis timestamp: {}", genesis_timestamp);
- // open json genesis file
- let file = std::fs::File::open(&genesis_config_path).map_err(|e| {
- format!(
- "Error opening gen conf file `{}`: {}",
- genesis_config_path, e
- )
- })?;
-
- // memory map the file to avoid loading it in memory
- let bytes = unsafe {
- memmap2::Mmap::map(&file).map_err(|e| {
- format!(
- "Error mmaping gen conf file `{}`: {}",
- genesis_config_path, e
- )
- })?
- };
-
- // parse the json file
- let genesis_data: GenesisJson = serde_json::from_slice(&bytes)
- .map_err(|e| format!("Error parsing gen conf file: {}", e))?;
-
// declare variables for building genesis
// -------------------------------------
// track if fatal error occured, but let processing continue
diff --git a/node/src/command.rs b/node/src/command.rs
index a7a594f94..d151a7fb6 100644
--- a/node/src/command.rs
+++ b/node/src/command.rs
@@ -18,6 +18,8 @@
pub mod key;
pub mod utils;
+#[cfg(feature = "gtest")]
+use crate::chain_spec::{gtest, gtest_genesis};
use crate::cli::{Cli, Subcommand};
#[cfg(feature = "g1")]
use crate::service::G1Executor;
@@ -103,37 +105,73 @@ impl SubstrateCli for Cli {
fn load_spec(&self, id: &str) -> Result<Box<dyn sc_service::ChainSpec>, String> {
Ok(match id {
+ // === GDEV ===
+ // developement chainspec with Alice validator
+ // optionally from DUNITER_GENESIS_CONFIG file otherwise generated
#[cfg(feature = "gdev")]
"dev" => Box::new(chain_spec::gdev::development_chain_spec()?),
+ // local testnet with generated genesis and Alice validator
#[cfg(feature = "gdev")]
- "local" | "gdev_local" => Box::new(chain_spec::gdev::local_testnet_config(1, 3, 4)?),
- #[cfg(feature = "gdev")]
- "local2" => Box::new(chain_spec::gdev::local_testnet_config(2, 3, 4)?),
- #[cfg(feature = "gdev")]
- "local3" => Box::new(chain_spec::gdev::local_testnet_config(3, 3, 4)?),
+ "gdev_local" => Box::new(chain_spec::gdev::local_testnet_config(1, 3, 4)?),
+ // chainspecs for live network (generated or DUNITER_GENESIS_CONFIG)
+ // but must have a smith with declared session keys
#[cfg(feature = "gdev")]
- "local4" => Box::new(chain_spec::gdev::local_testnet_config(4, 4, 5)?),
- #[cfg(feature = "gdev")]
- "gdev-gl" | "gdev_gl" => Box::new(chain_spec::gdev::gen_live_conf()?),
+ "gdev_live" => Box::new(chain_spec::gdev::gen_live_conf()?),
+ // hardcoded previously generated raw chainspecs
+ // yields a pointlessly heavy binary because of hexadecimal-text-encoded values
#[cfg(feature = "gdev")]
"gdev" => Box::new(chain_spec::gdev::ChainSpec::from_json_bytes(
&include_bytes!("../specs/gdev-raw.json")[..],
)?),
+ // config used for benckmarks
#[cfg(feature = "gdev")]
"gdev-benchmark" => Box::new(chain_spec::gdev::benchmark_chain_spec()?),
+ // === GTEST ===
+ // dev chainspecs with Alice validator
+ // provide DUNITER_GTEST_GENESIS env var if you want to build genesis from json
+ // otherwise you get a local testnet with generated genesis
#[cfg(feature = "gtest")]
- "gtest_dev" => Box::new(chain_spec::gtest::development_chain_spec()?),
- #[cfg(feature = "gtest")]
- "gtest_local" => Box::new(chain_spec::gtest::local_testnet_config(2, 3, 4)?),
+ "gtest_dev" => Box::new(chain_spec::gtest::development_chainspecs()?),
+ // chainspecs for live network
+ // must have following files in node/specs folder:
+ // - gtest.json
+ // - gtest_client-specs.json
#[cfg(feature = "gtest")]
- "gtest_local3" => Box::new(chain_spec::gtest::local_testnet_config(3, 3, 4)?),
- #[cfg(feature = "gtest")]
- "gtest_local4" => Box::new(chain_spec::gtest::local_testnet_config(4, 4, 5)?),
+ "gtest_live" => {
+ const JSON_CLIENT_SPEC: &str = "./node/specs/gtest_client-specs.json";
+ const JSON_GENESIS: &str = "./node/specs/gtest_genesis.json";
+ let client_spec: gtest::ClientSpec = serde_json::from_slice(
+ &std::fs::read(JSON_CLIENT_SPEC)
+ .map_err(|e| format!("failed to read {JSON_CLIENT_SPEC} {e}"))?[..],
+ )
+ .map_err(|e| format!("failed to parse {e}"))?;
+ let genesis_data: gtest_genesis::GenesisJson = serde_json::from_slice(
+ &std::fs::read(JSON_GENESIS)
+ .map_err(|e| format!("failed to read {JSON_GENESIS} {e}"))?[..],
+ )
+ .map_err(|e| format!("failed to parse {e}"))?;
+ // rebuild chainspecs from these files
+ Box::new(chain_spec::gtest::live_chainspecs(
+ client_spec,
+ genesis_data,
+ )?)
+ }
+ // return hardcoded live chainspecs
+ // embed client spec and genesis to avoid embeding hexadecimal runtime
+ // and having hexadecimal runtime in git history
#[cfg(feature = "gtest")]
"gtest" => {
- unimplemented!()
- //Box::new(chain_spec::gtest::ChainSpec::from_json_file(file_path)?)
+ let client_spec: gtest::ClientSpec =
+ serde_json::from_slice(include_bytes!("../specs/gtest_client-specs.json"))
+ .unwrap();
+ let genesis_data: gtest_genesis::GenesisJson =
+ serde_json::from_slice(include_bytes!("../specs/gtest_genesis.json")).unwrap();
+ Box::new(chain_spec::gtest::live_chainspecs(
+ client_spec,
+ genesis_data,
+ )?)
}
+ // === G1 ===
#[cfg(feature = "g1")]
"g1" => {
unimplemented!()
@@ -153,8 +191,6 @@ impl SubstrateCli for Cli {
let runtime_type = if starts_with("g1") {
RuntimeType::G1
- } else if starts_with("gdem") {
- RuntimeType::GTest
} else if starts_with("dev") || starts_with("gdev") {
RuntimeType::GDev
} else if starts_with("gt") {
@@ -188,7 +224,7 @@ impl SubstrateCli for Cli {
RuntimeType::GTest => >est_runtime::VERSION,
#[cfg(feature = "gdev")]
RuntimeType::GDev => &gdev_runtime::VERSION,
- _ => panic!("unknown runtime"),
+ _ => panic!("unknown runtime {:?}", spec.runtime_type()),
}
}
}
diff --git a/node/src/service.rs b/node/src/service.rs
index 30edcb7ae..6562b6584 100644
--- a/node/src/service.rs
+++ b/node/src/service.rs
@@ -98,6 +98,7 @@ impl sc_executor::NativeExecutionDispatch for G1Executor {
}
}
+#[derive(Debug)]
pub enum RuntimeType {
G1,
GDev,
@@ -115,11 +116,9 @@ impl IdentifyRuntimeType for Box<dyn sc_chain_spec::ChainSpec> {
fn runtime_type(&self) -> RuntimeType {
if self.id().starts_with("g1") {
RuntimeType::G1
- } else if self.id().starts_with("gdem") {
- RuntimeType::GTest
} else if self.id().starts_with("dev") || self.id().starts_with("gdev") {
RuntimeType::GDev
- } else if self.id().starts_with("gt") {
+ } else if self.id().starts_with("gtest") {
RuntimeType::GTest
} else {
panic!("unknown runtime")
--
GitLab