tests(live): add sanity tests against gdev live chain

.cargo/config

 [alias]
 cucumber = "test -p duniter-end2end-tests --test cucumber_tests --"
+sanity-gdev = "test -p duniter-live-tests --test sanity_gdev --"
 tu = "test --workspace --exclude duniter-end2end-tests"
 xtask = "run --package xtask --"
 

.gitlab-ci.yml

@@ -124,6 +124,14 @@ build_release_manual:
       - build/
     expire_in: 3 day
 
+sanity_tests:
+  extends: .env
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - when: never
+  script:
+    - cargo sanity-gdev
+
 tests_debug:
   extends: .env
   rules:

Cargo.lock

@@ -1532,6 +1532,18 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "duniter-live-tests"
+version = "3.0.0"
+dependencies = [
+ "anyhow",
+ "hex-literal",
+ "parity-scale-codec",
+ "sp-core",
+ "subxt",
+ "tokio",
+]
+
 [[package]]
 name = "duniter-primitives"
 version = "3.0.0"

Cargo.toml

@@ -123,6 +123,7 @@ resolver = "2"
 
 members = [
     'end2end-tests',
+    'live-tests',
     'pallets/certification',
     'pallets/duniter-test-parameters',
     'pallets/duniter-test-parameters/macro',

live-tests/Cargo.toml

+[package]
+authors = ['Axiom-Team Developers <https://axiom-team.fr>']
+description = 'duniter live tests.'
+edition = '2018'
+homepage = 'https://substrate.dev'
+license = 'AGPL-3.0'
+name = 'duniter-live-tests'
+repository = 'https://git.duniter.org/nodes/rust/duniter-v2s'
+version = '3.0.0'
+
+[dev-dependencies]
+anyhow = "1.0"
+hex-literal = "0.3"
+parity-scale-codec = "2.3.1"
+sp-core = { git = 'https://github.com/librelois/substrate', branch = 'duniter-monthly-2022-02' }
+subxt = { git = 'https://github.com/librelois/subxt', branch = 'duniter-monthly-2022-02' }
+tokio = { version = "1.15.0", features = ["macros"] }

live-tests/README.md

+# Duniter live tests
+
+Kind of tests that run against a live chain!
+
+## Sanity tests
+
+Test suite that verifies the consistency of the onchain storage.
+
+### Run sanity tests
+
+1. Checkout the git tag of the runtime that you want to check
+2. run the tests again the default network of the specified runtime type: `cargo sanity-RUNTIME_TYPE`
+
+`RUNTIME_TYPE` should be replaced by `gdev`, `gtest` or `g1`.
+
+#### Custom RPC endpoint
+
+You can choose to use another RPC endpoint by setting the environment variable `WS_RPC_ENDPOINT`.
+This is also the only way to test against a different network that the default one.
+
+#### run against a specific block
+
+You can choose to use run the sanity tests against a specific block by setting the environment
+variable `AT_BLOCK_NUMBER`.
+
+**Be careful: this would require to use an archive node.**
+
+### Contribute to sanity tests
+
+The code is in the file `live-tests/tests/sanity_RUNTIME_TYPE.rs`
+
+There is 3 different parts:
+
+1. Runtime types definitions
+2. Collect storage data
+3. Verify consistency of collected data

live-tests/src/lib.rs

+// Copyright 2021 Axiom-Team
+//
+// This file is part of Substrate-Libre-Currency.
+//
+// Substrate-Libre-Currency is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, version 3 of the License.
+//
+// Substrate-Libre-Currency is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with Substrate-Libre-Currency. If not, see <https://www.gnu.org/licenses/>.

live-tests/tests/sanity_gdev.rs

+// Copyright 2021-2022 Axiom-Team
+//
+// This file is part of Duniter-v2S.
+//
+// Duniter-v2S is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, version 3 of the License.
+//
+// Duniter-v2S is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with Duniter-v2S. If not, see <https://www.gnu.org/licenses/>.
+
+#[subxt::subxt(runtime_metadata_path = "../resources/metadata.scale")]
+pub mod gdev_runtime {}
+
+use hex_literal::hex;
+use sp_core::crypto::AccountId32;
+use sp_core::{ByteArray, H256};
+use std::collections::HashMap;
+use subxt::{ClientBuilder, DefaultConfig, DefaultExtra};
+
+const DEFAULT_ENDPOINT: &str = "wss://gdev.librelois.fr:443/ws";
+
+const TREASURY_ACCOUNT_ID: [u8; 32] =
+    hex!("6d6f646c70792f74727372790000000000000000000000000000000000000000");
+
+type Api = gdev_runtime::RuntimeApi<DefaultConfig, DefaultExtra<DefaultConfig>>;
+type Client = subxt::Client<DefaultConfig>;
+
+// define gdev basic types
+type Balance = u64;
+type BlockNumber = u32;
+type Index = u32;
+
+// Define gdev types
+type AccountInfo = gdev_runtime::runtime_types::frame_system::AccountInfo<
+    Index,
+    gdev_runtime::runtime_types::pallet_duniter_account::types::AccountData<Balance>,
+>;
+type IdtyIndex = u32;
+type IdtyValue =
+    gdev_runtime::runtime_types::pallet_identity::types::IdtyValue<BlockNumber, AccountId32>;
+use gdev_runtime::runtime_types::pallet_identity::types::IdtyStatus;
+
+#[tokio::test(flavor = "current_thread")]
+async fn main() -> anyhow::Result<()> {
+    let ws_rpc_endpoint =
+        std::env::var("WS_RPC_ENDPOINT").unwrap_or_else(|_| DEFAULT_ENDPOINT.to_owned());
+    let client: Client = ClientBuilder::new()
+        .set_url(ws_rpc_endpoint)
+        .set_page_size(100)
+        .build()
+        .await
+        .expect("fail to connect to node");
+
+    let maybe_block_hash = if let Ok(block_number) = std::env::var("AT_BLOCK_NUMBER") {
+        let block_number: BlockNumber = block_number.parse()?;
+        println!("Run sanity tests against ĞDev at block #{}", block_number);
+        client.rpc().block_hash(Some(block_number.into())).await?
+    } else {
+        println!("Run sanity tests against ĞDev at last best block");
+        None
+    };
+
+    sanity_tests_at(client, maybe_block_hash).await
+}
+
+async fn sanity_tests_at(client: Client, maybe_block_hash: Option<H256>) -> anyhow::Result<()> {
+    // Get API
+    let api = client.clone().to_runtime_api::<Api>();
+
+    // ===== Collect storage ===== //
+
+    // Collect accounts
+    let mut accounts = HashMap::new();
+    let mut account_iter = api
+        .storage()
+        .system()
+        .account_iter(maybe_block_hash)
+        .await?;
+    while let Some((key, account_info)) = account_iter.next().await? {
+        let mut account_id_bytes = [0u8; 32];
+        account_id_bytes.copy_from_slice(&key.0[48..]);
+        accounts.insert(AccountId32::new(account_id_bytes), account_info);
+    }
+
+    // Collect identities
+    let mut identities = HashMap::new();
+    let mut idty_iter = api
+        .storage()
+        .identity()
+        .identities_iter(maybe_block_hash)
+        .await?;
+    while let Some((key, idty_value)) = idty_iter.next().await? {
+        let mut idty_index_bytes = [0u8; 4];
+        idty_index_bytes.copy_from_slice(&key.0[40..]);
+        identities.insert(IdtyIndex::from_le_bytes(idty_index_bytes), idty_value);
+    }
+
+    // ===== Verify storage ===== //
+
+    verify_accounts(&accounts).await?;
+    verify_identities(&accounts, &identities).await?;
+
+    Ok(())
+}
+
+async fn verify_accounts(accounts: &HashMap<AccountId32, AccountInfo>) -> anyhow::Result<()> {
+    for (account_id, account_info) in accounts {
+        if account_info.sufficients == 0 {
+            // Rule 1: If the account is not sufficient, it should have at least one provider
+            assert!(
+                account_info.providers > 0,
+                "Account {} has no providers nor sufficients",
+                account_id
+            );
+            // Rule 2: If the account is not sufficient, it should comply to the existential deposit
+            assert!(
+                (account_info.data.free + account_info.data.reserved) >= 200,
+                "Account {} not respect existential deposit rule",
+                account_id
+            );
+        }
+
+        // Rule 3: If the account have consumers, it shoul have at least one provider
+        if account_info.consumers > 0 {
+            // Rule 1: If the account is not s
+            assert!(
+                account_info.providers > 0,
+                "Account {} has no providers nor sufficients",
+                account_id
+            );
+        }
+
+        if account_id.as_slice() != TREASURY_ACCOUNT_ID {
+            // Rule 4: If the account is not a "special account",
+            // it should have a random id or a consumer
+            assert!(
+                account_info.data.random_id.is_some() || account_info.consumers > 0,
+                "Account {} has no random_id nor consumer",
+                account_id
+            );
+        }
+    }
+    Ok(())
+}
+
+async fn verify_identities(
+    accounts: &HashMap<AccountId32, AccountInfo>,
+    identities: &HashMap<IdtyIndex, IdtyValue>,
+) -> anyhow::Result<()> {
+    for (idty_index, idty_value) in identities {
+        // Rule 1: each identity should have an account
+        let idty_account = accounts
+            .get(&idty_value.owner_key)
+            .unwrap_or_else(|| panic!("Identity {} has no account", idty_index));
+
+        // Rule 2: each identity account should be sufficient
+        assert!(
+            idty_account.sufficients > 0,
+            "Identity {} is corrupted: idty_account.sufficients == 0",
+            idty_index
+        );
+
+        match idty_value.status {
+            IdtyStatus::Validated => {
+                // Rule 3: If the identity is validated, removable_on shoud be zero
+                assert!(
+                    idty_value.removable_on == 0,
+                    "Identity {} is corrupted: removable_on > 0 on validated idty",
+                    idty_index
+                );
+            }
+            _ => {
+                // Rule 4: If the identity is not validated, next_creatable_identity_on shoud be zero
+                assert!(
+                    idty_value.next_creatable_identity_on == 0,
+                    "Identity {} is corrupted: next_creatable_identity_on > 0 on non-validated idty",
+                    idty_index
+                );
+            }
+        }
+    }
+    Ok(())
+}