diff --git a/.env b/.env
index c59bbaf0917d6938a0ad4ddefaa9c7736cc44827..2f89c7773d17ab18d89978b440348d363fe8b2e0 100644
--- a/.env
+++ b/.env
@@ -1 +1,2 @@
-prod=false
\ No newline at end of file
+prod=false
+LEVELDB_PATH = "./leveldb"
diff --git a/adapters/__init__.py b/adapters/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/adapters/duniter_v18/__init__.py b/adapters/duniter_v18/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/adapters/duniter_v18/blocks.py b/adapters/duniter_v18/blocks.py
new file mode 100644
index 0000000000000000000000000000000000000000..07031a1d2a524e43dd22228fa1a6a142f72cc8ac
--- /dev/null
+++ b/adapters/duniter_v18/blocks.py
@@ -0,0 +1,45 @@
+import json
+import plyvel
+from pathlib import Path
+
+
+class LevelDBBlocksRepository:
+
+ DEFAULT_LEVELDB_PATH = "./leveldb"
+ DB_INDEX = "level_blockchain"
+
+ def __init__(self, leveldb_path: str):
+ """
+ Init connection
+
+ :param leveldb_path: Path of database
+ """
+ self.index = plyvel.DB(str(Path(leveldb_path).joinpath(self.DB_INDEX)))
+
+ def __iter__(self):
+ """
+ Iterate over number: int, block: dict
+
+ :return:
+ """
+ for key, value in self.index.__iter__():
+ yield int(key), get_block_from_db_entry(value)
+
+ def get(self, number: int) -> dict:
+ """
+ Return block dict from number
+
+ :param number: Block number
+ :return:
+ """
+ return get_block_from_db_entry(self.index.get(number))
+
+
+def get_block_from_db_entry(json_string: str) -> dict:
+ """
+ Get block dict from json string
+
+ :param json_string: Json entry
+ :return:
+ """
+ return json.loads(json_string)
diff --git a/adapters/duniter_v18/certifications.py b/adapters/duniter_v18/certifications.py
new file mode 100644
index 0000000000000000000000000000000000000000..62230e88241bcc91cd72eb20edfd81cba48308ba
--- /dev/null
+++ b/adapters/duniter_v18/certifications.py
@@ -0,0 +1,45 @@
+import json
+import plyvel
+from pathlib import Path
+
+
+class LevelDBCertificationsRepository:
+
+ DEFAULT_LEVELDB_PATH = "./leveldb"
+ DB_INDEX = "level_cindex"
+
+ def __init__(self, leveldb_path: str):
+ """
+ Init connection
+
+ :param leveldb_path: Path of database
+ """
+ self.index = plyvel.DB(str(Path(leveldb_path).joinpath(self.DB_INDEX)))
+
+ def __iter__(self):
+ """
+ Iterate over pubkey: str, certifications: dict
+
+ :return:
+ """
+ for key, value in self.index.__iter__():
+ yield key.decode("utf-8"), get_certifications_from_db_entry(value)
+
+ def get(self, pubkey: str) -> dict:
+ """
+ Return certifications dict from pubkey
+
+ :param pubkey: Identity account pubkey
+ :return:q
+ """
+ return get_certifications_from_db_entry(self.index.get(pubkey.encode("utf-8")))
+
+
+def get_certifications_from_db_entry(json_string: str) -> dict:
+ """
+ Get certifications dict from json string
+
+ :param json_string: Json entry
+ :return:
+ """
+ return json.loads(json_string)
diff --git a/adapters/duniter_v18/identities.py b/adapters/duniter_v18/identities.py
new file mode 100644
index 0000000000000000000000000000000000000000..066fb393acee08f025d4536e34dc6a23532ca489
--- /dev/null
+++ b/adapters/duniter_v18/identities.py
@@ -0,0 +1,48 @@
+import json
+import plyvel
+from pathlib import Path
+
+
+class LevelDBIdentitiesRepository:
+
+ DEFAULT_LEVELDB_PATH = "./leveldb"
+ DB_INDEX = "level_iindex"
+
+ def __init__(self, leveldb_path: str):
+ """
+ Init connection
+
+ :param leveldb_path: Path of database
+ """
+ self.index = plyvel.DB(str(Path(leveldb_path).joinpath(self.DB_INDEX)))
+
+ def __iter__(self):
+ """
+ Iterate over pubkey: str, identity: dict
+
+ :return:
+ """
+ for key, value in self.index.__iter__():
+ yield key.decode("utf-8"), get_identity_from_db_entry(value)
+
+ def get(self, pubkey: str):
+ """
+ Return identity dict from pubkey
+
+ :param pubkey: Identity account pubkey
+ :return:
+ """
+ return get_identity_from_db_entry(self.index.get(pubkey.encode("utf-8")))
+
+
+def get_identity_from_db_entry(json_string: str) -> dict:
+ """
+ Get identity dict list from json string
+ Parse list to get last updated status of identity
+
+ :param json_string: Json entry
+ :return:
+ """
+ identity_list = json.loads(json_string) # type: list
+ # update first identity properties except for None value
+ return {key: value for list_item in identity_list for (key, value) in list_item.items() if value is not None or list_item == identity_list[0]}
diff --git a/adapters/duniter_v18/memberships.py b/adapters/duniter_v18/memberships.py
new file mode 100644
index 0000000000000000000000000000000000000000..0c5fc0a4a2c0cf3b07ef7d5f7baf9cbb3821b8a2
--- /dev/null
+++ b/adapters/duniter_v18/memberships.py
@@ -0,0 +1,48 @@
+import json
+import plyvel
+from pathlib import Path
+
+
+class LevelDBMembershipsRepository:
+
+ DEFAULT_LEVELDB_PATH = "./leveldb"
+ DB_INDEX = "level_mindex"
+
+ def __init__(self, leveldb_path: str):
+ """
+ Init connection
+
+ :param leveldb_path: Path of database
+ """
+ self.index = plyvel.DB(str(Path(leveldb_path).joinpath(self.DB_INDEX)))
+
+ def __iter__(self):
+ """
+ Iterate over pubkey: str, membership: dict
+
+ :return:
+ """
+ for key, value in self.index.__iter__():
+ yield key.decode("utf-8"), get_membership_from_db_entry(value)
+
+ def get(self, pubkey: str) -> dict:
+ """
+ Return membership dict from pubkey
+
+ :param pubkey: Identity account pubkey
+ :return:
+ """
+ return get_membership_from_db_entry(self.index.get(pubkey.encode("utf-8")))
+
+
+def get_membership_from_db_entry(json_string: str) -> dict:
+ """
+ Get membership dict list from json string
+ Parse list to get last updated status of membership
+
+ :param json_string: Json entry
+ :return:
+ """
+ membership_list = json.loads(json_string) # type: list
+ # update first membership properties except for None value
+ return {key: value for list_item in membership_list for (key, value) in list_item.items() if value is not None or list_item == membership_list[0]}
diff --git a/adapters/duniter_v18/wallets.py b/adapters/duniter_v18/wallets.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb4b04e64e060e58ee44b046cc2e315d86b2f2ec
--- /dev/null
+++ b/adapters/duniter_v18/wallets.py
@@ -0,0 +1,45 @@
+import json
+import plyvel
+from pathlib import Path
+
+
+class LevelDBWalletsRepository:
+
+ DEFAULT_LEVELDB_PATH = "./leveldb"
+ DB_INDEX = "level_wallet"
+
+ def __init__(self, leveldb_path: str):
+ """
+ Init connection
+
+ :param leveldb_path: Path of database
+ """
+ self.index = plyvel.DB(str(Path(leveldb_path).joinpath(self.DB_INDEX)))
+
+ def __iter__(self):
+ """
+ Iterate over key: str, wallet: dict
+
+ :return:
+ """
+ for key, value in self.index.__iter__():
+ yield key.decode("utf-8"), get_wallet_from_db_entry(value)
+
+ def get(self, unlock_expression: str) -> dict:
+ """
+ Return wallet dict from unlock_expression
+
+ :param unlock_expression: Account money unlock expression
+ :return:
+ """
+ return get_wallet_from_db_entry(self.index.get(unlock_expression.encode("utf-8")))
+
+
+def get_wallet_from_db_entry(json_string: str) -> dict:
+ """
+ Get wallet dict from json string
+
+ :param json_string: Json entry
+ :return:
+ """
+ return json.loads(json_string)
diff --git a/lib/functions.py b/lib/functions.py
index 08fa0d99de650211a6373527822eb0b30702c39a..7a28e4df37cae4289965e95c170bd2f62fef85d8 100644
--- a/lib/functions.py
+++ b/lib/functions.py
@@ -1,5 +1,13 @@
+import os
+
+from adapters.duniter_v18.blocks import LevelDBBlocksRepository
+from adapters.duniter_v18.certifications import LevelDBCertificationsRepository
+from adapters.duniter_v18.identities import LevelDBIdentitiesRepository
+from adapters.duniter_v18.memberships import LevelDBMembershipsRepository
+from adapters.duniter_v18.wallets import LevelDBWalletsRepository
from lib.utility import *
-from time import time
+
+DEFAULT_LEVELDB_PATH = "./leveldb"
def load_json_url(path):
@@ -9,16 +17,16 @@ def load_json_url(path):
def get_wallets_data():
# Get wallets balances data
- wallets_data = load_json_url("inputs/wallets.json")
+ wallets_repository = LevelDBWalletsRepository(os.getenv("LEVELDB_PATH", DEFAULT_LEVELDB_PATH))
wallets = {}
total_money = 0 # counter
ignored_money = 0 # counter
- for wallet in wallets_data:
- balance = wallet["value"]["balance"]
- if "&&" in wallet["key"]:
+ for unlock_expression, wallet in wallets_repository:
+ balance = wallet["balance"]
+ if "&&" in unlock_expression:
ignored_money += balance
continue
- pubkey = wallet["key"].split("(")[1].split(")")[0]
+ pubkey = unlock_expression.split("(")[1].split(")")[0]
# Remove pubkeys > 32 bytes
# d2meevcahfts2gqmvmrw5hzi25jddikk4nc4u1fkwrau
@@ -28,24 +36,16 @@ def get_wallets_data():
# jUPLL2BgY2QpheWEY3R13edV2Y4tvQMCXjJVM8PGDvyd
# gatrpfmgsuez193bja5snivz3dsvsqn5kcm4ydtpknsy
pubkey_bytes = base58.b58decode(pubkey)
- pubkey_lenght = len(pubkey_bytes)
- if pubkey_lenght > 32 or balance == 0:
+ pubkey_length = len(pubkey_bytes)
+ if pubkey_length > 32 or balance == 0:
ignored_money += balance
continue
wallets.update({v1_pubkey_to_v2_address(pubkey): int(balance)})
total_money += balance
-
- return (wallets, total_money, ignored_money)
-
-def get_membership_expiry():
- """get membership expiry from input file"""
- # Get Dex membership data
- membership_data = load_json_url("inputs/membership.json")
- membership_expiry = {}
- for membership in membership_data:
- membership_expiry[membership["key"]] = membership["value"][0]["expires_on"]
- return membership_expiry
+
+ return wallets, total_money, ignored_money
+
def get_identities_and_wallets(start_timestamp):
"""get identities with certifications and wallets with their balance
@@ -63,21 +63,21 @@ def get_identities_and_wallets(start_timestamp):
initial_monetary_mass = last_block["mass"]
last_block_time = last_block["medianTime"]
- # Get wallets data
- (wallets, total_money, ignored_money) = get_wallets_data()
- # Get membership expiry
- membership_expiry = get_membership_expiry()
- # Get Dex idty data
- idty_data = load_json_url("inputs/idty.json")
- # Get Dex certs data
- certs_data = load_json_url("inputs/certs.json")
- # Get blocs number with dates
- blocs_data = load_json_url("inputs/blocs.json")
+ print(" parse Identities...")
+ # Get leveldb indices as Dex data
+ memberships_repository = LevelDBMembershipsRepository(os.getenv("LEVELDB_PATH", DEFAULT_LEVELDB_PATH))
+ identities_repository = LevelDBIdentitiesRepository(os.getenv("LEVELDB_PATH", DEFAULT_LEVELDB_PATH))
+ certifications_repository = LevelDBCertificationsRepository(os.getenv("LEVELDB_PATH", DEFAULT_LEVELDB_PATH))
+ blocks_repository = LevelDBBlocksRepository(os.getenv("LEVELDB_PATH", DEFAULT_LEVELDB_PATH))
+
# Get identities switches
addresses_switches = load_json("custom/addresses_switches.json")
# Get custom identities
custom_identities = load_json("custom/identities.json")
+ # Get wallets data
+ print(" parse Wallets...")
+ (wallets, total_money, ignored_money) = get_wallets_data()
# add ignored money to treasury and check initial monetary mass
treasury += ignored_money # add ignored money to treasury
wallet_sum = total_money + ignored_money
@@ -94,16 +94,15 @@ def get_identities_and_wallets(start_timestamp):
# TODO make sure that index respects order of arrival
# Get identities names by pubkey
- for idty in idty_data:
- pubkey = idty["key"]
+ for pubkey, identity in identities_repository:
address = v1_pubkey_to_v2_address(pubkey)
- value = idty["value"][0]
- index = value["wotb_id"] + 1
- uid = value["uid"]
- is_member = value["member"]
+ index = identity["wotb_id"] + 1
+ uid = identity["uid"]
+ is_member = identity["member"]
identity_names[pubkey] = uid
- membership_expire_on = date_to_bloc_number(membership_expiry[pubkey], start_timestamp)
- if membership_expire_on < 0 : membership_expire_on = 0 # forget old expiry date
+ membership_expire_on = date_to_bloc_number(memberships_repository.get(pubkey)["expires_on"], start_timestamp)
+ if membership_expire_on < 0:
+ membership_expire_on = 0 # forget old expiry date
# add address and balance to identity
if address not in wallets:
@@ -135,21 +134,20 @@ def get_identities_and_wallets(start_timestamp):
identities.update(custom_identities)
# get info from block
- for bloc in blocs_data:
- blocs[int(bloc["key"])] = bloc["value"]["medianTime"]
+ for block_number, block in blocks_repository:
+ blocs[block_number] = block["medianTime"]
# Generate identities Ğ1v2 genesis json bloc
# certs are stored per issuer in input file
# certs are stored per receiver in output file
print(" parse certification...")
- for issuer in certs_data:
- i_pubkey = issuer["key"]
+ for i_pubkey, issuer in certifications_repository:
i_uid = identity_names[i_pubkey]
i_address = v1_pubkey_to_v2_address(i_pubkey)
- for cert in issuer["value"]["issued"]:
+ for cert in issuer["issued"]:
# if certification expired, skip silently
- if cert["expired_on"] != 0 :
+ if cert["expired_on"] != 0:
continue
r_pubkey = cert["receiver"]
@@ -168,11 +166,11 @@ def get_identities_and_wallets(start_timestamp):
cert_expire_on = date_to_bloc_number(cert_expire_at, start_timestamp)
# if certification expiration date is before export,
- # it is a renewed certification and can be ignored
+ # it is a renewed certification and can be ignored
if cert_expire_at < last_block_time:
continue
# certifications can also have expired between export and start_timestamp
- # in this case we display a warning because these missing certification
+ # in this case we display a warning because these missing certification
# could lead to a genesis with not enough certifications received for some members
if cert_expire_at <= start_timestamp:
print(f"⚠️ {i_uid} → {r_uid} cert expired between export and start")
@@ -184,7 +182,7 @@ def get_identities_and_wallets(start_timestamp):
# add received certification to identity
identities[r_uid]["certs_received"][i_uid] = cert_expire_on
- return (identities, wallets)
+ return identities, wallets
def get_smiths():
@@ -207,4 +205,4 @@ def fix_identities_with_not_enough_certs(identities):
received_count = len(value["certs_received"])
if value["membership_expire_on"] > 0 and received_count < CERT_MIN_RECEIVED_CERT_TO_ISSUE_CERT:
print(f"⚠️ {key} has received only {received_count} certs, disabling it")
- value["membership_expire_on"] = 0
\ No newline at end of file
+ value["membership_expire_on"] = 0
diff --git a/main.py b/main.py
index 0e6fa5a3558212035ce4c4932d510927da269aa4..03021a80a9a21cd48a6d89c2aeafe758bb81e338 100755
--- a/main.py
+++ b/main.py
@@ -16,8 +16,9 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import sys
+from time import time
+
from lib.functions import *
-from lib.get_parameters import *
def load_json_url(path):
@@ -31,11 +32,12 @@ print("Generate ĞTest genesis with up-to-date Ğ1 data")
opt1 = ""
if len(sys.argv) > 1:
opt1 = int(sys.argv[1])
-
+
# define start timestamp
start_timestamp = opt1
# if not defined set start time to now
-if start_timestamp == "": start_timestamp = int(time())
+if start_timestamp == "":
+ start_timestamp = int(time())
# Get ĞTest parameters
print(" get ĞTest parameters...")
@@ -47,12 +49,11 @@ sudo_key = "5Dq8xjvkmbz7q4g2LbZgyExD26VSCutfEc6n4W4AfQeVHZqz"
print(" dump ĞTest parameters...")
last_block = load_json_url("inputs/ud_value.json")[0]["value"]
FIRST_UD_VALUE = last_block["dividend"]
-FIRST_UD_REEVAL = date_to_bloc_number(last_block["udReevalTime"] , start_timestamp)
+FIRST_UD_REEVAL = date_to_bloc_number(last_block["udReevalTime"], start_timestamp)
INITAL_MONETARY_MASS = last_block["mass"]
LAST_BLOCK_TIME = last_block["medianTime"]
-# Add identities bloc
-print(" parse identities...")
+# Add identities and wallets
(identities, other_wallets) = get_identities_and_wallets(start_timestamp)
# FIXME avoid this
diff --git a/requirements.txt b/requirements.txt
index 6d6d7ee5e8dd32d43bda6f083b0d6c1b7e13ee2b..ca3dce0c59a8c411f7fe613c31edf60db6aa64c3 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
substrate-interface
-base58
\ No newline at end of file
+base58
+plyvel