diff --git a/.env b/.env index c59bbaf0917d6938a0ad4ddefaa9c7736cc44827..2f89c7773d17ab18d89978b440348d363fe8b2e0 100644 --- a/.env +++ b/.env @@ -1 +1,2 @@ -prod=false \ No newline at end of file +prod=false +LEVELDB_PATH = "./leveldb" diff --git a/adapters/__init__.py b/adapters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/adapters/duniter_v18/__init__.py b/adapters/duniter_v18/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/adapters/duniter_v18/blocks.py b/adapters/duniter_v18/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..07031a1d2a524e43dd22228fa1a6a142f72cc8ac --- /dev/null +++ b/adapters/duniter_v18/blocks.py @@ -0,0 +1,45 @@ +import json +import plyvel +from pathlib import Path + + +class LevelDBBlocksRepository: + + DEFAULT_LEVELDB_PATH = "./leveldb" + DB_INDEX = "level_blockchain" + + def __init__(self, leveldb_path: str): + """ + Init connection + + :param leveldb_path: Path of database + """ + self.index = plyvel.DB(str(Path(leveldb_path).joinpath(self.DB_INDEX))) + + def __iter__(self): + """ + Iterate over number: int, block: dict + + :return: + """ + for key, value in self.index.__iter__(): + yield int(key), get_block_from_db_entry(value) + + def get(self, number: int) -> dict: + """ + Return block dict from number + + :param number: Block number + :return: + """ + return get_block_from_db_entry(self.index.get(number)) + + +def get_block_from_db_entry(json_string: str) -> dict: + """ + Get block dict from json string + + :param json_string: Json entry + :return: + """ + return json.loads(json_string) diff --git a/adapters/duniter_v18/certifications.py b/adapters/duniter_v18/certifications.py new file mode 100644 index 0000000000000000000000000000000000000000..62230e88241bcc91cd72eb20edfd81cba48308ba --- /dev/null +++ b/adapters/duniter_v18/certifications.py @@ -0,0 +1,45 @@ +import json +import plyvel +from pathlib import Path + + +class LevelDBCertificationsRepository: + + DEFAULT_LEVELDB_PATH = "./leveldb" + DB_INDEX = "level_cindex" + + def __init__(self, leveldb_path: str): + """ + Init connection + + :param leveldb_path: Path of database + """ + self.index = plyvel.DB(str(Path(leveldb_path).joinpath(self.DB_INDEX))) + + def __iter__(self): + """ + Iterate over pubkey: str, certifications: dict + + :return: + """ + for key, value in self.index.__iter__(): + yield key.decode("utf-8"), get_certifications_from_db_entry(value) + + def get(self, pubkey: str) -> dict: + """ + Return certifications dict from pubkey + + :param pubkey: Identity account pubkey + :return:q + """ + return get_certifications_from_db_entry(self.index.get(pubkey.encode("utf-8"))) + + +def get_certifications_from_db_entry(json_string: str) -> dict: + """ + Get certifications dict from json string + + :param json_string: Json entry + :return: + """ + return json.loads(json_string) diff --git a/adapters/duniter_v18/identities.py b/adapters/duniter_v18/identities.py new file mode 100644 index 0000000000000000000000000000000000000000..066fb393acee08f025d4536e34dc6a23532ca489 --- /dev/null +++ b/adapters/duniter_v18/identities.py @@ -0,0 +1,48 @@ +import json +import plyvel +from pathlib import Path + + +class LevelDBIdentitiesRepository: + + DEFAULT_LEVELDB_PATH = "./leveldb" + DB_INDEX = "level_iindex" + + def __init__(self, leveldb_path: str): + """ + Init connection + + :param leveldb_path: Path of database + """ + self.index = plyvel.DB(str(Path(leveldb_path).joinpath(self.DB_INDEX))) + + def __iter__(self): + """ + Iterate over pubkey: str, identity: dict + + :return: + """ + for key, value in self.index.__iter__(): + yield key.decode("utf-8"), get_identity_from_db_entry(value) + + def get(self, pubkey: str): + """ + Return identity dict from pubkey + + :param pubkey: Identity account pubkey + :return: + """ + return get_identity_from_db_entry(self.index.get(pubkey.encode("utf-8"))) + + +def get_identity_from_db_entry(json_string: str) -> dict: + """ + Get identity dict list from json string + Parse list to get last updated status of identity + + :param json_string: Json entry + :return: + """ + identity_list = json.loads(json_string) # type: list + # update first identity properties except for None value + return {key: value for list_item in identity_list for (key, value) in list_item.items() if value is not None or list_item == identity_list[0]} diff --git a/adapters/duniter_v18/memberships.py b/adapters/duniter_v18/memberships.py new file mode 100644 index 0000000000000000000000000000000000000000..0c5fc0a4a2c0cf3b07ef7d5f7baf9cbb3821b8a2 --- /dev/null +++ b/adapters/duniter_v18/memberships.py @@ -0,0 +1,48 @@ +import json +import plyvel +from pathlib import Path + + +class LevelDBMembershipsRepository: + + DEFAULT_LEVELDB_PATH = "./leveldb" + DB_INDEX = "level_mindex" + + def __init__(self, leveldb_path: str): + """ + Init connection + + :param leveldb_path: Path of database + """ + self.index = plyvel.DB(str(Path(leveldb_path).joinpath(self.DB_INDEX))) + + def __iter__(self): + """ + Iterate over pubkey: str, membership: dict + + :return: + """ + for key, value in self.index.__iter__(): + yield key.decode("utf-8"), get_membership_from_db_entry(value) + + def get(self, pubkey: str) -> dict: + """ + Return membership dict from pubkey + + :param pubkey: Identity account pubkey + :return: + """ + return get_membership_from_db_entry(self.index.get(pubkey.encode("utf-8"))) + + +def get_membership_from_db_entry(json_string: str) -> dict: + """ + Get membership dict list from json string + Parse list to get last updated status of membership + + :param json_string: Json entry + :return: + """ + membership_list = json.loads(json_string) # type: list + # update first membership properties except for None value + return {key: value for list_item in membership_list for (key, value) in list_item.items() if value is not None or list_item == membership_list[0]} diff --git a/adapters/duniter_v18/wallets.py b/adapters/duniter_v18/wallets.py new file mode 100644 index 0000000000000000000000000000000000000000..cb4b04e64e060e58ee44b046cc2e315d86b2f2ec --- /dev/null +++ b/adapters/duniter_v18/wallets.py @@ -0,0 +1,45 @@ +import json +import plyvel +from pathlib import Path + + +class LevelDBWalletsRepository: + + DEFAULT_LEVELDB_PATH = "./leveldb" + DB_INDEX = "level_wallet" + + def __init__(self, leveldb_path: str): + """ + Init connection + + :param leveldb_path: Path of database + """ + self.index = plyvel.DB(str(Path(leveldb_path).joinpath(self.DB_INDEX))) + + def __iter__(self): + """ + Iterate over key: str, wallet: dict + + :return: + """ + for key, value in self.index.__iter__(): + yield key.decode("utf-8"), get_wallet_from_db_entry(value) + + def get(self, unlock_expression: str) -> dict: + """ + Return wallet dict from unlock_expression + + :param unlock_expression: Account money unlock expression + :return: + """ + return get_wallet_from_db_entry(self.index.get(unlock_expression.encode("utf-8"))) + + +def get_wallet_from_db_entry(json_string: str) -> dict: + """ + Get wallet dict from json string + + :param json_string: Json entry + :return: + """ + return json.loads(json_string) diff --git a/lib/functions.py b/lib/functions.py index 08fa0d99de650211a6373527822eb0b30702c39a..7a28e4df37cae4289965e95c170bd2f62fef85d8 100644 --- a/lib/functions.py +++ b/lib/functions.py @@ -1,5 +1,13 @@ +import os + +from adapters.duniter_v18.blocks import LevelDBBlocksRepository +from adapters.duniter_v18.certifications import LevelDBCertificationsRepository +from adapters.duniter_v18.identities import LevelDBIdentitiesRepository +from adapters.duniter_v18.memberships import LevelDBMembershipsRepository +from adapters.duniter_v18.wallets import LevelDBWalletsRepository from lib.utility import * -from time import time + +DEFAULT_LEVELDB_PATH = "./leveldb" def load_json_url(path): @@ -9,16 +17,16 @@ def load_json_url(path): def get_wallets_data(): # Get wallets balances data - wallets_data = load_json_url("inputs/wallets.json") + wallets_repository = LevelDBWalletsRepository(os.getenv("LEVELDB_PATH", DEFAULT_LEVELDB_PATH)) wallets = {} total_money = 0 # counter ignored_money = 0 # counter - for wallet in wallets_data: - balance = wallet["value"]["balance"] - if "&&" in wallet["key"]: + for unlock_expression, wallet in wallets_repository: + balance = wallet["balance"] + if "&&" in unlock_expression: ignored_money += balance continue - pubkey = wallet["key"].split("(")[1].split(")")[0] + pubkey = unlock_expression.split("(")[1].split(")")[0] # Remove pubkeys > 32 bytes # d2meevcahfts2gqmvmrw5hzi25jddikk4nc4u1fkwrau @@ -28,24 +36,16 @@ def get_wallets_data(): # jUPLL2BgY2QpheWEY3R13edV2Y4tvQMCXjJVM8PGDvyd # gatrpfmgsuez193bja5snivz3dsvsqn5kcm4ydtpknsy pubkey_bytes = base58.b58decode(pubkey) - pubkey_lenght = len(pubkey_bytes) - if pubkey_lenght > 32 or balance == 0: + pubkey_length = len(pubkey_bytes) + if pubkey_length > 32 or balance == 0: ignored_money += balance continue wallets.update({v1_pubkey_to_v2_address(pubkey): int(balance)}) total_money += balance - - return (wallets, total_money, ignored_money) - -def get_membership_expiry(): - """get membership expiry from input file""" - # Get Dex membership data - membership_data = load_json_url("inputs/membership.json") - membership_expiry = {} - for membership in membership_data: - membership_expiry[membership["key"]] = membership["value"][0]["expires_on"] - return membership_expiry + + return wallets, total_money, ignored_money + def get_identities_and_wallets(start_timestamp): """get identities with certifications and wallets with their balance @@ -63,21 +63,21 @@ def get_identities_and_wallets(start_timestamp): initial_monetary_mass = last_block["mass"] last_block_time = last_block["medianTime"] - # Get wallets data - (wallets, total_money, ignored_money) = get_wallets_data() - # Get membership expiry - membership_expiry = get_membership_expiry() - # Get Dex idty data - idty_data = load_json_url("inputs/idty.json") - # Get Dex certs data - certs_data = load_json_url("inputs/certs.json") - # Get blocs number with dates - blocs_data = load_json_url("inputs/blocs.json") + print(" parse Identities...") + # Get leveldb indices as Dex data + memberships_repository = LevelDBMembershipsRepository(os.getenv("LEVELDB_PATH", DEFAULT_LEVELDB_PATH)) + identities_repository = LevelDBIdentitiesRepository(os.getenv("LEVELDB_PATH", DEFAULT_LEVELDB_PATH)) + certifications_repository = LevelDBCertificationsRepository(os.getenv("LEVELDB_PATH", DEFAULT_LEVELDB_PATH)) + blocks_repository = LevelDBBlocksRepository(os.getenv("LEVELDB_PATH", DEFAULT_LEVELDB_PATH)) + # Get identities switches addresses_switches = load_json("custom/addresses_switches.json") # Get custom identities custom_identities = load_json("custom/identities.json") + # Get wallets data + print(" parse Wallets...") + (wallets, total_money, ignored_money) = get_wallets_data() # add ignored money to treasury and check initial monetary mass treasury += ignored_money # add ignored money to treasury wallet_sum = total_money + ignored_money @@ -94,16 +94,15 @@ def get_identities_and_wallets(start_timestamp): # TODO make sure that index respects order of arrival # Get identities names by pubkey - for idty in idty_data: - pubkey = idty["key"] + for pubkey, identity in identities_repository: address = v1_pubkey_to_v2_address(pubkey) - value = idty["value"][0] - index = value["wotb_id"] + 1 - uid = value["uid"] - is_member = value["member"] + index = identity["wotb_id"] + 1 + uid = identity["uid"] + is_member = identity["member"] identity_names[pubkey] = uid - membership_expire_on = date_to_bloc_number(membership_expiry[pubkey], start_timestamp) - if membership_expire_on < 0 : membership_expire_on = 0 # forget old expiry date + membership_expire_on = date_to_bloc_number(memberships_repository.get(pubkey)["expires_on"], start_timestamp) + if membership_expire_on < 0: + membership_expire_on = 0 # forget old expiry date # add address and balance to identity if address not in wallets: @@ -135,21 +134,20 @@ def get_identities_and_wallets(start_timestamp): identities.update(custom_identities) # get info from block - for bloc in blocs_data: - blocs[int(bloc["key"])] = bloc["value"]["medianTime"] + for block_number, block in blocks_repository: + blocs[block_number] = block["medianTime"] # Generate identities Ğ1v2 genesis json bloc # certs are stored per issuer in input file # certs are stored per receiver in output file print(" parse certification...") - for issuer in certs_data: - i_pubkey = issuer["key"] + for i_pubkey, issuer in certifications_repository: i_uid = identity_names[i_pubkey] i_address = v1_pubkey_to_v2_address(i_pubkey) - for cert in issuer["value"]["issued"]: + for cert in issuer["issued"]: # if certification expired, skip silently - if cert["expired_on"] != 0 : + if cert["expired_on"] != 0: continue r_pubkey = cert["receiver"] @@ -168,11 +166,11 @@ def get_identities_and_wallets(start_timestamp): cert_expire_on = date_to_bloc_number(cert_expire_at, start_timestamp) # if certification expiration date is before export, - # it is a renewed certification and can be ignored + # it is a renewed certification and can be ignored if cert_expire_at < last_block_time: continue # certifications can also have expired between export and start_timestamp - # in this case we display a warning because these missing certification + # in this case we display a warning because these missing certification # could lead to a genesis with not enough certifications received for some members if cert_expire_at <= start_timestamp: print(f"⚠️ {i_uid} → {r_uid} cert expired between export and start") @@ -184,7 +182,7 @@ def get_identities_and_wallets(start_timestamp): # add received certification to identity identities[r_uid]["certs_received"][i_uid] = cert_expire_on - return (identities, wallets) + return identities, wallets def get_smiths(): @@ -207,4 +205,4 @@ def fix_identities_with_not_enough_certs(identities): received_count = len(value["certs_received"]) if value["membership_expire_on"] > 0 and received_count < CERT_MIN_RECEIVED_CERT_TO_ISSUE_CERT: print(f"⚠️ {key} has received only {received_count} certs, disabling it") - value["membership_expire_on"] = 0 \ No newline at end of file + value["membership_expire_on"] = 0 diff --git a/main.py b/main.py index 0e6fa5a3558212035ce4c4932d510927da269aa4..03021a80a9a21cd48a6d89c2aeafe758bb81e338 100755 --- a/main.py +++ b/main.py @@ -16,8 +16,9 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. import sys +from time import time + from lib.functions import * -from lib.get_parameters import * def load_json_url(path): @@ -31,11 +32,12 @@ print("Generate ĞTest genesis with up-to-date Ğ1 data") opt1 = "" if len(sys.argv) > 1: opt1 = int(sys.argv[1]) - + # define start timestamp start_timestamp = opt1 # if not defined set start time to now -if start_timestamp == "": start_timestamp = int(time()) +if start_timestamp == "": + start_timestamp = int(time()) # Get ĞTest parameters print(" get ĞTest parameters...") @@ -47,12 +49,11 @@ sudo_key = "5Dq8xjvkmbz7q4g2LbZgyExD26VSCutfEc6n4W4AfQeVHZqz" print(" dump ĞTest parameters...") last_block = load_json_url("inputs/ud_value.json")[0]["value"] FIRST_UD_VALUE = last_block["dividend"] -FIRST_UD_REEVAL = date_to_bloc_number(last_block["udReevalTime"] , start_timestamp) +FIRST_UD_REEVAL = date_to_bloc_number(last_block["udReevalTime"], start_timestamp) INITAL_MONETARY_MASS = last_block["mass"] LAST_BLOCK_TIME = last_block["medianTime"] -# Add identities bloc -print(" parse identities...") +# Add identities and wallets (identities, other_wallets) = get_identities_and_wallets(start_timestamp) # FIXME avoid this diff --git a/requirements.txt b/requirements.txt index 6d6d7ee5e8dd32d43bda6f083b0d6c1b7e13ee2b..ca3dce0c59a8c411f7fe613c31edf60db6aa64c3 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ substrate-interface -base58 \ No newline at end of file +base58 +plyvel