WIP multiple fixes

- export timestamp vs start timestamp
- initial monetary mass
- TODO treasury to collect ignored money
- membership expiration date
- warning if expired certs

lib/functions.py

@@ -11,11 +11,13 @@ def get_wallets_data():
     # Get wallets balances data
     wallets_data = load_json_url("inputs/wallets.json")
     wallets = {}
+    ignored_money = 0
     for wallet in wallets_data:
+        balance = wallet["value"]["balance"]
         if "&&" in wallet["key"]:
+            ignored_money += balance
             continue
         pubkey = wallet["key"].split("(")[1].split(")")[0]
-        balance = wallet["value"]["balance"]
 
         # Remove pubkeys > 32 bytes
         # d2meevcahfts2gqmvmrw5hzi25jddikk4nc4u1fkwrau
@@ -27,25 +29,41 @@ def get_wallets_data():
         pubkey_bytes = base58.b58decode(pubkey)
         pubkey_lenght = len(pubkey_bytes)
         if pubkey_lenght > 32 or balance == 0:
+            ignored_money += balance
             continue
 
         wallets.update({v1_pubkey_to_v2_address(pubkey): int(balance)})
     
-    return wallets
+    # add ignored money to treasury
+    # FIXME get treasury address
+    wallets["5EYCAe5ijiYfyeZ2JJCGq56LmPyNRAKzpG4QkoQkkQNB5e6Z"] = ignored_money
 
+    return wallets
 
-def get_identities_and_wallets(start_timestamp):
+def get_membership_expiry():
+    """get membership expiry from input file"""
+    # Get Dex membership data
+    membership_data = load_json_url("inputs/membership.json")
+    membership_expiry = {}
+    for membership in membership_data:
+        membership_expiry[membership["key"]] = membership["value"][0]["expires_on"]
+    return membership_expiry
+
+def get_identities_and_wallets(last_block_time, start_timestamp):
+    """get identities with certifications and wallets with their balance
+    last_block_time is the time of the last exported block
+    start_timestamp is the timestamp of the v2 genesis
+        used to estimate cert expiration in number of blocks
+    """
     # initialize
     blocs = {}
     identity_names = {}
     identities = {}
 
-    # if no arguments, set start time to now
-    if start_timestamp == "":
-        start_timestamp = int(time())
-
     # Get wallets data
     wallets = get_wallets_data()
+    # Get membership expiry
+    membership_expiry = get_membership_expiry()
     # Get Dex idty data
     idty_data = load_json_url("inputs/idty.json")
     # Get Dex certs data
@@ -57,21 +75,18 @@ def get_identities_and_wallets(start_timestamp):
     # Get custom identities
     custom_identities = load_json("custom/identities.json")
 
-    # TODO manage membership expiration
     # TODO make sure that index respects order of arrival
     # Get identities names by pubkey
-    for i, idty in enumerate(idty_data):
+    for idty in idty_data:
         pubkey = idty["key"]
         address = v1_pubkey_to_v2_address(pubkey)
         value = idty["value"][0]
+        index = value["wotb_id"] + 1
         uid = value["uid"]
+        is_member = value["member"]
         identity_names[pubkey] = uid
-
-        # FIXME
-        if value["member"]:
-            membership_expire_on = 666
-        else:
-            membership_expire_on = 0
+        membership_expire_on = date_to_bloc_number(membership_expiry[pubkey], start_timestamp)
+        if membership_expire_on < 0 : membership_expire_on = 0 # forget old expiry date
 
         # add address and balance to identity
         if address not in wallets:
@@ -84,11 +99,11 @@ def get_identities_and_wallets(start_timestamp):
 
         # fill in identity entry
         identities[uid] = {
-            "index": i,
+            "index": index,
             "owner_key": address,
             "balance": balance,
-            "membership_expire_on": membership_expire_on,
-            "next_cert_issuable_on": 0,
+            "membership_expire_on": membership_expire_on if is_member else 0,
+            "next_cert_issuable_on": 0, # initialized to zero, modified later
             "certs_received": {},
         }
 
@@ -116,23 +131,34 @@ def get_identities_and_wallets(start_timestamp):
         i_address = v1_pubkey_to_v2_address(i_pubkey)
 
         for cert in issuer["value"]["issued"]:
+            # if certification expired, skip silently
+            if cert["expired_on"] != 0 :
+                continue
+
             r_pubkey = cert["receiver"]
             r_uid = identity_names[r_pubkey]
             r_address = v1_pubkey_to_v2_address(r_pubkey)
 
-            # get expiration bloc number from cert written time
-            # ⚠️ cert expire based on creation date, not write date
+            # get expiration of certification
             # timestamp of cert creation
-            created_on = blocs[cert["created_on"]]
+            created_at = blocs[cert["created_on"]]
             # block of cert creation
-            created_on = date_to_bloc_number(created_on, start_timestamp)
+            created_on = date_to_bloc_number(created_at, start_timestamp)
             # block of next issuable cert
             next_issuable_on = created_on + CERT_PERIOD
-            # block of cert expiration
-            cert_expire_on = created_on + CERT_VALIDITY_PERIOD
+            # timestamp of cert expiration
+            cert_expire_at = cert["expires_on"]
+            cert_expire_on = date_to_bloc_number(cert_expire_at, start_timestamp)
 
-            # if certification is expired, skip
-            if cert_expire_on <= 0:
+            # if certification expiration date is before export,
+            # it is a renewed certification and can be ignored 
+            if cert_expire_at < last_block_time:
+                continue
+            # certifications can also have expired between export and start_timestamp
+            # in this case we display a warning because these missing certification 
+            # could lead to a genesis with not enough certifications received for some members
+            if cert_expire_at <= start_timestamp:
+                print(f"⚠️ {i_uid} → {r_uid} cert expired between export and start")
                 continue
             # bump the next issuable date if necessary
             elif next_issuable_on > identities[i_uid]["next_cert_issuable_on"]:

lib/utility.py

@@ -13,6 +13,7 @@ def v1_pubkey_to_v2_address(pubkey):
         pubkey_bytes = bytes([0] * (32 - pubkey_length)) + pubkey_bytes
 
     # get incomplete Substrate keypair (only public key) from public key bytes
+    # FIXME use correct prefix (not 42)
     keypair = Keypair(
         public_key=pubkey_bytes, ss58_format=42, crypto_type=KeypairType.ED25519
     )

main.py

@@ -30,7 +30,12 @@ print("Generate ĞTest genesis with up-to-date Ğ1 data")
 # Get optional arguments
 opt1 = ""
 if len(sys.argv) > 1:
-    opt1 = sys.argv[1]
+    opt1 = int(sys.argv[1])
+
+# define start timestamp
+start_timestamp = opt1
+# if not defined set start time to now
+if start_timestamp == "": start_timestamp = int(time())
 
 # Get ĞTest parameters
 print("    get ĞTest parameters...")
@@ -40,12 +45,14 @@ sudo_key = "5Dq8xjvkmbz7q4g2LbZgyExD26VSCutfEc6n4W4AfQeVHZqz"
 
 # Dump ĞTest parameters
 print("    dump ĞTest parameters...")
-ud_data = load_json_url("inputs/ud_value.json")
-FIRST_UD = ud_data[0]["value"]["dividend"]
+last_block = load_json_url("inputs/ud_value.json")[0]["value"]
+FIRST_UD = last_block["dividend"]
+INITAL_MONETARY_MASS = last_block["mass"]
+LAST_BLOCK_TIME = last_block["medianTime"]
 
 # Add identities bloc
 print("    parse identities...")
-identities_wallets = get_identities_and_wallets(opt1)
+(identities_wallets, other_wallets) = get_identities_and_wallets(LAST_BLOCK_TIME, start_timestamp)
 
 # Add wallets bloc
 print("    add simple wallets...")
@@ -54,12 +61,12 @@ print("    add simple wallets...")
 gtest_genesis = {
     "first_ud": FIRST_UD,
     "first_ud_reeval": FIRST_UD_REEVAL,
-    "initial_monetary_mass": 0,  # TODO
+    "initial_monetary_mass": INITAL_MONETARY_MASS,
     "smiths": smiths,
     "technical_committee": technical_committee,
     "sudo_key": sudo_key,
-    "identities": identities_wallets[0],
-    "wallets": identities_wallets[1],
+    "identities": identities_wallets,
+    "wallets": other_wallets,
 }
 
 # Dump JSON to file