Commit adbc9117 authored by Pierre-Jean CHANCELLIER's avatar Pierre-Jean CHANCELLIER

Merge branch 'master' into 'master'

[enh] add Python json-generator script

See merge request !1
parents 256a0563 f0daf8c7
Pipeline #4354 failed with stages
in 20 seconds
#!/usr/bin/env python3
"""
Copyright 2019 Pascal Engélibert <tuxmain@zettascript.org>
License GNU AGPL v3 (http://www.gnu.org/licenses/#AGPL)
Dependances: python3-plyvel
Command for installing:
$ sudo pip3 install plyvel
"""
import sys, os, time, math, json, urllib.request, plyvel
STEPMAX = 5
verbose = False
def getargv(arg, default=""):
if arg in sys.argv and len(sys.argv) > sys.argv.index(arg)+1:
return sys.argv[sys.argv.index(arg)+1]
else:
return default
def log(msg, end="\n"):
if verbose:
print(msg, end=end)
if __name__ == "__main__":
if "--help" in sys.argv or "-h" in sys.argv:
print("""Options:
-i <dir> Input duniter db (LevelDB)
-o <file> Output JSON
--es <url> URL of the ElasticSearch server
-d Download avatars
-a <dir> Avatars local path
-A <dir> Avatars public local path
-m <int> Avatars cache max age (s)
-M <file> Default avatar public local path
-v Verbose
Default options:
-i ~/.config/duniter/duniter_default/data/leveldb
-o wot.json
-e http://g1.data.le-sou.org
-a ../img/photos
-A img/photos
-m 604800 (7 days)
-M img/logo-g1.png""")
exit()
# Initialize
outdata = {"nodes":[], "edges":[]}
index = {}# pub -> id
# Read args
dbpath = os.path.expanduser(getargv("-i", "~/.config/duniter/duniter_default/data/leveldb"))
outpath = os.path.expanduser(getargv("-o", "wot.json"))
url_es = getargv("-e", "http://g1.data.le-sou.org")
download_avatars = "-d" in sys.argv
avatars_path = os.path.expanduser(getargv("-a", "../img/photos"))
avatars_url_path = getargv("-A", "img/photos")
avatars_max_age = time.time() - int(getargv("-m", 604800))# 7 days
avatars_default_url_path = getargv("-M", "img/logo-g1.png")
verbose = "-v" in sys.argv
# Get datas
log("Opening DBs...")
iindex = plyvel.DB(dbpath+"/level_iindex/")
cindex = plyvel.DB(dbpath+"/level_cindex/")
log("Generating nodes...")
i = 0
for pub, row in iindex:# Nodes
idty = json.loads(row.decode())[0]
certs = json.loads(cindex.get(pub).decode())
index[idty["pub"]] = str(i)
neighbors = certs["received"].copy()
degree_in = len(neighbors)
degree_out = 0
for cert in certs["issued"]:
if cert["expired_on"] == 0:
if not cert["receiver"] in neighbors:
neighbors.append(cert["receiver"])
degree_out += 1
degree = len(neighbors)
avatar_path = avatars_path+"/"+idty["pub"]+".png"
avatar_exists = os.path.isfile(avatar_path)
if download_avatars:
if not avatar_exists or os.path.getmtime(avatar_path) < avatars_max_age:
try:
log("Download avatar: " + idty["pub"], " ")
response = urllib.request.urlopen(url_es+"/user/profile/"+idty["pub"]+"/_image/avatar.png")
f = open(avatar_path, "wb")
f.write(response.read())
f.close()
log("OK")
avatar_url = avatars_url_path+"/"+idty["pub"]+".png"
except urllib.error.HTTPError:
avatar_url = avatars_default_url_path
log("ERR")
else:
avatar_url = avatars_url_path+"/"+idty["pub"]+".png"
elif avatar_exists:
avatar_url = avatars_url_path+"/"+idty["pub"]+".png"
else:
avatar_url = avatars_default_url_path
outdata["nodes"].append({
"id": str(i),
"label": idty["uid"],
"type": "image",
"url": avatar_url,
"x": 0,
"y": 0,
"size": degree,
"color": "rgba(255,0,0,1)",
"attributes": {
"pubkey": idty["pub"],
"degree": degree,
"referent": False,
"outDegree": degree_out,
"inDegree": degree_in
}
})
i += 1
iindex.close()
nb_idties = i
log("Generating edges...")
i = 0
for pub, row in cindex:# Edges
certs = json.loads(row.decode())
issuer_id = index[pub.decode()]
for cert in certs["issued"]:
if cert["expired_on"] == 0:
outdata["edges"].append({
"id": str(i),
"source": issuer_id,
"target": index[cert["receiver"]]
})
i += 1
cindex.close()
# Update referents
log("Updating referents...")
crit_referent = math.ceil((nb_idties+1) ** (1/STEPMAX))
for i in range(nb_idties):
if outdata["nodes"][i]["attributes"]["inDegree"] >= crit_referent and \
outdata["nodes"][i]["attributes"]["outDegree"] >= crit_referent:
outdata["nodes"][i]["attributes"]["referent"] = True
outdata["nodes"][i]["color"] = "rgba(0,128,0,1)"
# Save output
log("Saving output...")
outfile = open(outpath, "w")
json.dump(outdata, outfile)
outfile.close()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment