From db0822b3947112bb587a05a32d847fc288aef803 Mon Sep 17 00:00:00 2001
From: Benoit Lavenier <benoit.lavenier@e-is.pro>
Date: Mon, 15 May 2023 15:43:08 +0200
Subject: [PATCH] [enh] Allow to call '/wot/requirements/:search' with a query
param (e.g. '?pubkey=true') to optimize DB queries - close #1439
---
app/lib/dal/fileDAL.ts | 136 ++++++++++--------
app/lib/dal/indexDAL/abstract/IIndexDAO.ts | 2 +
app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts | 10 ++
app/lib/dal/indexDAL/sqlite/SqliteIIndex.ts | 10 ++
app/lib/dal/sqliteDAL/IdentityDAL.ts | 4 +
app/modules/bma/lib/controllers/wot.ts | 10 +-
app/service/IdentityService.ts | 4 +
test/integration/identity/identity-test.ts | 36 +++++
8 files changed, 153 insertions(+), 59 deletions(-)
diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts
index 57c9eb988..0f1586155 100644
--- a/app/lib/dal/fileDAL.ts
+++ b/app/lib/dal/fileDAL.ts
@@ -13,12 +13,12 @@
import * as fs from "fs";
import * as path from "path";
-import { SQLiteDriver } from "./drivers/SQLiteDriver";
-import { ConfDAL } from "./fileDALs/ConfDAL";
-import { ConfDTO } from "../dto/ConfDTO";
-import { BlockDTO } from "../dto/BlockDTO";
-import { DBHead } from "../db/DBHead";
-import { DBIdentity, IdentityDAL } from "./sqliteDAL/IdentityDAL";
+import {SQLiteDriver} from "./drivers/SQLiteDriver";
+import {ConfDAL} from "./fileDALs/ConfDAL";
+import {ConfDTO} from "../dto/ConfDTO";
+import {BlockDTO} from "../dto/BlockDTO";
+import {DBHead} from "../db/DBHead";
+import {DBIdentity, IdentityDAL} from "./sqliteDAL/IdentityDAL";
import {
CindexEntry,
FullCindexEntry,
@@ -31,56 +31,56 @@ import {
SimpleUdEntryForWallet,
SindexEntry,
} from "../indexer";
-import { TransactionDTO } from "../dto/TransactionDTO";
-import { CertDAL, DBCert } from "./sqliteDAL/CertDAL";
-import { DBBlock } from "../db/DBBlock";
-import { DBMembership, MembershipDAL } from "./sqliteDAL/MembershipDAL";
-import { MerkleDTO } from "../dto/MerkleDTO";
-import { CommonConstants } from "../common-libs/constants";
-import { PowDAL } from "./fileDALs/PowDAL";
-import { Initiable } from "./sqliteDAL/Initiable";
-import { MetaDAL } from "./sqliteDAL/MetaDAL";
-import { DataErrors } from "../common-libs/errors";
-import { BasicRevocableIdentity, IdentityDTO } from "../dto/IdentityDTO";
-import { FileSystem } from "../system/directory";
-import { RustDbTx, RustServer, RustServerConf, Wot } from "../../../neon/lib";
-import { IIndexDAO } from "./indexDAL/abstract/IIndexDAO";
-import { BIndexDAO } from "./indexDAL/abstract/BIndexDAO";
-import { MIndexDAO } from "./indexDAL/abstract/MIndexDAO";
-import { SIndexDAO } from "./indexDAL/abstract/SIndexDAO";
-import { CIndexDAO } from "./indexDAL/abstract/CIndexDAO";
-import { IdentityForRequirements } from "../../service/BlockchainService";
-import { NewLogger } from "../logger";
-import { BlockchainDAO } from "./indexDAL/abstract/BlockchainDAO";
-import { WalletDAO } from "./indexDAL/abstract/WalletDAO";
-import { PeerDAO } from "./indexDAL/abstract/PeerDAO";
-import { DBTx } from "../db/DBTx";
-import { DBWallet } from "../db/DBWallet";
-import { Tristamp } from "../common/Tristamp";
-import { CFSCore } from "./fileDALs/CFSCore";
-import { Underscore } from "../common-libs/underscore";
-import { DBPeer } from "../db/DBPeer";
-import { MonitorFlushedIndex } from "../debug/MonitorFlushedIndex";
-import { cliprogram } from "../common-libs/programOptions";
-import { DividendDAO, UDSource } from "./indexDAL/abstract/DividendDAO";
-import { HttpSource, HttpUD } from "../../modules/bma/lib/dtos";
-import { GenericDAO } from "./indexDAL/abstract/GenericDAO";
-import { MonitorExecutionTime } from "../debug/MonitorExecutionTime";
-import { LevelDBDividend } from "./indexDAL/leveldb/LevelDBDividend";
-import { LevelDBBindex } from "./indexDAL/leveldb/LevelDBBindex";
-
-import { LevelUp } from "levelup";
-import { LevelDBBlockchain } from "./indexDAL/leveldb/LevelDBBlockchain";
-import { LevelDBSindex } from "./indexDAL/leveldb/LevelDBSindex";
-import { SqlitePeers } from "./indexDAL/sqlite/SqlitePeers";
-import { LevelDBWallet } from "./indexDAL/leveldb/LevelDBWallet";
-import { LevelDBCindex } from "./indexDAL/leveldb/LevelDBCindex";
-import { LevelDBIindex } from "./indexDAL/leveldb/LevelDBIindex";
-import { LevelDBMindex } from "./indexDAL/leveldb/LevelDBMindex";
-import { ConfDAO } from "./indexDAL/abstract/ConfDAO";
-import { ServerDAO } from "./server-dao";
-import { PeerDTO } from "../dto/PeerDTO";
-import { RustPendingTx } from "../../../neon/native/server";
+import {TransactionDTO} from "../dto/TransactionDTO";
+import {CertDAL, DBCert} from "./sqliteDAL/CertDAL";
+import {DBBlock} from "../db/DBBlock";
+import {DBMembership, MembershipDAL} from "./sqliteDAL/MembershipDAL";
+import {MerkleDTO} from "../dto/MerkleDTO";
+import {CommonConstants} from "../common-libs/constants";
+import {PowDAL} from "./fileDALs/PowDAL";
+import {Initiable} from "./sqliteDAL/Initiable";
+import {MetaDAL} from "./sqliteDAL/MetaDAL";
+import {DataErrors} from "../common-libs/errors";
+import {BasicRevocableIdentity, IdentityDTO} from "../dto/IdentityDTO";
+import {FileSystem} from "../system/directory";
+import {RustDbTx, RustServer, Wot} from "../../../neon/lib";
+import {IIndexDAO} from "./indexDAL/abstract/IIndexDAO";
+import {BIndexDAO} from "./indexDAL/abstract/BIndexDAO";
+import {MIndexDAO} from "./indexDAL/abstract/MIndexDAO";
+import {SIndexDAO} from "./indexDAL/abstract/SIndexDAO";
+import {CIndexDAO} from "./indexDAL/abstract/CIndexDAO";
+import {IdentityForRequirements} from "../../service/BlockchainService";
+import {NewLogger} from "../logger";
+import {BlockchainDAO} from "./indexDAL/abstract/BlockchainDAO";
+import {WalletDAO} from "./indexDAL/abstract/WalletDAO";
+import {PeerDAO} from "./indexDAL/abstract/PeerDAO";
+import {DBTx} from "../db/DBTx";
+import {DBWallet} from "../db/DBWallet";
+import {Tristamp} from "../common/Tristamp";
+import {CFSCore} from "./fileDALs/CFSCore";
+import {Underscore} from "../common-libs/underscore";
+import {DBPeer} from "../db/DBPeer";
+import {MonitorFlushedIndex} from "../debug/MonitorFlushedIndex";
+import {cliprogram} from "../common-libs/programOptions";
+import {DividendDAO, UDSource} from "./indexDAL/abstract/DividendDAO";
+import {HttpSource, HttpUD} from "../../modules/bma/lib/dtos";
+import {GenericDAO} from "./indexDAL/abstract/GenericDAO";
+import {MonitorExecutionTime} from "../debug/MonitorExecutionTime";
+import {LevelDBDividend} from "./indexDAL/leveldb/LevelDBDividend";
+import {LevelDBBindex} from "./indexDAL/leveldb/LevelDBBindex";
+
+import {LevelUp} from "levelup";
+import {LevelDBBlockchain} from "./indexDAL/leveldb/LevelDBBlockchain";
+import {LevelDBSindex} from "./indexDAL/leveldb/LevelDBSindex";
+import {SqlitePeers} from "./indexDAL/sqlite/SqlitePeers";
+import {LevelDBWallet} from "./indexDAL/leveldb/LevelDBWallet";
+import {LevelDBCindex} from "./indexDAL/leveldb/LevelDBCindex";
+import {LevelDBIindex} from "./indexDAL/leveldb/LevelDBIindex";
+import {LevelDBMindex} from "./indexDAL/leveldb/LevelDBMindex";
+import {ConfDAO} from "./indexDAL/abstract/ConfDAO";
+import {ServerDAO} from "./server-dao";
+import {PeerDTO} from "../dto/PeerDTO";
+import {RustPendingTx} from "../../../neon/native/server";
const readline = require("readline");
const indexer = require("../indexer").Indexer;
@@ -929,8 +929,30 @@ export class FileDAL implements ServerDAO {
return i;
})
);
+ return this.fillIdentitiesRevocation(found);
+ }
+
+ async searchJustIdentitiesByPubkey(pubkey: string): Promise<DBIdentity[]> {
+ const pendings = await this.idtyDAL.getFromPubkey(pubkey);
+ const writtens = await this.iindexDAL.searchByPubkey(pubkey);
+ const nonPendings = Underscore.filter(writtens, (w: IindexEntry) => {
+ return Underscore.where(pendings, { pubkey: w.pub }).length == 0;
+ });
+ const found = pendings.concat(
+ nonPendings.map((i: any) => {
+ // Use the correct field
+ i.pubkey = i.pub;
+ return i;
+ })
+ );
+ return this.fillIdentitiesRevocation(found);
+ }
+
+ private async fillIdentitiesRevocation(
+ identities: DBIdentity[]
+ ): Promise<DBIdentity[]> {
return await Promise.all<DBIdentity>(
- found.map(async (f) => {
+ identities.map(async (f) => {
const ms = await this.mindexDAL.getReducedMSForImplicitRevocation(
f.pubkey
);
diff --git a/app/lib/dal/indexDAL/abstract/IIndexDAO.ts b/app/lib/dal/indexDAL/abstract/IIndexDAO.ts
index 1c1a08cd8..ecf64d441 100644
--- a/app/lib/dal/indexDAL/abstract/IIndexDAO.ts
+++ b/app/lib/dal/indexDAL/abstract/IIndexDAO.ts
@@ -19,6 +19,8 @@ export interface IIndexDAO extends ReduceableDAO<IindexEntry> {
searchThoseMatching(search: string): Promise<OldIindexEntry[]>;
+ searchByPubkey(search: string): Promise<OldIindexEntry[]>;
+
getFullFromUID(uid: string): Promise<FullIindexEntry>;
getFullFromPubkey(pub: string): Promise<FullIindexEntry>;
diff --git a/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts b/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts
index c53f3dfc4..e7157b34e 100644
--- a/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts
+++ b/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts
@@ -280,4 +280,14 @@ export class LevelDBIindex extends LevelDBTable<IindexEntry[]>
.filter((u) => u.pub)
.concat(pubIdentities.filter((p) => p.pub));
}
+
+ async searchByPubkey(pub: string): Promise<OldIindexEntry[]> {
+ const identities = await this.findByPub(pub);
+ if (!identities.length) {
+ return [];
+ }
+ // TODO Why do we need to use reduce() on array ? This will merge items into one object
+ const mergedIdentities = OldTransformers.toOldIindexEntry(reduce(identities))
+ return [mergedIdentities];
+ }
}
diff --git a/app/lib/dal/indexDAL/sqlite/SqliteIIndex.ts b/app/lib/dal/indexDAL/sqlite/SqliteIIndex.ts
index fdbadbdaf..2cf0aeb7f 100644
--- a/app/lib/dal/indexDAL/sqlite/SqliteIIndex.ts
+++ b/app/lib/dal/indexDAL/sqlite/SqliteIIndex.ts
@@ -263,4 +263,14 @@ export class SqliteIIndex extends SqliteTable<IindexEntry>
])
).map(OldTransformers.toOldIindexEntry);
}
+
+ @MonitorExecutionTime()
+ async searchByPubkey(pub: string): Promise<OldIindexEntry[]> {
+ // TODO Why not need reduce() here ? As done in the LevelDB implementation
+ return (
+ await this.find("SELECT * FROM iindex WHERE pub = ?", [
+ pub
+ ])
+ ).map(OldTransformers.toOldIindexEntry);
+ }
}
diff --git a/app/lib/dal/sqliteDAL/IdentityDAL.ts b/app/lib/dal/sqliteDAL/IdentityDAL.ts
index 96d602a0a..9716b13f7 100644
--- a/app/lib/dal/sqliteDAL/IdentityDAL.ts
+++ b/app/lib/dal/sqliteDAL/IdentityDAL.ts
@@ -328,6 +328,10 @@ export class IdentityDAL extends AbstractSQLite<DBIdentity> {
});
}
+ getFromPubkey(pubkey: string) {
+ return this.sqlFind({pubkey});
+ }
+
async trimExpiredIdentities(medianTime: number) {
await this.exec(
"DELETE FROM " +
diff --git a/app/modules/bma/lib/controllers/wot.ts b/app/modules/bma/lib/controllers/wot.ts
index 17e57ed83..43c6e1e97 100644
--- a/app/modules/bma/lib/controllers/wot.ts
+++ b/app/modules/bma/lib/controllers/wot.ts
@@ -187,9 +187,15 @@ export class WOTBinding extends AbstractController {
async requirements(req: any): Promise<HttpRequirements> {
const search = await ParametersService.getSearchP(req);
- const identities: any = await this.IdentityService.searchIdentities(search);
+ let identities: any = [];
+ if (req.query?.pubkey === true || req.query?.pubkey === "true") {
+ identities = await this.IdentityService.searchIdentitiesByPubkey(search);
+ }
+ else {
+ identities = await this.IdentityService.searchIdentities(search);
+ }
const all: HttpIdentityRequirement[] = await this.BlockchainService.requirementsOfIdentities(
- identities
+ identities
);
if (!all || !all.length) {
throw BMAConstants.ERRORS.NO_IDTY_MATCHING_PUB_OR_UID;
diff --git a/app/service/IdentityService.ts b/app/service/IdentityService.ts
index 676b8602a..f3173f885 100644
--- a/app/service/IdentityService.ts
+++ b/app/service/IdentityService.ts
@@ -54,6 +54,10 @@ export class IdentityService extends FIFOService {
return this.dal.searchJustIdentities(search);
}
+ searchIdentitiesByPubkey(pubkey: string) {
+ return this.dal.searchJustIdentitiesByPubkey(pubkey);
+ }
+
async findMember(search: string) {
let idty = null;
if (search.match(constants.PUBLIC_KEY)) {
diff --git a/test/integration/identity/identity-test.ts b/test/integration/identity/identity-test.ts
index 847ff19e8..d87331eff 100644
--- a/test/integration/identity/identity-test.ts
+++ b/test/integration/identity/identity-test.ts
@@ -291,6 +291,42 @@ describe("Identities collision", function() {
});
});
+ it('requirements by pubkey of cat', function() {
+ return expectAnswer(rp('http://127.0.0.1:7799/wot/requirements/HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd?pubkey=true', { json: true }), function(res:HttpRequirements) {
+ res.should.have.property('identities').be.an.Array;
+ res.should.have.property('identities').have.length(1);
+ res.identities[0].should.have.property('pubkey').equal('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+ res.identities[0].should.have.property('uid').equal('cat');
+ res.identities[0].should.have.property('meta').property('timestamp');
+ res.identities[0].should.have.property('wasMember').equal(true);
+ res.identities[0].should.have.property('expired').equal(false); // Because it has been a member once! So its identity will exist forever.
+ res.identities[0].should.have.property('outdistanced').equal(false);
+ res.identities[0].should.have.property('isSentry').equal(true); // dSen = 2, cat has issued and received 2 certs with tic and toc
+ res.identities[0].should.have.property('certifications').have.length(2);
+ res.identities[0].should.have.property('membershipPendingExpiresIn').equal(0);
+ res.identities[0].should.have.property('membershipExpiresIn').greaterThan(9000);
+ });
+ });
+
+ it('requirements by pubkey of man1', function() {
+ return expectAnswer(rp('http://127.0.0.1:7799/wot/requirements/12AbjvYY5hxV4v2KrN9pnGzgFxogwrzgYyncYHHsyFDK?pubkey=true', { json: true }), function(res:HttpRequirements) {
+ res.should.have.property('identities').be.an.Array;
+ res.should.have.property('identities').have.length(1);
+ res.identities[0].should.have.property('pubkey').equal('12AbjvYY5hxV4v2KrN9pnGzgFxogwrzgYyncYHHsyFDK');
+ res.identities[0].should.have.property('uid').equal('man1');
+ res.identities[0].should.have.property('meta').property('timestamp');
+ res.identities[0].should.have.property('expired').equal(false);
+ res.identities[0].should.have.property('outdistanced').equal(false);
+ res.identities[0].should.have.property('isSentry').equal(false); // Not a member, also dSen = 2, but man1 has only 1 certification
+ res.identities[0].should.have.property('certifications').length(1);
+ res.identities[0].certifications[0].should.have.property('from').equal('2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc');
+ res.identities[0].certifications[0].should.have.property('to').equal('12AbjvYY5hxV4v2KrN9pnGzgFxogwrzgYyncYHHsyFDK');
+ res.identities[0].certifications[0].should.have.property('expiresIn').greaterThan(0);
+ res.identities[0].should.have.property('membershipPendingExpiresIn').greaterThan(9000);
+ res.identities[0].should.have.property('membershipExpiresIn').equal(0);
+ });
+ });
+
it('should have certified-by/tic giving results', function() {
return expectAnswer(rp('http://127.0.0.1:7799/wot/certified-by/tic', { json: true }), function(res:HttpCertifications) {
res.should.have.property('pubkey').equal('DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV');
--
GitLab