From add7d283a6162ca8073164d2199bf3339a45d294 Mon Sep 17 00:00:00 2001
From: Benoit Lavenier <benoit.lavenier@e-is.pro>
Date: Tue, 30 May 2023 14:26:39 +0200
Subject: [PATCH] [fix] BMA: optimize `/tx/requirements/:search` for Duniter
1.8 - close #1439
---
app/lib/dal/fileDAL.ts | 37 ++++++++-
app/lib/dal/indexDAL/abstract/IIndexDAO.ts | 2 +
app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts | 8 ++
app/lib/dal/indexDAL/sqlite/SqliteIIndex.ts | 14 +++-
app/lib/dal/sqliteDAL/IdentityDAL.ts | 4 +
app/modules/bma/lib/controllers/wot.ts | 36 ++++++--
app/modules/bma/lib/parameters.ts | 10 +++
app/service/IdentityService.ts | 4 +
test/integration/identity/identity-test.ts | 82 +++++++++++++++++++
9 files changed, 188 insertions(+), 9 deletions(-)
diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts
index a809685f3..585e0d2a6 100644
--- a/app/lib/dal/fileDAL.ts
+++ b/app/lib/dal/fileDAL.ts
@@ -707,6 +707,17 @@ export class FileDAL implements ServerDAO {
return await this.iindexDAL.getFromPubkeyOrUid(search);
}
+ async getWrittenIdtyByPubkeyForHashingAndIsMember(
+ pub: string
+ ): Promise<{
+ uid: string;
+ created_on: string;
+ pub: string;
+ member: boolean;
+ } | null> {
+ return await this.iindexDAL.getFromPubkey(pub);
+ }
+
async getWrittenIdtyByPubkeyForRevocationCheck(
pubkey: string
): Promise<{
@@ -866,8 +877,32 @@ export class FileDAL implements ServerDAO {
return i;
})
);
+ return this.fillIdentitiesRevocation(found);
+ }
+
+ async searchJustIdentitiesByPubkey(pubkey: string): Promise<DBIdentity[]> {
+ const pendings = await this.idtyDAL.findByPub(pubkey);
+ const writtenIdty = await this.iindexDAL.getOldFromPubkey(pubkey);
+ const nonPendings =
+ writtenIdty &&
+ Underscore.where(pendings, { pubkey: writtenIdty.pub }).length === 0
+ ? [writtenIdty]
+ : [];
+ const found = pendings.concat(
+ nonPendings.map((i: any) => {
+ // Use the correct field
+ i.pubkey = i.pub;
+ return i;
+ })
+ );
+ return this.fillIdentitiesRevocation(found);
+ }
+
+ private async fillIdentitiesRevocation(
+ identities: DBIdentity[]
+ ): Promise<DBIdentity[]> {
return await Promise.all<DBIdentity>(
- found.map(async (f) => {
+ identities.map(async (f) => {
const ms = await this.mindexDAL.getReducedMSForImplicitRevocation(
f.pubkey
);
diff --git a/app/lib/dal/indexDAL/abstract/IIndexDAO.ts b/app/lib/dal/indexDAL/abstract/IIndexDAO.ts
index 1c1a08cd8..e6ec8965a 100644
--- a/app/lib/dal/indexDAL/abstract/IIndexDAO.ts
+++ b/app/lib/dal/indexDAL/abstract/IIndexDAO.ts
@@ -19,6 +19,8 @@ export interface IIndexDAO extends ReduceableDAO<IindexEntry> {
searchThoseMatching(search: string): Promise<OldIindexEntry[]>;
+ getOldFromPubkey(pub: string): Promise<OldIindexEntry | null>;
+
getFullFromUID(uid: string): Promise<FullIindexEntry>;
getFullFromPubkey(pub: string): Promise<FullIindexEntry>;
diff --git a/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts b/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts
index c53f3dfc4..fd5010024 100644
--- a/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts
+++ b/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts
@@ -280,4 +280,12 @@ export class LevelDBIindex extends LevelDBTable<IindexEntry[]>
.filter((u) => u.pub)
.concat(pubIdentities.filter((p) => p.pub));
}
+
+ async getOldFromPubkey(pub: string): Promise<OldIindexEntry | null> {
+ const identities = await this.findByPub(pub);
+ if (!identities.length) {
+ return null;
+ }
+ return OldTransformers.toOldIindexEntry(reduce(identities));
+ }
}
diff --git a/app/lib/dal/indexDAL/sqlite/SqliteIIndex.ts b/app/lib/dal/indexDAL/sqlite/SqliteIIndex.ts
index fdbadbdaf..0ad31596c 100644
--- a/app/lib/dal/indexDAL/sqlite/SqliteIIndex.ts
+++ b/app/lib/dal/indexDAL/sqlite/SqliteIIndex.ts
@@ -1,4 +1,4 @@
-import { FullIindexEntry, IindexEntry, Indexer } from "../../../indexer";
+import {FullIindexEntry, IindexEntry, Indexer, reduce} from "../../../indexer";
import { SQLiteDriver } from "../../drivers/SQLiteDriver";
import { MonitorExecutionTime } from "../../../debug/MonitorExecutionTime";
import { IIndexDAO } from "../abstract/IIndexDAO";
@@ -212,6 +212,18 @@ export class SqliteIIndex extends SqliteTable<IindexEntry>
return (await this.getFromUID(uid)) as FullIindexEntry;
}
+ @MonitorExecutionTime()
+ async getOldFromPubkey(pub: string): Promise<OldIindexEntry | null> {
+ const entries = await this.find(
+ "SELECT * FROM iindex WHERE pub = ? order by writtenOn ASC",
+ [pub]
+ );
+ if (!entries.length) {
+ return null;
+ }
+ return OldTransformers.toOldIindexEntry(reduce(entries));
+ }
+
@MonitorExecutionTime()
async getMembers(): Promise<{ pubkey: string; uid: string | null }[]> {
const members = await this.find(
diff --git a/app/lib/dal/sqliteDAL/IdentityDAL.ts b/app/lib/dal/sqliteDAL/IdentityDAL.ts
index 96d602a0a..851b27c0d 100644
--- a/app/lib/dal/sqliteDAL/IdentityDAL.ts
+++ b/app/lib/dal/sqliteDAL/IdentityDAL.ts
@@ -328,6 +328,10 @@ export class IdentityDAL extends AbstractSQLite<DBIdentity> {
});
}
+ findByPub(pub: string) {
+ return this.sqlFind({ pubkey: pub });
+ }
+
async trimExpiredIdentities(medianTime: number) {
await this.exec(
"DELETE FROM " +
diff --git a/app/modules/bma/lib/controllers/wot.ts b/app/modules/bma/lib/controllers/wot.ts
index 61e869b06..73422c518 100644
--- a/app/modules/bma/lib/controllers/wot.ts
+++ b/app/modules/bma/lib/controllers/wot.ts
@@ -132,9 +132,16 @@ export class WOTBinding extends AbstractController {
async certifiersOf(req: any): Promise<HttpCertifications> {
const search = await ParametersService.getSearchP(req);
- const idty = (await this.server.dal.getWrittenIdtyByPubkeyOrUIdForHashingAndIsMember(
- search
- )) as FullIindexEntry;
+ let idty: FullIindexEntry;
+ if (ParametersService.getIsPubkey(req)) {
+ idty = (await this.server.dal.getWrittenIdtyByPubkeyForHashingAndIsMember(
+ search
+ )) as FullIindexEntry;
+ } else {
+ idty = (await this.server.dal.getWrittenIdtyByPubkeyOrUIdForHashingAndIsMember(
+ search
+ )) as FullIindexEntry;
+ }
const certs = await this.server.dal.certsToTarget(
idty.pub,
IdentityDTO.getTargetHash(idty)
@@ -180,7 +187,15 @@ export class WOTBinding extends AbstractController {
async requirements(req: any): Promise<HttpRequirements> {
const search = await ParametersService.getSearchP(req);
- const identities: any = await this.IdentityService.searchIdentities(search);
+ let identities: any = [];
+ if (ParametersService.getIsPubkey(req)) {
+ if (!BMAConstants.PUBLIC_KEY.test(search)) {
+ throw BMAConstants.ERRORS.NO_IDTY_MATCHING_PUB_OR_UID;
+ }
+ identities = await this.IdentityService.searchIdentitiesByPubkey(search);
+ } else {
+ identities = await this.IdentityService.searchIdentities(search);
+ }
const all: HttpIdentityRequirement[] = await this.BlockchainService.requirementsOfIdentities(
identities
);
@@ -229,9 +244,16 @@ export class WOTBinding extends AbstractController {
async certifiedBy(req: any): Promise<HttpCertifications> {
const search = await ParametersService.getSearchP(req);
- const idty = (await this.server.dal.getWrittenIdtyByPubkeyOrUIdForHashingAndIsMember(
- search
- )) as FullIindexEntry;
+ let idty: FullIindexEntry;
+ if (ParametersService.getIsPubkey(req)) {
+ idty = (await this.server.dal.getWrittenIdtyByPubkeyForHashingAndIsMember(
+ search
+ )) as FullIindexEntry;
+ } else {
+ idty = (await this.server.dal.getWrittenIdtyByPubkeyOrUIdForHashingAndIsMember(
+ search
+ )) as FullIindexEntry;
+ }
const certs = await this.server.dal.certsFrom(idty.pub);
const theCerts: HttpCertification[] = [];
for (const cert of certs) {
diff --git a/app/modules/bma/lib/parameters.ts b/app/modules/bma/lib/parameters.ts
index 49a437f26..c5463ecd9 100644
--- a/app/modules/bma/lib/parameters.ts
+++ b/app/modules/bma/lib/parameters.ts
@@ -96,6 +96,16 @@ export class ParametersService {
callback(null, matches[0]);
};
+ static getIsPubkey(req: any): boolean {
+ const value = req.query.pubkey;
+ return (
+ value !== null &&
+ value !== undefined &&
+ value !== "false" &&
+ value !== false
+ );
+ }
+
static getPubkeyP(req: any) {
return Q.nbind(ParametersService.getPubkey, this)(req);
}
diff --git a/app/service/IdentityService.ts b/app/service/IdentityService.ts
index 676b8602a..f3173f885 100644
--- a/app/service/IdentityService.ts
+++ b/app/service/IdentityService.ts
@@ -54,6 +54,10 @@ export class IdentityService extends FIFOService {
return this.dal.searchJustIdentities(search);
}
+ searchIdentitiesByPubkey(pubkey: string) {
+ return this.dal.searchJustIdentitiesByPubkey(pubkey);
+ }
+
async findMember(search: string) {
let idty = null;
if (search.match(constants.PUBLIC_KEY)) {
diff --git a/test/integration/identity/identity-test.ts b/test/integration/identity/identity-test.ts
index 6966e9971..2a60a7ffb 100644
--- a/test/integration/identity/identity-test.ts
+++ b/test/integration/identity/identity-test.ts
@@ -192,6 +192,13 @@ describe("Identities collision", function() {
});
});
+ it('should have certifiers-of/:pubkey of cat giving results', function() {
+ return expectAnswer(rp('http://127.0.0.1:7799/wot/certifiers-of/HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd?pubkey', { json: true }), function(res:HttpCertifications) {
+ res.should.have.property('pubkey').equal('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+ res.should.have.property('uid').equal('cat');
+ });
+ });
+
it('should have certifiers-of/tic giving results', function() {
return expectAnswer(rp('http://127.0.0.1:7799/wot/certifiers-of/tic', { json: true }), function(res:HttpCertifications) {
res.should.have.property('pubkey').equal('DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV');
@@ -213,6 +220,13 @@ describe("Identities collision", function() {
});
});
+ it('should have certifiers-of/:pubkey of tic giving results', function() {
+ return expectAnswer(rp('http://127.0.0.1:7799/wot/certifiers-of/DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV?pubkey', { json: true }), function(res:HttpCertifications) {
+ res.should.have.property('pubkey').equal('DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV');
+ res.should.have.property('uid').equal('tic');
+ });
+ });
+
it('should have certifiers-of/toc giving results', function() {
return expectAnswer(rp('http://127.0.0.1:7799/wot/certifiers-of/toc', { json: true }), function(res:HttpCertifications) {
res.should.have.property('pubkey').equal('DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo');
@@ -234,6 +248,13 @@ describe("Identities collision", function() {
});
});
+ it('should have certifiers-of/:pubkey of toc giving results', function() {
+ return expectAnswer(rp('http://127.0.0.1:7799/wot/certifiers-of/DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo?pubkey', { json: true }), function(res:HttpCertifications) {
+ res.should.have.property('pubkey').equal('DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo');
+ res.should.have.property('uid').equal('toc');
+ });
+ });
+
it('requirements of cat', function() {
return expectAnswer(rp('http://127.0.0.1:7799/wot/requirements/cat', { json: true }), function(res:HttpRequirements) {
res.should.have.property('identities').be.an.Array;
@@ -270,6 +291,46 @@ describe("Identities collision", function() {
});
});
+ it('requirements by pubkey of cat', function() {
+ return expectAnswer(rp('http://127.0.0.1:7799/wot/requirements/HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd?pubkey', { json: true }), function(res:HttpRequirements) {
+ res.should.have.property('identities').be.an.Array;
+ res.should.have.property('identities').have.length(1);
+ res.identities[0].should.have.property('pubkey').equal('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+ res.identities[0].should.have.property('uid').equal('cat');
+ res.identities[0].should.have.property('meta').property('timestamp');
+ res.identities[0].should.have.property('wasMember').equal(true);
+ res.identities[0].should.have.property('expired').equal(false); // Because it has been a member once! So its identity will exist forever.
+ res.identities[0].should.have.property('outdistanced').equal(false);
+ res.identities[0].should.have.property('isSentry').equal(true); // dSen = 2, cat has issued and received 2 certs with tic and toc
+ res.identities[0].should.have.property('certifications').have.length(2);
+ res.identities[0].should.have.property('membershipPendingExpiresIn').equal(0);
+ res.identities[0].should.have.property('membershipExpiresIn').greaterThan(9000);
+ });
+ });
+
+ it('requirements by pubkey of man1', function() {
+ return expectAnswer(rp('http://127.0.0.1:7799/wot/requirements/12AbjvYY5hxV4v2KrN9pnGzgFxogwrzgYyncYHHsyFDK?pubkey', { json: true }), function(res:HttpRequirements) {
+ res.should.have.property('identities').be.an.Array;
+ res.should.have.property('identities').have.length(1);
+ res.identities[0].should.have.property('pubkey').equal('12AbjvYY5hxV4v2KrN9pnGzgFxogwrzgYyncYHHsyFDK');
+ res.identities[0].should.have.property('uid').equal('man1');
+ res.identities[0].should.have.property('meta').property('timestamp');
+ res.identities[0].should.have.property('expired').equal(false);
+ res.identities[0].should.have.property('outdistanced').equal(false);
+ res.identities[0].should.have.property('isSentry').equal(false); // Not a member, also dSen = 2, but man1 has only 1 certification
+ res.identities[0].should.have.property('certifications').length(1);
+ res.identities[0].certifications[0].should.have.property('from').equal('2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc');
+ res.identities[0].certifications[0].should.have.property('to').equal('12AbjvYY5hxV4v2KrN9pnGzgFxogwrzgYyncYHHsyFDK');
+ res.identities[0].certifications[0].should.have.property('expiresIn').greaterThan(0);
+ res.identities[0].should.have.property('membershipPendingExpiresIn').greaterThan(9000);
+ res.identities[0].should.have.property('membershipExpiresIn').equal(0);
+ });
+ });
+
+ it('requirements by invalid pubkey', function() {
+ return expectError(404, "No identity matching this pubkey or uid", rp('http://127.0.0.1:7799/wot/requirements/cat?pubkey', { json: true }));
+ });
+
it('should have certified-by/tic giving results', function() {
return expectAnswer(rp('http://127.0.0.1:7799/wot/certified-by/tic', { json: true }), function(res:HttpCertifications) {
res.should.have.property('pubkey').equal('DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV');
@@ -298,6 +359,13 @@ describe("Identities collision", function() {
});
});
+ it('should have certified-by/:pubkey of tic giving results', function() {
+ return expectAnswer(rp('http://127.0.0.1:7799/wot/certified-by/DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV?pubkey', { json: true }), function(res:HttpCertifications) {
+ res.should.have.property('pubkey').equal('DNann1Lh55eZMEDXeYt59bzHbA3NJR46DeQYCS2qQdLV');
+ res.should.have.property('uid').equal('tic');
+ });
+ });
+
it('should have certified-by/tac giving results', function() {
return expectAnswer(rp('http://127.0.0.1:7799/wot/certified-by/tac', { json: true }), function(res:HttpCertifications) {
res.should.have.property('pubkey').equal('2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc');
@@ -308,6 +376,13 @@ describe("Identities collision", function() {
});
});
+ it('should have certified-by/:pubkey of tac giving results', function() {
+ return expectAnswer(rp('http://127.0.0.1:7799/wot/certified-by/2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc?pubkey', { json: true }), function(res:HttpCertifications) {
+ res.should.have.property('pubkey').equal('2LvDg21dVXvetTD9GdkPLURavLYEqP3whauvPWX4c2qc');
+ res.should.have.property('uid').equal('tac');
+ });
+ });
+
it('should have certified-by/cat giving results', function() {
return expectAnswer(rp('http://127.0.0.1:7799/wot/certified-by/cat', { json: true }), function(res:HttpCertifications) {
res.should.have.property('pubkey').equal('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
@@ -343,6 +418,13 @@ describe("Identities collision", function() {
});
});
+ it('should have certified-by/:pubkey of cat giving results', function() {
+ return expectAnswer(rp('http://127.0.0.1:7799/wot/certified-by/HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd?pubkey', { json: true }), function(res:HttpCertifications) {
+ res.should.have.property('pubkey').equal('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd');
+ res.should.have.property('uid').equal('cat');
+ });
+ });
+
it('requirements of man2', function() {
return expectAnswer(rp('http://127.0.0.1:7799/wot/requirements/man2', { json: true }), function(res:HttpRequirements) {
res.should.have.property('identities').be.an.Array;
--
GitLab