From d6eb97ca954298a7ca5a43368e5768e282742d51 Mon Sep 17 00:00:00 2001
From: Benoit Lavenier <benoit.lavenier@e-is.pro>
Date: Wed, 31 May 2023 13:28:58 +0200
Subject: [PATCH] [fix] Report changes from merge request !1420 for release/1.8
- close #1438
---
app/lib/common-libs/constants.ts | 4 +-
app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts | 132 +++++++++++++++---
2 files changed, 119 insertions(+), 17 deletions(-)
diff --git a/app/lib/common-libs/constants.ts b/app/lib/common-libs/constants.ts
index a0b6151cf..0cf4179ea 100755
--- a/app/lib/common-libs/constants.ts
+++ b/app/lib/common-libs/constants.ts
@@ -53,7 +53,7 @@ const CONDITIONS =
"\\)|CSV\\(" +
CSV_INTEGER +
"\\))))*";
-
+const CONDITION_SIG_PUBKEY = "SIG\\((" + PUBKEY + ")\\)";
const BMA_REGEXP = /^BASIC_MERKLED_API( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/;
const BMAS_REGEXP = /^BMAS( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))( (\/.+))?$/;
const BMATOR_REGEXP = /^BMATOR( ([a-z0-9]{16})\.onion)( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/;
@@ -533,6 +533,8 @@ export const CommonConstants = {
LOCKTIME: find("Locktime: (" + INTEGER + ")"),
INLINE_COMMENT: exact(COMMENT),
OUTPUT_CONDITION: exact(CONDITIONS),
+ OUTPUT_CONDITION_SIG_PUBKEY: find(CONDITION_SIG_PUBKEY),
+ OUTPUT_CONDITION_SIG_PUBKEY_UNIQUE: exact(CONDITION_SIG_PUBKEY),
},
PEER: {
BLOCK: find("Block: (" + INTEGER + "-" + FINGERPRINT + ")"),
diff --git a/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts b/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts
index a8b4aceba..f81e4f9d2 100644
--- a/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts
+++ b/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts
@@ -12,12 +12,14 @@ import { SIndexDAO } from "../abstract/SIndexDAO";
import { Underscore } from "../../../common-libs/underscore";
import { pint } from "../../../common-libs/pint";
import { arrayPruneAllCopy } from "../../../common-libs/array-prune";
+import { CommonConstants } from "../../../common-libs/constants";
export class LevelDBSindex extends LevelDBTable<SindexEntry>
implements SIndexDAO {
private indexForTrimming: LevelDBTable<string[]>;
private indexForConsumed: LevelDBTable<string[]>;
private indexForConditions: LevelDBTable<string[]>;
+ private indexOfComplexeConditionForPubkeys: LevelDBTable<string[]>;
constructor(protected getLevelDB: (dbName: string) => Promise<LevelUp>) {
super("level_sindex", getLevelDB);
@@ -41,9 +43,14 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
"level_sindex/conditions",
this.getLevelDB
);
+ this.indexOfComplexeConditionForPubkeys = new LevelDBTable<string[]>(
+ "level_sindex/complex_condition_pubkeys",
+ this.getLevelDB
+ );
await this.indexForTrimming.init();
await this.indexForConsumed.init();
await this.indexForConditions.init();
+ await this.indexOfComplexeConditionForPubkeys.init();
}
async close(): Promise<void> {
@@ -51,6 +58,7 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
await this.indexForTrimming.close();
await this.indexForConsumed.close();
await this.indexForConditions.close();
+ await this.indexOfComplexeConditionForPubkeys.close();
}
/**
@@ -127,14 +135,14 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
pos: number;
}[]
> {
- // TODO: very costly: needs a full scan, would be better to change this implementatio
- const entries = await this.findWhere((e) =>
- e.conditions.includes(`SIG(${pubkey})`)
+ const forSimpleConditions = await this.getForConditions(`SIG(${pubkey})`);
+ const forComplexConditions = await this.getForComplexeConditionPubkey(
+ pubkey
+ );
+ const reduced = Indexer.DUP_HELPERS.reduceBy(
+ forSimpleConditions.concat(forComplexConditions),
+ ["identifier", "pos"]
);
- const reduced = Indexer.DUP_HELPERS.reduceBy(entries, [
- "identifier",
- "pos",
- ]);
return reduced.filter((r) => !r.consumed);
}
@@ -269,6 +277,20 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
return found;
}
+ async getForComplexeConditionPubkey(pubkey: string): Promise<SindexEntry[]> {
+ const ids =
+ (await this.indexOfComplexeConditionForPubkeys.getOrNull(pubkey)) || [];
+ const found: SindexEntry[] = [];
+ for (const id of ids) {
+ const entries = await this.findByIdentifierAndPos(
+ id.split("-")[0],
+ pint(id.split("-")[1])
+ );
+ entries.forEach((e) => found.push(e));
+ }
+ return found;
+ }
+
async removeBlock(blockstamp: string): Promise<void> {
const writtenOn = pint(blockstamp);
// We look at records written on this blockstamp: `indexForTrimming` allows to get them
@@ -316,24 +338,25 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
}
private async trimConditions(condition: string, id: string) {
- // Get all the account's TX sources
+ // Get all the condition's sources
const existing = (await this.indexForConditions.getOrNull(condition)) || [];
- // Prune the source from the account
+ // Prune the source from the condition
const trimmed = arrayPruneAllCopy(existing, id);
if (trimmed.length) {
- // If some sources are left for this "account", persist what remains
+ // If some sources are left for this "condition", persist what remains
await this.indexForConditions.put(condition, trimmed);
} else {
// Otherwise just delete the "account"
await this.indexForConditions.del(condition);
}
+
+ // If complex conditions
+ if (this.isComplexCondition(condition)) {
+ const pubkeys = this.getDistinctPubkeysFromCondition(condition);
+ await this.trimComplexeConditionPubkeys(pubkeys, id);
+ }
}
- /**
- * Duplicate with trimConditions?!
- * @param writtenOn
- * @param id
- */
private async trimWrittenOn(writtenOn: number, id: string) {
const k = LevelDBSindex.trimWrittenOnKey(writtenOn);
const existing = await this.getWrittenOnSourceIds(writtenOn);
@@ -356,6 +379,28 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
}
}
+ private async trimComplexeConditionPubkeys(pubkeys: string[], id: string) {
+ if (!pubkeys || !pubkeys.length) return;
+ for (const p of pubkeys) {
+ await this.trimComplexeConditionPubkey(p, id);
+ }
+ }
+
+ private async trimComplexeConditionPubkey(pubkey: string, id: string) {
+ // Get all the condition's sources
+ const existing =
+ (await this.indexOfComplexeConditionForPubkeys.getOrNull(pubkey)) || [];
+ // Prune the source from the condition
+ const trimmed = arrayPruneAllCopy(existing, id);
+ if (trimmed.length) {
+ // If some sources are left for this "condition", persist what remains
+ await this.indexOfComplexeConditionForPubkeys.put(pubkey, trimmed);
+ } else {
+ // Otherwise just delete the "account"
+ await this.indexOfComplexeConditionForPubkeys.del(pubkey);
+ }
+ }
+
private async getWrittenOnSourceIds(writtenOn: number) {
const indexForTrimmingId = LevelDBSindex.trimWrittenOnKey(writtenOn);
return (await this.indexForTrimming.getOrNull(indexForTrimmingId)) || [];
@@ -393,6 +438,7 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
const byConsumed: { [k: number]: SindexEntry[] } = {};
const byWrittenOn: { [k: number]: SindexEntry[] } = {};
const byConditions: { [k: string]: SindexEntry[] } = {};
+ const byPubkeys: { [k: string]: SindexEntry[] } = {};
records
.filter((r) => r.consumed)
.forEach((r) => {
@@ -410,12 +456,24 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
arrWO = byWrittenOn[r.writtenOn] = [];
}
arrWO.push(r);
- // Conditiosn
+ // Conditions
let arrCN = byConditions[r.conditions];
if (!arrCN) {
arrCN = byConditions[r.conditions] = [];
}
arrCN.push(r);
+
+ // If complex condition
+ if (this.isComplexCondition(r.conditions)) {
+ const pubkeys = this.getDistinctPubkeysFromCondition(r.conditions);
+ pubkeys.forEach((pub) => {
+ let arrPub = byPubkeys[pub];
+ if (!arrPub) {
+ arrPub = byPubkeys[pub] = [];
+ }
+ arrPub.push(r);
+ });
+ }
});
// Index consumed => (identifier + pos)[]
for (const k of Underscore.keys(byConsumed)) {
@@ -446,5 +504,47 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
Underscore.uniq(existing.concat(newSources))
);
}
+ // Index pubkeys => (identifier + pos)[]
+ for (const k of Underscore.keys(byPubkeys).map(String)) {
+ const existing =
+ (await this.indexOfComplexeConditionForPubkeys.getOrNull(k)) || [];
+ const newSources = byPubkeys[k].map((r) =>
+ LevelDBSindex.trimPartialKey(r.identifier, r.pos)
+ );
+ await this.indexOfComplexeConditionForPubkeys.put(
+ k,
+ Underscore.uniq(existing.concat(newSources))
+ );
+ }
+ }
+
+ private isComplexCondition(condition: string): boolean {
+ return (
+ (condition &&
+ !CommonConstants.TRANSACTION.OUTPUT_CONDITION_SIG_PUBKEY_UNIQUE.test(
+ condition
+ )) ||
+ false
+ );
+ }
+ /**
+ * Get all pubkeys used by an output condition (e.g. 'SIG(A) && SIG(B)' will return ['A', 'B']
+ * @param condition
+ * @private
+ */
+ private getDistinctPubkeysFromCondition(condition: string): string[] {
+ const pubKeys: string[] = [];
+ if (!condition) return pubKeys;
+ let match: RegExpExecArray | null;
+ while (
+ (match = CommonConstants.TRANSACTION.OUTPUT_CONDITION_SIG_PUBKEY.exec(
+ condition
+ )) !== null
+ ) {
+ pubKeys.push(match[1]);
+ condition = condition.substring(match.index + match[0].length);
+ }
+
+ return Underscore.uniq(pubKeys);
}
}
--
GitLab