[fix] Report changes from merge request !1420 for release/1.8 - close #1438

app/lib/common-libs/constants.ts

@@ -53,7 +53,7 @@ const CONDITIONS =
   "\\)|CSV\\(" +
   CSV_INTEGER +
   "\\))))*";
-
+const CONDITION_SIG_PUBKEY = "SIG\\((" + PUBKEY + ")\\)";
 const BMA_REGEXP = /^BASIC_MERKLED_API( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/;
 const BMAS_REGEXP = /^BMAS( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))( (\/.+))?$/;
 const BMATOR_REGEXP = /^BMATOR( ([a-z0-9]{16})\.onion)( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/;
@@ -533,6 +533,8 @@ export const CommonConstants = {
     LOCKTIME: find("Locktime: (" + INTEGER + ")"),
     INLINE_COMMENT: exact(COMMENT),
     OUTPUT_CONDITION: exact(CONDITIONS),
+    OUTPUT_CONDITION_SIG_PUBKEY: find(CONDITION_SIG_PUBKEY),
+    OUTPUT_CONDITION_SIG_PUBKEY_UNIQUE: exact(CONDITION_SIG_PUBKEY),
   },
   PEER: {
     BLOCK: find("Block: (" + INTEGER + "-" + FINGERPRINT + ")"),

app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts

@@ -12,12 +12,14 @@ import { SIndexDAO } from "../abstract/SIndexDAO";
 import { Underscore } from "../../../common-libs/underscore";
 import { pint } from "../../../common-libs/pint";
 import { arrayPruneAllCopy } from "../../../common-libs/array-prune";
+import { CommonConstants } from "../../../common-libs/constants";
 
 export class LevelDBSindex extends LevelDBTable<SindexEntry>
   implements SIndexDAO {
   private indexForTrimming: LevelDBTable<string[]>;
   private indexForConsumed: LevelDBTable<string[]>;
   private indexForConditions: LevelDBTable<string[]>;
+  private indexOfComplexeConditionForPubkeys: LevelDBTable<string[]>;
 
   constructor(protected getLevelDB: (dbName: string) => Promise<LevelUp>) {
     super("level_sindex", getLevelDB);
@@ -41,9 +43,14 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
       "level_sindex/conditions",
       this.getLevelDB
     );
+    this.indexOfComplexeConditionForPubkeys = new LevelDBTable<string[]>(
+      "level_sindex/complex_condition_pubkeys",
+      this.getLevelDB
+    );
     await this.indexForTrimming.init();
     await this.indexForConsumed.init();
     await this.indexForConditions.init();
+    await this.indexOfComplexeConditionForPubkeys.init();
   }
 
   async close(): Promise<void> {
@@ -51,6 +58,7 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
     await this.indexForTrimming.close();
     await this.indexForConsumed.close();
     await this.indexForConditions.close();
+    await this.indexOfComplexeConditionForPubkeys.close();
   }
 
   /**
@@ -127,14 +135,14 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
       pos: number;
     }[]
   > {
-    // TODO: very costly: needs a full scan, would be better to change this implementatio
-    const entries = await this.findWhere((e) =>
-      e.conditions.includes(`SIG(${pubkey})`)
+    const forSimpleConditions = await this.getForConditions(`SIG(${pubkey})`);
+    const forComplexConditions = await this.getForComplexeConditionPubkey(
+      pubkey
+    );
+    const reduced = Indexer.DUP_HELPERS.reduceBy(
+      forSimpleConditions.concat(forComplexConditions),
+      ["identifier", "pos"]
     );
-    const reduced = Indexer.DUP_HELPERS.reduceBy(entries, [
-      "identifier",
-      "pos",
-    ]);
     return reduced.filter((r) => !r.consumed);
   }
 
@@ -269,6 +277,20 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
     return found;
   }
 
+  async getForComplexeConditionPubkey(pubkey: string): Promise<SindexEntry[]> {
+    const ids =
+      (await this.indexOfComplexeConditionForPubkeys.getOrNull(pubkey)) || [];
+    const found: SindexEntry[] = [];
+    for (const id of ids) {
+      const entries = await this.findByIdentifierAndPos(
+        id.split("-")[0],
+        pint(id.split("-")[1])
+      );
+      entries.forEach((e) => found.push(e));
+    }
+    return found;
+  }
+
   async removeBlock(blockstamp: string): Promise<void> {
     const writtenOn = pint(blockstamp);
     // We look at records written on this blockstamp: `indexForTrimming` allows to get them
@@ -316,24 +338,25 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
   }
 
   private async trimConditions(condition: string, id: string) {
-    // Get all the account's TX sources
+    // Get all the condition's sources
     const existing = (await this.indexForConditions.getOrNull(condition)) || [];
-    // Prune the source from the account
+    // Prune the source from the condition
     const trimmed = arrayPruneAllCopy(existing, id);
     if (trimmed.length) {
-      // If some sources are left for this "account", persist what remains
+      // If some sources are left for this "condition", persist what remains
       await this.indexForConditions.put(condition, trimmed);
     } else {
       // Otherwise just delete the "account"
       await this.indexForConditions.del(condition);
     }
+
+    // If complex conditions
+    if (this.isComplexCondition(condition)) {
+      const pubkeys = this.getDistinctPubkeysFromCondition(condition);
+      await this.trimComplexeConditionPubkeys(pubkeys, id);
+    }
   }
 
-  /**
-   * Duplicate with trimConditions?!
-   * @param writtenOn
-   * @param id
-   */
   private async trimWrittenOn(writtenOn: number, id: string) {
     const k = LevelDBSindex.trimWrittenOnKey(writtenOn);
     const existing = await this.getWrittenOnSourceIds(writtenOn);
@@ -356,6 +379,28 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
     }
   }
 
+  private async trimComplexeConditionPubkeys(pubkeys: string[], id: string) {
+    if (!pubkeys || !pubkeys.length) return;
+    for (const p of pubkeys) {
+      await this.trimComplexeConditionPubkey(p, id);
+    }
+  }
+
+  private async trimComplexeConditionPubkey(pubkey: string, id: string) {
+    // Get all the condition's sources
+    const existing =
+      (await this.indexOfComplexeConditionForPubkeys.getOrNull(pubkey)) || [];
+    // Prune the source from the condition
+    const trimmed = arrayPruneAllCopy(existing, id);
+    if (trimmed.length) {
+      // If some sources are left for this "condition", persist what remains
+      await this.indexOfComplexeConditionForPubkeys.put(pubkey, trimmed);
+    } else {
+      // Otherwise just delete the "account"
+      await this.indexOfComplexeConditionForPubkeys.del(pubkey);
+    }
+  }
+
   private async getWrittenOnSourceIds(writtenOn: number) {
     const indexForTrimmingId = LevelDBSindex.trimWrittenOnKey(writtenOn);
     return (await this.indexForTrimming.getOrNull(indexForTrimmingId)) || [];
@@ -393,6 +438,7 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
     const byConsumed: { [k: number]: SindexEntry[] } = {};
     const byWrittenOn: { [k: number]: SindexEntry[] } = {};
     const byConditions: { [k: string]: SindexEntry[] } = {};
+    const byPubkeys: { [k: string]: SindexEntry[] } = {};
     records
       .filter((r) => r.consumed)
       .forEach((r) => {
@@ -410,12 +456,24 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
         arrWO = byWrittenOn[r.writtenOn] = [];
       }
       arrWO.push(r);
-      // Conditiosn
+      // Conditions
       let arrCN = byConditions[r.conditions];
       if (!arrCN) {
         arrCN = byConditions[r.conditions] = [];
       }
       arrCN.push(r);
+
+      // If complex condition
+      if (this.isComplexCondition(r.conditions)) {
+        const pubkeys = this.getDistinctPubkeysFromCondition(r.conditions);
+        pubkeys.forEach((pub) => {
+          let arrPub = byPubkeys[pub];
+          if (!arrPub) {
+            arrPub = byPubkeys[pub] = [];
+          }
+          arrPub.push(r);
+        });
+      }
     });
     // Index consumed => (identifier + pos)[]
     for (const k of Underscore.keys(byConsumed)) {
@@ -446,5 +504,47 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
         Underscore.uniq(existing.concat(newSources))
       );
     }
+    // Index pubkeys => (identifier + pos)[]
+    for (const k of Underscore.keys(byPubkeys).map(String)) {
+      const existing =
+        (await this.indexOfComplexeConditionForPubkeys.getOrNull(k)) || [];
+      const newSources = byPubkeys[k].map((r) =>
+        LevelDBSindex.trimPartialKey(r.identifier, r.pos)
+      );
+      await this.indexOfComplexeConditionForPubkeys.put(
+        k,
+        Underscore.uniq(existing.concat(newSources))
+      );
+    }
+  }
+
+  private isComplexCondition(condition: string): boolean {
+    return (
+      (condition &&
+        !CommonConstants.TRANSACTION.OUTPUT_CONDITION_SIG_PUBKEY_UNIQUE.test(
+          condition
+        )) ||
+      false
+    );
+  }
+  /**
+   * Get all pubkeys used by an output condition (e.g. 'SIG(A) && SIG(B)' will return ['A', 'B']
+   * @param condition
+   * @private
+   */
+  private getDistinctPubkeysFromCondition(condition: string): string[] {
+    const pubKeys: string[] = [];
+    if (!condition) return pubKeys;
+    let match: RegExpExecArray | null;
+    while (
+      (match = CommonConstants.TRANSACTION.OUTPUT_CONDITION_SIG_PUBKEY.exec(
+        condition
+      )) !== null
+    ) {
+      pubKeys.push(match[1]);
+      condition = condition.substring(match.index + match[0].length);
+    }
+
+    return Underscore.uniq(pubKeys);
   }
 }