fix(1447): Make sure to clear both CREATE and UPDATE records, when trimming...

fix(1447): Make sure to clear both CREATE and UPDATE records, when trimming indexForTriming - close #1447

app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts

@@ -175,8 +175,9 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
     let belowNumberIds: string[] = [];
     const mapIds: {
       [k: string]: {
+        createdOn: number;
+        updatedOn: number;
         conditions: string;
-        writtenOn: number;
       };
     } = {};
     const mapIds2WrittenOn: { [k: string]: number } = {};
@@ -199,17 +200,19 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
       // Remove consumed sources
       const identifier = id.split("-")[0];
       const pos = pint(id.split("-")[1]);
-      const entry = await this.getOrNull(
-        LevelDBSindex.trimKey(identifier, pos, true)
-      );
-      if (entry && entry.writtenOn < belowNumber) {
+      const createKey = LevelDBSindex.trimKey(identifier, pos, false);
+      const createRecord = await this.getOrNull(createKey);
+      const updateKey = LevelDBSindex.trimKey(identifier, pos, true);
+      const updateRecord = await this.getOrNull(updateKey);
+      if (createRecord && updateRecord && updateRecord.writtenOn < belowNumber) {
         // We remember the trimmed source id to remove it from the writtenOn and conditions index
         mapIds[id] = {
-          writtenOn: mapIds2WrittenOn[id],
-          conditions: entry.conditions,
+          createdOn: createRecord.writtenOn,
+          updatedOn: updateRecord.writtenOn,
+          conditions: updateRecord.conditions,
         };
-        await this.del(LevelDBSindex.trimKey(identifier, pos, false));
-        await this.del(LevelDBSindex.trimKey(identifier, pos, true));
+        await this.del(createKey);
+        await this.del(updateKey);
       }
     }
 
@@ -217,8 +220,9 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
     for (const id of Underscore.keys(mapIds).map(String)) {
       const map = mapIds[id];
       await this.trimConditions(map.conditions, id);
-      await this.trimConsumed(map.writtenOn, id);
-      await this.trimWrittenOn(map.writtenOn, id);
+      await this.trimConsumed(map.updatedOn, id);
+      await this.trimWrittenOn(map.createdOn, id);
+      await this.trimWrittenOn(map.updatedOn, id);
     }
   }
 

test/dal/triming-dal.ts

@@ -120,8 +120,8 @@ describe("Triming", function(){
     await dal.sindexDAL.insertBatch([
       { op: 'CREATE', identifier: 'SOURCE_1', pos: 4, written_on: '126-H', writtenOn: 126, written_time: 2000, consumed: false, conditions: 'COND(SOURCE_1)'},
       { op: 'UPDATE', identifier: 'SOURCE_1', pos: 4, written_on: '139-H', writtenOn: 139, written_time: 4500, consumed: true, conditions: 'COND(SOURCE_1)'},
-      { op: 'CREATE', identifier: 'SOURCE_2', pos: 4, written_on: '126-H', writtenOn: 126, written_time: 2000, consumed: false, conditions: 'COND(SOURCE_2)'},
-      { op: 'CREATE', identifier: 'SOURCE_3', pos: 4, written_on: '126-H', writtenOn: 126, written_time: 2000, consumed: false, conditions: 'SIG(PUB_1)'}
+      { op: 'CREATE', identifier: 'SOURCE_2', pos: 4, written_on: '127-H', writtenOn: 127, written_time: 2500, consumed: false, conditions: 'COND(SOURCE_2)'},
+      { op: 'CREATE', identifier: 'SOURCE_3', pos: 4, written_on: '127-H', writtenOn: 127, written_time: 2500, consumed: false, conditions: 'SIG(PUB_1)'}
     ] as any);
     (await dal.sindexDAL.findByIdentifier('SOURCE_1')).should.have.length(2);
     (await dal.sindexDAL.getAvailableForConditions('COND(SOURCE_2)')).should.have.length(1);
@@ -142,11 +142,12 @@ describe("Triming", function(){
 
     // Check internal index
     // FIXME another issue here
-    /*for (let index of sindexDAL.getInternalIndexes()) {
-      const res = await index.findAllKeys();
-      res.should.not.containEql("COND(SOURCE_1)")
-      res.should.not.containEql("0000000126")
-    }*/
+    for (let index of sindexDAL.getInternalIndexes()) {
+      const keys = await index.findAllKeys();
+      keys.should.not.containEql("COND(SOURCE_1)");
+      keys.should.not.containEql("0000000126");
+      keys.should.not.containEql("0000000139");
+    }
 
     // Now we consume all sources
     await dal.sindexDAL.insertBatch([