From 67b05f1a165d78c5bb635e86726e4c0d1e713929 Mon Sep 17 00:00:00 2001
From: Benoit Lavenier <benoit.lavenier@e-is.pro>
Date: Fri, 9 Jun 2023 11:01:12 +0200
Subject: [PATCH] fix(1447): Make sure to clear both CREATE and UPDATE records,
when trimming indexForTriming - close #1447
---
app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts | 26 +++++++++++--------
test/dal/triming-dal.ts | 15 ++++++-----
2 files changed, 23 insertions(+), 18 deletions(-)
diff --git a/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts b/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts
index bdb046713..dfb026f42 100644
--- a/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts
+++ b/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts
@@ -175,8 +175,9 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
let belowNumberIds: string[] = [];
const mapIds: {
[k: string]: {
+ createdOn: number;
+ updatedOn: number;
conditions: string;
- writtenOn: number;
};
} = {};
const mapIds2WrittenOn: { [k: string]: number } = {};
@@ -199,17 +200,19 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
// Remove consumed sources
const identifier = id.split("-")[0];
const pos = pint(id.split("-")[1]);
- const entry = await this.getOrNull(
- LevelDBSindex.trimKey(identifier, pos, true)
- );
- if (entry && entry.writtenOn < belowNumber) {
+ const createKey = LevelDBSindex.trimKey(identifier, pos, false);
+ const createRecord = await this.getOrNull(createKey);
+ const updateKey = LevelDBSindex.trimKey(identifier, pos, true);
+ const updateRecord = await this.getOrNull(updateKey);
+ if (createRecord && updateRecord && updateRecord.writtenOn < belowNumber) {
// We remember the trimmed source id to remove it from the writtenOn and conditions index
mapIds[id] = {
- writtenOn: mapIds2WrittenOn[id],
- conditions: entry.conditions,
+ createdOn: createRecord.writtenOn,
+ updatedOn: updateRecord.writtenOn,
+ conditions: updateRecord.conditions,
};
- await this.del(LevelDBSindex.trimKey(identifier, pos, false));
- await this.del(LevelDBSindex.trimKey(identifier, pos, true));
+ await this.del(createKey);
+ await this.del(updateKey);
}
}
@@ -217,8 +220,9 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
for (const id of Underscore.keys(mapIds).map(String)) {
const map = mapIds[id];
await this.trimConditions(map.conditions, id);
- await this.trimConsumed(map.writtenOn, id);
- await this.trimWrittenOn(map.writtenOn, id);
+ await this.trimConsumed(map.updatedOn, id);
+ await this.trimWrittenOn(map.createdOn, id);
+ await this.trimWrittenOn(map.updatedOn, id);
}
}
diff --git a/test/dal/triming-dal.ts b/test/dal/triming-dal.ts
index 76a7dd0aa..535dcff85 100644
--- a/test/dal/triming-dal.ts
+++ b/test/dal/triming-dal.ts
@@ -120,8 +120,8 @@ describe("Triming", function(){
await dal.sindexDAL.insertBatch([
{ op: 'CREATE', identifier: 'SOURCE_1', pos: 4, written_on: '126-H', writtenOn: 126, written_time: 2000, consumed: false, conditions: 'COND(SOURCE_1)'},
{ op: 'UPDATE', identifier: 'SOURCE_1', pos: 4, written_on: '139-H', writtenOn: 139, written_time: 4500, consumed: true, conditions: 'COND(SOURCE_1)'},
- { op: 'CREATE', identifier: 'SOURCE_2', pos: 4, written_on: '126-H', writtenOn: 126, written_time: 2000, consumed: false, conditions: 'COND(SOURCE_2)'},
- { op: 'CREATE', identifier: 'SOURCE_3', pos: 4, written_on: '126-H', writtenOn: 126, written_time: 2000, consumed: false, conditions: 'SIG(PUB_1)'}
+ { op: 'CREATE', identifier: 'SOURCE_2', pos: 4, written_on: '127-H', writtenOn: 127, written_time: 2500, consumed: false, conditions: 'COND(SOURCE_2)'},
+ { op: 'CREATE', identifier: 'SOURCE_3', pos: 4, written_on: '127-H', writtenOn: 127, written_time: 2500, consumed: false, conditions: 'SIG(PUB_1)'}
] as any);
(await dal.sindexDAL.findByIdentifier('SOURCE_1')).should.have.length(2);
(await dal.sindexDAL.getAvailableForConditions('COND(SOURCE_2)')).should.have.length(1);
@@ -142,11 +142,12 @@ describe("Triming", function(){
// Check internal index
// FIXME another issue here
- /*for (let index of sindexDAL.getInternalIndexes()) {
- const res = await index.findAllKeys();
- res.should.not.containEql("COND(SOURCE_1)")
- res.should.not.containEql("0000000126")
- }*/
+ for (let index of sindexDAL.getInternalIndexes()) {
+ const keys = await index.findAllKeys();
+ keys.should.not.containEql("COND(SOURCE_1)");
+ keys.should.not.containEql("0000000126");
+ keys.should.not.containEql("0000000139");
+ }
// Now we consume all sources
await dal.sindexDAL.insertBatch([
--
GitLab