diff --git a/app/lib/blockchain/DuniterBlockchain.ts b/app/lib/blockchain/DuniterBlockchain.ts
index 063a095053c85b91d8b9495532c5bdde7c93e24b..81603eb1b8d90bc6167ef36c11a8aef5b6607f25 100644
--- a/app/lib/blockchain/DuniterBlockchain.ts
+++ b/app/lib/blockchain/DuniterBlockchain.ts
@@ -319,14 +319,21 @@ export class DuniterBlockchain {
await this.removeMembershipsFromSandbox(block, dal);
// Compute to be revoked members
await this.computeToBeRevoked(indexes.mindex, dal);
+
// Delete eventually present transactions
- await this.deleteTransactions(block, dal);
+ // NOT NEED => already done in saveTxsInFiles()
+ //await this.deleteTransactions(block, dal);
await dal.trimSandboxes(block);
// Saves the block (DAL)
await dal.saveBlock(dbb);
+ // Save Transactions (DAL)
+ if (conf.storage?.transactions !== false) {
+ await dal.saveTxsInFiles(dbb.transactions, dbb.number, dbb.medianTime);
+ }
+
// Save wot file
if (!dal.fs.isMemoryOnly()) {
const wotbFilepath = Directory.getWotbFilePath(dal.rootPath);
@@ -684,7 +691,7 @@ export class DuniterBlockchain {
try {
// Saves the block (DAL)
block.wrong = false;
- await dal.saveSideBlockInFile(block);
+ await dal.saveSideBlock(block);
logger.info(
"SIDE Block #%s-%s added to the blockchain in %s ms",
block.number,
diff --git a/app/lib/common-libs/constants.ts b/app/lib/common-libs/constants.ts
index 91266e0aa9849d407ca7d2cfac5bdc6bc8ab1698..04fa63eb59012f589bb2c63c5319952bfc1190ff 100755
--- a/app/lib/common-libs/constants.ts
+++ b/app/lib/common-libs/constants.ts
@@ -548,6 +548,7 @@ export const CommonConstants = {
MILESTONES_PER_PAGE: 50,
CHUNK_PREFIX: "chunk_",
BLOCKS_IN_MEMORY_MAX: 288 * 60, // 288 = 1 day
+ MAX_SQLITE_WRITE_PENDINGS: 200,
MAX_AGE_OF_PEER_IN_BLOCKS: 200, // blocks
INITIAL_DOWNLOAD_SLOTS: 1, // 1 peer
diff --git a/app/lib/common-libs/underscore.ts b/app/lib/common-libs/underscore.ts
index 3f567e3d81296ec7161300d45a01d39a0fd080af..a9d9073af7e0ef38892de5785edb02844999ccb8 100644
--- a/app/lib/common-libs/underscore.ts
+++ b/app/lib/common-libs/underscore.ts
@@ -92,8 +92,8 @@ export const Underscore = {
return _underscore_.extend(t1, t2);
},
- range: (count: number, end?: number): number[] => {
- return _underscore_.range(count, end);
+ range: (count: number, end?: number, step?: number): number[] => {
+ return _underscore_.range(count, end, step);
},
chain: <T>(element: T[]): UnderscoreClass<T> => {
diff --git a/app/lib/dal/drivers/LevelDBDriver.ts b/app/lib/dal/drivers/LevelDBDriver.ts
index 0fd4c034488a0a9c40fa892c52a7969c90a71876..c6f1a461ec1f1add98db6afad47e0189c4e44806 100644
--- a/app/lib/dal/drivers/LevelDBDriver.ts
+++ b/app/lib/dal/drivers/LevelDBDriver.ts
@@ -13,7 +13,6 @@
import * as levelup from "levelup";
import { LevelUp } from "levelup";
-import { AbstractLevelDOWN, ErrorCallback } from "abstract-leveldown";
import * as leveldown from "leveldown";
import * as memdown from "memdown";
diff --git a/app/lib/dal/drivers/SQLiteDriver.ts b/app/lib/dal/drivers/SQLiteDriver.ts
index 0cc9700382c7bae95d6e2760beb10a3fbc722cdd..2145d965ac44d01b2cf29c8a73c65a7f9868dcaa 100644
--- a/app/lib/dal/drivers/SQLiteDriver.ts
+++ b/app/lib/dal/drivers/SQLiteDriver.ts
@@ -29,10 +29,15 @@ export class SQLiteDriver {
getDB(): Promise<any> {
if (!this.dbPromise) {
this.dbPromise = (async () => {
- this.logger.debug('Opening SQLite database "%s"...', this.path);
+ this.logger.trace('Opening SQLite database "%s"...', this.path);
+
let sqlite = new sqlite3.Database(this.path);
await new Promise<any>((resolve) => sqlite.once("open", resolve));
+
// Database is opened
+ //this.logger.debug('Database "%s" opened', this.path);
+
+ // Trace SQL queries
if (OtherConstants.SQL_TRACES) {
sqlite.on("trace", (trace: any) => {
this.logger.trace(trace);
@@ -116,9 +121,9 @@ export class SQLiteDriver {
db.open; // For an unknown reason, we need this line.
}
await new Promise<void>((resolve, reject) => {
- this.logger.debug('Closing SQLite database "%s"...', this.path);
+ this.logger.trace('Closing SQLite database "%s"...', this.path);
db.on("close", () => {
- this.logger.info('Database "%s" closed.', this.path);
+ //this.logger.info('Database "%s" closed.', this.path);
this.dbPromise = null;
resolve();
});
diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts
index 2d04a61636ba9be157619801b22795d084a05e23..bfb07a09ce45f38fe9bcf473dc684fba6339eddf 100644
--- a/app/lib/dal/fileDAL.ts
+++ b/app/lib/dal/fileDAL.ts
@@ -234,14 +234,6 @@ export class FileDAL implements ServerDAO {
return this.metaDAL.getVersion();
}
- writeFileOfBlock(block: DBBlock) {
- return this.blockDAL.saveBlock(block);
- }
-
- writeSideFileOfBlock(block: DBBlock) {
- return this.blockDAL.saveSideBlock(block);
- }
-
listAllPeers() {
return this.peerDAL.listAll();
}
@@ -1243,12 +1235,16 @@ export class FileDAL implements ServerDAO {
}
}
- async saveBlock(dbb: DBBlock) {
+ saveBlock(dbb: DBBlock) {
dbb.wrong = false;
- await Promise.all([
- this.saveBlockInFile(dbb),
- this.saveTxsInFiles(dbb.transactions, dbb.number, dbb.medianTime),
- ]);
+ return this.blockDAL.saveBlock(dbb);
+
+ // Since v1.8.7, saveTxsInFiles() should be call only if TX storage enabled, by the caller
+ //await this.saveTxsInFiles(dbb.transactions, dbb.number, dbb.medianTime);
+ }
+
+ saveSideBlock(block: DBBlock) {
+ return this.blockDAL.saveSideBlock(block);
}
async generateIndexes(
@@ -1343,14 +1339,6 @@ export class FileDAL implements ServerDAO {
return this.msDAL.savePendingMembership(ms);
}
- async saveBlockInFile(block: DBBlock) {
- await this.writeFileOfBlock(block);
- }
-
- saveSideBlockInFile(block: DBBlock) {
- return this.writeSideFileOfBlock(block);
- }
-
/**
* Map tx DTO into DBtxs
* @param txs
@@ -1399,9 +1387,14 @@ export class FileDAL implements ServerDAO {
medianTime: number
): Promise<DBTx[]> {
if (!txs.length) return [];
- const dbTxs = await this.mapToDBTxs(txs, block_number, medianTime);
- await this.txsDAL.insertBatch(dbTxs);
- return dbTxs;
+ const records = await this.mapToDBTxs(txs, block_number, medianTime);
+ await this.txsDAL.insertBatch(records);
+ return records;
+ }
+
+ removeAllTxs() {
+ logger.debug("Removing all existing txs...");
+ return this.txsDAL.removeAll();
}
async merkleForPeers() {
@@ -1583,17 +1576,19 @@ export class FileDAL implements ServerDAO {
}
}
- async cleanCaches() {
- await Underscore.values(this.dals).map(
- (dal: Initiable) => dal.cleanCache && dal.cleanCache()
+ cleanCaches() {
+ return Promise.all(
+ Underscore.values(this.dals)
+ .filter((dal: Initiable) => typeof dal.cleanCache === "function")
+ .map((dal: Initiable) => dal.cleanCache())
);
}
async close() {
await Promise.all(
- Underscore.values(this.dals).map(async (dal: Initiable) => {
+ Underscore.values(this.dals).map((dal: Initiable) => {
dal.cleanCache();
- await dal.close();
+ return dal.close();
})
);
await this.sqliteDriver.closeConnection();
diff --git a/app/lib/dal/indexDAL/leveldb/LevelDBBlockchain.ts b/app/lib/dal/indexDAL/leveldb/LevelDBBlockchain.ts
index e0101e8bbc79ee670eb043e51493fb21f6986c2c..6a76cb67d2b9308e8257edcb17f3e1241f95b793 100644
--- a/app/lib/dal/indexDAL/leveldb/LevelDBBlockchain.ts
+++ b/app/lib/dal/indexDAL/leveldb/LevelDBBlockchain.ts
@@ -106,8 +106,10 @@ export class LevelDBBlockchain extends LevelDBTable<DBBlock>
async close(): Promise<void> {
await super.close();
- await this.forks.close();
- await Promise.all(this.indexers.map((i) => i.close()));
+ if (this.indexers.length) {
+ await this.forks.close();
+ await Promise.all(this.indexers.map((i) => i.close()));
+ }
}
/**
diff --git a/app/lib/dal/indexDAL/leveldb/LevelDBDividend.ts b/app/lib/dal/indexDAL/leveldb/LevelDBDividend.ts
index 161ddca3883413a691077d090b03296ac9d21493..896abf561440c210d60192c5d15e11b78d327752 100644
--- a/app/lib/dal/indexDAL/leveldb/LevelDBDividend.ts
+++ b/app/lib/dal/indexDAL/leveldb/LevelDBDividend.ts
@@ -31,7 +31,6 @@ export class LevelDBDividend extends LevelDBTable<DividendEntry>
* TECHNICAL
*/
- cleanCache(): void {}
async init(): Promise<void> {
await super.init();
this.indexForTrimming = new LevelDBTable<string[]>(
diff --git a/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts b/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts
index fd5010024ec958c2e6768c2ac5de27427f3c9ba1..becb73a5af5af1eb78572bd5c5b3392ec183a2ea 100644
--- a/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts
+++ b/app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts
@@ -61,10 +61,10 @@ export class LevelDBIindex extends LevelDBTable<IindexEntry[]>
async close(): Promise<void> {
await super.close();
- await this.indexForHash.close();
- await this.indexForUid.close();
- await this.indexForKick.close();
- await this.indexForWrittenOn.close();
+ await this.indexForHash?.close();
+ await this.indexForUid?.close();
+ await this.indexForKick?.close();
+ await this.indexForWrittenOn?.close();
}
/**
diff --git a/app/lib/dal/indexDAL/leveldb/LevelDBMindex.ts b/app/lib/dal/indexDAL/leveldb/LevelDBMindex.ts
index dcadf4ad721d46e668bc6809c47dd91f68334672..6def894d35697ea8a48f73330cf7b4622758d8e8 100644
--- a/app/lib/dal/indexDAL/leveldb/LevelDBMindex.ts
+++ b/app/lib/dal/indexDAL/leveldb/LevelDBMindex.ts
@@ -52,9 +52,9 @@ export class LevelDBMindex extends LevelDBTable<MindexEntry[]>
async close(): Promise<void> {
await super.close();
- await this.indexForExpiresOn.close();
- await this.indexForRevokesOn.close();
- await this.indexForWrittenOn.close();
+ await this.indexForExpiresOn?.close();
+ await this.indexForRevokesOn?.close();
+ await this.indexForWrittenOn?.close();
}
/**
diff --git a/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts b/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts
index 4fd8de92bfd2fa6ff76002c5059d67831a0139b4..a32ed465681498589915aafc9860ae17d29c4b9d 100644
--- a/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts
+++ b/app/lib/dal/indexDAL/leveldb/LevelDBSindex.ts
@@ -215,7 +215,11 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
const createRecord = await this.getOrNull(createKey);
const updateKey = LevelDBSindex.trimKey(identifier, pos, true);
const updateRecord = await this.getOrNull(updateKey);
- if (createRecord && updateRecord && updateRecord.writtenOn < belowNumber) {
+ if (
+ createRecord &&
+ updateRecord &&
+ updateRecord.writtenOn < belowNumber
+ ) {
// We remember the trimmed source id to remove it from the writtenOn and conditions index
mapIds[id] = {
createdOn: createRecord.writtenOn,
@@ -357,16 +361,21 @@ export class LevelDBSindex extends LevelDBTable<SindexEntry>
* Allow to get internal index, for unit test (see test triming-dal.ts)
*/
getInternalIndexes(): LevelDBTable<any>[] {
- return [this.indexForTrimming, this.indexForConditions, this.indexForConsumed, this.indexOfComplexeConditionForPubkeys];
+ return [
+ this.indexForTrimming,
+ this.indexForConditions,
+ this.indexForConsumed,
+ this.indexOfComplexeConditionForPubkeys,
+ ];
}
getIndexForTrimming(): LevelDBTable<any> {
- return this.indexForTrimming
+ return this.indexForTrimming;
}
getIndexForConditions(): LevelDBTable<any> {
- return this.indexForConditions
+ return this.indexForConditions;
}
getIndexForConsumed(): LevelDBTable<any> {
- return this.indexForConsumed
+ return this.indexForConsumed;
}
private async trimConditions(condition: string, id: string) {
diff --git a/app/lib/dal/indexDAL/leveldb/LevelDBTable.ts b/app/lib/dal/indexDAL/leveldb/LevelDBTable.ts
index f9d018a11a4afaef0ffe07e57d739cbb9d6d93ed..668351f06334fa0c03faa66834ecc3e01e0c60e6 100644
--- a/app/lib/dal/indexDAL/leveldb/LevelDBTable.ts
+++ b/app/lib/dal/indexDAL/leveldb/LevelDBTable.ts
@@ -12,12 +12,13 @@ export class LevelDBTable<T> {
cleanCache(): void {}
- async close() {
- await this.db.close();
- }
-
async init(): Promise<void> {
this.db = await this.getLevelDB(`${this.name}`);
+ await this.db.open();
+ }
+
+ async close() {
+ await this.db?.close();
}
public async get(k: string): Promise<T> {
diff --git a/app/lib/dal/indexDAL/sqlite/SqliteMIndex.ts b/app/lib/dal/indexDAL/sqlite/SqliteMIndex.ts
index 0d8b60d2bb4768e50ddbd91490cf6ec222449d10..db86d71f1e3fa2f5b5deff54b085fd4cf7f1d9e6 100644
--- a/app/lib/dal/indexDAL/sqlite/SqliteMIndex.ts
+++ b/app/lib/dal/indexDAL/sqlite/SqliteMIndex.ts
@@ -74,6 +74,11 @@ export class SqliteMIndex extends SqliteTable<MindexEntry>
`);
}
+ async close(): Promise<void> {
+ await super.close();
+ await this.d2?.close();
+ }
+
/**
* INSERT
*/
diff --git a/app/lib/dal/indexDAL/sqlite/SqliteNodeIOManager.ts b/app/lib/dal/indexDAL/sqlite/SqliteNodeIOManager.ts
index be65c3308f6d23dc9121f2016eb30bd2b4191a32..915cf21800683c5a2fb0ebbd5855649b6a3a6785 100644
--- a/app/lib/dal/indexDAL/sqlite/SqliteNodeIOManager.ts
+++ b/app/lib/dal/indexDAL/sqlite/SqliteNodeIOManager.ts
@@ -2,9 +2,11 @@ import { SQLiteDriver } from "../../drivers/SQLiteDriver";
import { MonitorExecutionTime } from "../../../debug/MonitorExecutionTime";
import { NewLogger } from "../../../logger";
import { ExitCodes } from "../../../common-libs/exit-codes";
+import { CommonConstants } from "../../../common-libs/constants";
export class SqliteNodeIOManager<T> {
private writePromise: Promise<any> | null = null;
+ private writePendingCount = 0;
constructor(private driver: SQLiteDriver, private id: string) {}
@@ -13,12 +15,19 @@ export class SqliteNodeIOManager<T> {
await this.writePromise;
// We no more need to wait
this.writePromise = null;
+ this.writePendingCount = 0;
}
public async sqlWrite(sql: string, params: any[]) {
- // // Just promise that the writing will be done
+ if (this.writePendingCount >= CommonConstants.MAX_SQLITE_WRITE_PENDINGS) {
+ await this.wait4writing();
+ }
+
+ this.writePendingCount++;
+ // Just promise that the writing will be done
this.writePromise = (this.writePromise || Promise.resolve())
.then(() => this.driver.executeAll(sql, params))
+ .then(() => this.writePendingCount--)
.catch((e) => {
NewLogger().error(e);
process.exit(ExitCodes.MINDEX_WRITING_ERROR);
@@ -42,6 +51,10 @@ export class SqliteNodeIOManager<T> {
}
async close() {
+ if (this.writePromise) {
+ // Wait for current writings to be done
+ await this.wait4writing();
+ }
await this.driver.closeConnection();
}
}
diff --git a/app/lib/dal/indexDAL/sqlite/SqliteTable.ts b/app/lib/dal/indexDAL/sqlite/SqliteTable.ts
index a0ee566e0a120a2b74af4bd8cd7fe040907ce34f..18738529666ce4e2f8da1f1b87235b031b39063f 100644
--- a/app/lib/dal/indexDAL/sqlite/SqliteTable.ts
+++ b/app/lib/dal/indexDAL/sqlite/SqliteTable.ts
@@ -22,14 +22,14 @@ export class SqliteTable<T> {
await this.driver.sqlExec(`
BEGIN;
${this.generateCreateTable()};
- ${this.generateUpgradeSql()};
- ${this.generateCreateIndexes()};
+ ${this.generateUpgradeSql()}
+ ${this.generateCreateIndexes()}
COMMIT;
`);
}
async close(): Promise<void> {
- await this.driver.close();
+ await this.driver?.close();
}
async disableCheckConstraints(): Promise<void> {
diff --git a/app/lib/dal/indexDAL/sqlite/SqliteTransactions.ts b/app/lib/dal/indexDAL/sqlite/SqliteTransactions.ts
index f739136e5e47d3bdb3f1927a26c41038b140dbbb..4c2439b4bb0704bba08b1badabd79b289091a576 100644
--- a/app/lib/dal/indexDAL/sqlite/SqliteTransactions.ts
+++ b/app/lib/dal/indexDAL/sqlite/SqliteTransactions.ts
@@ -9,6 +9,7 @@ import { DBTx } from "../../../db/DBTx";
import { TxsDAO } from "../abstract/TxsDAO";
import { SandBox } from "../../sqliteDAL/SandBox";
import { TransactionDTO } from "../../../dto/TransactionDTO";
+import { Underscore } from "../../../common-libs/underscore";
const constants = require("../../../constants");
@@ -112,17 +113,15 @@ export class SqliteTransactions extends SqliteTable<DBTx> implements TxsDAO {
@MonitorExecutionTime()
async insertBatch(records: DBTx[]): Promise<void> {
- if (records.length) {
- return this.insertBatchInTable(this.driver, records);
- }
+ if (!records.length) return;
+ return this.insertBatchInTable(this.driver, records);
}
@MonitorExecutionTime()
async saveBatch(records: DBTx[]): Promise<void> {
- if (records.length) {
- await this.removeByHashBatch(records.map((t) => t.hash));
- await this.insertBatch(records);
- }
+ if (!records.length) return;
+ await this.removeByHashBatch(Underscore.uniq(records.map((t) => t.hash)));
+ return this.insertBatchInTable(this.driver, records);
}
sandbox: SandBox<{
@@ -352,16 +351,17 @@ export class SqliteTransactions extends SqliteTable<DBTx> implements TxsDAO {
}
async removeByHashBatch(hashArray: string[]): Promise<void> {
- let i = 0;
- // Delete by slice of 500 items (because SQLite IN operator is limited)
- while (i < hashArray.length - 1) {
- const slice = hashArray.slice(i, i + 500);
- await this.driver.sqlWrite(
- `DELETE FROM txs WHERE hash IN (${slice.map((_) => "?").join(",")})`,
- slice
- );
- i += 500;
- }
+ // Delete by slice of 100 items (because SQLite IN operator is limited)
+ await Promise.all(
+ Underscore.range(0, hashArray.length, 100)
+ .map(
+ (start) =>
+ `DELETE FROM txs WHERE hash IN ('${hashArray
+ .slice(start, start + 100)
+ .join("', '")}')`
+ )
+ .map((sql) => this.driver.sqlWrite(sql, []))
+ );
}
removeByHash(hash: string): Promise<void> {
diff --git a/app/modules/bma/lib/controllers/node.ts b/app/modules/bma/lib/controllers/node.ts
index 2bd24394fe1ae7dc7381aff50d29efd87d8dcc88..770a2eb93000ba47db1167e3355932e69cb26131 100644
--- a/app/modules/bma/lib/controllers/node.ts
+++ b/app/modules/bma/lib/controllers/node.ts
@@ -23,10 +23,8 @@ export class NodeBinding extends AbstractController {
version: this.server.version,
forkWindowSize: this.server.conf.forksize,
storage: {
- transactions: this.server.conf.storage
- ? this.server.conf.storage.transactions
- : true, // true by default (see app/module/config.ts)
- wotwizard: this.server.conf.storage?.wotwizard || false, // false by default (see app/module/config.ts)
+ transactions: this.server.conf.storage?.transactions !== false, // true by default
+ wotwizard: this.server.conf.storage?.wotwizard === true, // false by default
},
},
};
diff --git a/app/modules/config.ts b/app/modules/config.ts
index f5e90b3e0826e8422859303cb8b8a58492a9bb68..e69bd60f0af470f80c3006b40a10948a2445e7da 100644
--- a/app/modules/config.ts
+++ b/app/modules/config.ts
@@ -37,22 +37,18 @@ module.exports = {
CommonConstants.SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS;
// Transactions storage
+ conf.storage = conf.storage || {
+ transactions: false,
+ wotwizard: false,
+ };
if (
program.storeTxs ||
(program.storeTxs === undefined && !conf.nobma)
) {
- if (!conf.storage) {
- conf.storage = { transactions: true, wotwizard: false };
- } else {
- conf.storage.transactions = true;
- }
+ conf.storage.transactions = true;
}
if (program.storeWw) {
- if (!conf.storage) {
- conf.storage = { transactions: false, wotwizard: true };
- } else {
- conf.storage.wotwizard = true;
- }
+ conf.storage.wotwizard = true;
}
},
beforeSave: async (conf: ConfDTO) => {
diff --git a/app/modules/crawler/lib/sync.ts b/app/modules/crawler/lib/sync.ts
index 715c64f874b76f3e6c2989dec3c3d6f3a2968a0d..45bab02f99b2aee0fbb6d418c937807bcc3054f2 100644
--- a/app/modules/crawler/lib/sync.ts
+++ b/app/modules/crawler/lib/sync.ts
@@ -186,6 +186,10 @@ export class Synchroniser extends stream.Duplex {
// Disable check constraints
if (!cautious) await this.server.dal.disableCheckConstraints();
+ // Remove existing TX, because we will all download it again
+ // This is required to be able to insert, instead of save
+ if (fullSync) await this.dal.removeAllTxs();
+
const milestonesStream = new ValidatorStream(
localNumber,
to,
@@ -228,7 +232,7 @@ export class Synchroniser extends stream.Duplex {
this.watcher.storagePercent(100.0);
this.watcher.appliedPercent(100.0);
- this.server.dal.blockDAL.cleanCache();
+ await this.server.dal.cleanCaches();
if (!cliprogram.nosbx) {
//=======
diff --git a/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts b/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts
index 120578daa16ede72927bdef3cdae5fcc4e290a5d..f27fb91493b1893ae53679ee78798c7605c3f1aa 100644
--- a/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts
+++ b/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts
@@ -422,18 +422,18 @@ export class GlobalIndexStream extends Duplex {
@MonitorExecutionTime()
private async beforeBlocks(blocks: BlockDTO[]) {
+ let newTxCount = 0;
await this.dal.blockDAL.insertBatch(
blocks.map((b) => {
- txCount += b.transactions.length;
+ newTxCount += b.transactions.length;
const block = DBBlock.fromBlockDTO(b);
block.fork = false;
return block;
})
);
- if (this.conf.storage?.transactions) {
- // if cautious, use a save (insert or update)
- if (this.cautious) {
+ if (newTxCount && this.conf.storage?.transactions !== false) {
+ if (this.cautious || this.localNumber >= 0) {
await Promise.all(
blocks.map((block) =>
this.dal.saveTxsInFiles(
@@ -443,9 +443,7 @@ export class GlobalIndexStream extends Duplex {
)
)
);
- }
- // If not cautious: use insert only
- else {
+ } else {
await Promise.all(
blocks.map((block) =>
this.dal.insertTxsInFiles(
@@ -456,9 +454,9 @@ export class GlobalIndexStream extends Duplex {
)
);
}
+ txCount += newTxCount;
+ logger.debug("Total tx count: %s", txCount);
}
-
- logger.debug("Total tx count: %s", txCount);
}
@MonitorExecutionTime()
diff --git a/app/modules/ws2p/lib/WS2PCluster.ts b/app/modules/ws2p/lib/WS2PCluster.ts
index f5a7fcb287a89f5357391afb4dda864791b1ae10..711d238bb86fe2ae8cc4eef8ca6d62aab5455496 100644
--- a/app/modules/ws2p/lib/WS2PCluster.ts
+++ b/app/modules/ws2p/lib/WS2PCluster.ts
@@ -11,28 +11,28 @@
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
-import {WS2PServer} from "./WS2PServer";
-import {Server} from "../../../../server";
-import {WS2PClient} from "./WS2PClient";
-import {WS2PConnection} from "./WS2PConnection";
-import {randomPick} from "../../../lib/common-libs/randomPick";
-import {CrawlerConstants} from "../../crawler/lib/constants";
-import {WS2PBlockPuller} from "./WS2PBlockPuller";
-import {WS2PDocpoolPuller} from "./WS2PDocpoolPuller";
-import {WS2PConstants} from "./constants";
-import {PeerDTO, WS2PEndpoint} from "../../../lib/dto/PeerDTO";
-import {GlobalFifoPromise} from "../../../service/GlobalFifoPromise";
-import {OtherConstants} from "../../../lib/other_constants";
-import {Key} from "../../../lib/common-libs/crypto/keyring";
-import {verify} from "../../../../neon/lib";
-import {WS2PServerMessageHandler} from "./interface/WS2PServerMessageHandler";
-import {WS2PMessageHandler} from "./impl/WS2PMessageHandler";
-import {CommonConstants} from "../../../lib/common-libs/constants";
-import {Package} from "../../../lib/common/package";
-import {ProverConstants} from "../../prover/lib/constants";
-import {ProxiesConf} from "../../../lib/proxy";
-import {Underscore} from "../../../lib/common-libs/underscore";
-import {NewLogger} from "../../../lib/logger";
+import { WS2PServer } from "./WS2PServer";
+import { Server } from "../../../../server";
+import { WS2PClient } from "./WS2PClient";
+import { WS2PConnection } from "./WS2PConnection";
+import { randomPick } from "../../../lib/common-libs/randomPick";
+import { CrawlerConstants } from "../../crawler/lib/constants";
+import { WS2PBlockPuller } from "./WS2PBlockPuller";
+import { WS2PDocpoolPuller } from "./WS2PDocpoolPuller";
+import { WS2PConstants } from "./constants";
+import { PeerDTO, WS2PEndpoint } from "../../../lib/dto/PeerDTO";
+import { GlobalFifoPromise } from "../../../service/GlobalFifoPromise";
+import { OtherConstants } from "../../../lib/other_constants";
+import { Key } from "../../../lib/common-libs/crypto/keyring";
+import { verify } from "../../../../neon/lib";
+import { WS2PServerMessageHandler } from "./interface/WS2PServerMessageHandler";
+import { WS2PMessageHandler } from "./impl/WS2PMessageHandler";
+import { CommonConstants } from "../../../lib/common-libs/constants";
+import { Package } from "../../../lib/common/package";
+import { ProverConstants } from "../../prover/lib/constants";
+import { ProxiesConf } from "../../../lib/proxy";
+import { Underscore } from "../../../lib/common-libs/underscore";
+import { NewLogger } from "../../../lib/logger";
const es = require("event-stream");
const nuuid = require("node-uuid");
diff --git a/server.ts b/server.ts
index 97f067ffa985d39c9d2af94c82cab73fa37ad81a..199ebe35f6dee5bb79d49d18668c532d6d7d6608 100644
--- a/server.ts
+++ b/server.ts
@@ -175,7 +175,7 @@ export class Server extends stream.Duplex implements HookableServer {
}
async getLevelDB(dbName: string, home: string) {
- // Check in cach (useful to avoid migration task to create a new driver on the same DB file)
+ // Check in cache (useful to avoid migration task to create a new driver on the same DB file)
let driver: LevelUp = this.levelDBs[dbName];
if (!driver || driver.isClosed()) {
driver = await Directory.getHomeLevelDB(this.memoryOnly, dbName, home);
diff --git a/test/integration/network/network-merkle.ts b/test/integration/network/network-merkle.ts
index d1ea5d084537f2be595610d24fb14aad9ece9638..4f624defdaaa0021ab515ad30e76f50076677bb9 100644
--- a/test/integration/network/network-merkle.ts
+++ b/test/integration/network/network-merkle.ts
@@ -13,7 +13,7 @@
import {Underscore} from "../../../app/lib/common-libs/underscore"
import {HttpMerkleOfPeers} from "../../../app/modules/bma/lib/dtos"
-import {NewTestingServer} from "../tools/toolbox"
+import {NewTestingServer, TestingServer} from "../tools/toolbox"
import {expectAnswer, expectHttpCode} from "../tools/http-expect"
const rp = require('request-promise');
@@ -28,34 +28,37 @@ const commonConf = {
sigQty: 1
};
-const s1 = NewTestingServer(Underscore.extend({
- name: 'bb33',
- ipv4: '127.0.0.1',
- port: '20501',
- remoteport: '20501',
- ws2p: { upnp: false },
- pair: {
- pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd',
- sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'
- },
- rootoffset: 10,
- sigQty: 1, dt: 0, ud0: 120
-}, commonConf));
-
-const s2 = NewTestingServer(Underscore.extend({
- name: 'bb12',
- port: '20502',
- remoteport: '20502',
- ws2p: { upnp: false },
- pair: {
- pub: 'DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo',
- sec: '64EYRvdPpTfLGGmaX5nijLXRqWXaVz8r1Z1GtaahXwVSJGQRn7tqkxLb288zwSYzELMEG5ZhXSBYSxsTsz1m9y8F'
- }
-}, commonConf));
-
describe("Network Merkle", function() {
+ let s1: TestingServer, s2: TestingServer;
+
before(async () => {
+
+ s1 = NewTestingServer(Underscore.extend({
+ name: 'bb33',
+ ipv4: '127.0.0.1',
+ port: '20501',
+ remoteport: '20501',
+ ws2p: { upnp: false },
+ pair: {
+ pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd',
+ sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'
+ },
+ rootoffset: 10,
+ sigQty: 1, dt: 0, ud0: 120
+ }, commonConf));
+
+ s2 = NewTestingServer(Underscore.extend({
+ name: 'bb12',
+ port: '20502',
+ remoteport: '20502',
+ ws2p: { upnp: false },
+ pair: {
+ pub: 'DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo',
+ sec: '64EYRvdPpTfLGGmaX5nijLXRqWXaVz8r1Z1GtaahXwVSJGQRn7tqkxLb288zwSYzELMEG5ZhXSBYSxsTsz1m9y8F'
+ }
+ }, commonConf));
+
await s1.initDalBmaConnections()
await s2.initDalBmaConnections()
await s1._server.PeeringService.generateSelfPeer(s1._server.conf, 0)
diff --git a/test/integration/tools/toolbox.ts b/test/integration/tools/toolbox.ts
index 5c3b65dcf327550c55409430c8bee8b3c3e2ddc6..4d18549d85e4f52b9f3c9807d0f4d0e0d0847e1d 100644
--- a/test/integration/tools/toolbox.ts
+++ b/test/integration/tools/toolbox.ts
@@ -243,6 +243,11 @@ export const NewTestingServer = (conf:any) => {
httpLogs: true,
forksize: conf.forksize !== undefined ? conf.forksize : 3,
nonWoTPeersLimit: CommonConstants.DEFAULT_NON_WOT_PEERS_LIMIT,
+ storage: {
+ transaction: true,
+ wotwizard: false,
+ ...conf.storage
+ }
};
if (conf.sigQty === undefined) {
conf.sigQty = 1;
diff --git a/test/integration/ws2p/ws2p_pulling.ts b/test/integration/ws2p/ws2p_pulling.ts
index 52f7aaaf448e026d850a17eace500ac704cae3e2..7794605a2d0686c95c15e95a23333818ef7960b7 100644
--- a/test/integration/ws2p/ws2p_pulling.ts
+++ b/test/integration/ws2p/ws2p_pulling.ts
@@ -74,7 +74,7 @@ describe("WS2P block pulling", function() {
cluster2 = network.cluster2
})
- after(() => wss.close())
+ after(() => wss?.close())
it('should have b#6 on s1, b#2 on s2', async () => {
const currentS1 = await s1.BlockchainService.current()