From 2d83ee9e27ab820d8a9cd05a4b0d921abe332f99 Mon Sep 17 00:00:00 2001
From: Benoit Lavenier <benoit.lavenier@e-is.pro>
Date: Fri, 11 Aug 2023 19:36:54 +0200
Subject: [PATCH] enh(tx): Compute and use an tx.id, in ng-repeat enh(bma):
Avoid too many request error, when loading tx history very quickly
---
www/js/controllers/wallet-controllers.js | 38 ++---
www/js/services/bma-services.js | 186 +++++++++++++++++------
www/js/services/network-services.js | 2 +-
www/js/services/settings-services.js | 9 +-
www/js/services/tx-services.js | 88 +++++++----
www/templates/wallet/view_wallet_tx.html | 8 +-
6 files changed, 226 insertions(+), 105 deletions(-)
diff --git a/www/js/controllers/wallet-controllers.js b/www/js/controllers/wallet-controllers.js
index c58079fb5..20c2f3c8b 100644
--- a/www/js/controllers/wallet-controllers.js
+++ b/www/js/controllers/wallet-controllers.js
@@ -727,12 +727,14 @@ function WalletTxController($scope, $ionicPopover, $state, $timeout, $location,
if (!wallet) return $q.reject('Missing wallet');
var hasMinData = wallet.isDataLoaded({minData: true});
+ var fromTime = csHttp.date.now() - csSettings.data.walletHistoryTimeSecond;
var options = {
requirements: !hasMinData, // load requirements (=minData) once
minData: !hasMinData,
sources: true,
tx: {
- enable: true
+ enable: true,
+ fromTime: fromTime
}
};
@@ -773,10 +775,13 @@ function WalletTxController($scope, $ionicPopover, $state, $timeout, $location,
// Update view
$scope.updateView = function() {
- if (!$scope.formData || $scope.loading) return;
+ if (!$scope.formData || !$scope.formData.tx || $scope.loading) return;
- var fetchMoreMinTime = csHttp.date.now() - csSettings.data.walletHistoryTimeSecond * 5;
- $scope.formData.tx.canFetchMore = $scope.formData.tx.fromTime > 0 && $scope.formData.tx.fromTime > fetchMoreMinTime;
+ $scope.minScrollMoreTime = moment().utc()
+ .subtract(csSettings.data.walletHistoryScrollMaxTimeSecond, 'second')
+ .startOf('day')
+ .unix();
+ $scope.canScrollMore = $scope.formData.tx.fromTime > $scope.minScrollMoreTime;
$scope.$broadcast('$$rebind::balance'); // force rebind balance
$scope.$broadcast('$$rebind::rebind'); // force rebind
@@ -793,7 +798,7 @@ function WalletTxController($scope, $ionicPopover, $state, $timeout, $location,
// Updating wallet data
$scope.doUpdate = function(silent) {
console.debug('[wallet] TX history reloading...');
- var fromTime = $scope.formData && $scope.formData.tx && $scope.formData.tx.fromTime || undefined;
+ var fromTime = $scope.formData && $scope.formData.tx && $scope.formData.tx.fromTime || undefined;
var options = {
sources: true,
tx: {
@@ -802,6 +807,7 @@ function WalletTxController($scope, $ionicPopover, $state, $timeout, $location,
},
api: false
};
+
return (silent ?
// If silent: just refresh
wallet.refreshData(options) :
@@ -910,9 +916,9 @@ function WalletTxController($scope, $ionicPopover, $state, $timeout, $location,
$scope.showMoreTx = function(fromTime) {
- if ($scope.formData.tx.loadingMore) return; // Skip
+ if ($scope.loadingMore) return; // Skip
- $scope.formData.tx.loadingMore = true;
+ $scope.loadingMore = true;
fromTime = fromTime ||
($scope.formData.tx.fromTime - csSettings.data.walletHistoryTimeSecond) ||
@@ -920,26 +926,24 @@ function WalletTxController($scope, $ionicPopover, $state, $timeout, $location,
console.info('[wallet-tx] Fetching more TX, since: ' + fromTime);
- return wallet.refreshData({tx: {enable: true, fromTime: fromTime}})
+ return wallet.refreshData({sources: true, tx: {enable: true, fromTime: fromTime}})
.then(function() {
+ $scope.loadingMore = false;
$scope.updateView();
- $scope.formData.tx.loadingMore = false;
$scope.$broadcast('scroll.infiniteScrollComplete');
})
.catch(function(err) {
// If http rest limitation: wait then retry
- if (err.ucode == BMA.errorCodes.HTTP_LIMITATION) {
- $timeout(function() {
+ if (err.ucode === BMA.errorCodes.HTTP_LIMITATION) {
+ return $timeout(function() {
return $scope.showMoreTx(fromTime); // Loop
}, 2000);
}
- else {
- $scope.formData.tx.loadingMore = false;
- $scope.formData.tx.canFetchMore = false;
- $scope.$broadcast('scroll.infiniteScrollComplete');
- UIUtils.onError('ERROR.REFRESH_WALLET_DATA')(err);
- }
+ $scope.loadingMore = false;
+ $scope.canScrollMore = false;
+ $scope.$broadcast('scroll.infiniteScrollComplete');
+ UIUtils.onError('ERROR.REFRESH_WALLET_DATA')(err);
});
};
diff --git a/www/js/services/bma-services.js b/www/js/services/bma-services.js
index 1f5cbba5f..6d768752a 100644
--- a/www/js/services/bma-services.js
+++ b/www/js/services/bma-services.js
@@ -91,6 +91,7 @@ angular.module('cesium.bma.services', ['ngApi', 'cesium.http.services', 'cesium.
that.raw = {
getByPath: {},
+ getCountByPath: {},
postByPath: {},
wsByPath: {}
};
@@ -154,33 +155,37 @@ angular.module('cesium.bma.services', ['ngApi', 'cesium.http.services', 'cesium.
that.raw.wsByPath = {};
}
- function cleanCache() {
- console.debug("[BMA] Cleaning cache {prefix: '{0}'}...".format(cachePrefix));
- csCache.clear(cachePrefix);
+ function cleanCache() {
+ console.debug("[BMA] Cleaning cache {prefix: '{0}'}...".format(cachePrefix));
+ csCache.clear(cachePrefix);
- // Clean raw requests by path cache
- that.raw.getByPath = {};
- that.raw.postByPath = {};
- that.raw.wsByPath = {};
- }
+ // Clean raw requests by path cache
+ that.raw.getByPath = {};
+ that.raw.getCountByPath = {};
+ that.raw.postByPath = {};
+ that.raw.wsByPath = {};
+ }
- function get(path, cacheTime, forcedTimeout) {
+ function getCacheable(path, cacheTime, forcedTimeout) {
cacheTime = that.useCache && cacheTime || 0 /* no cache*/ ;
forcedTimeout = forcedTimeout || timeout;
var cacheKey = path + (cacheTime ? ('#'+cacheTime) : '');
- var getRequestFn = function(params) {
+ // Store requestFn into a variable a function, to be able to call it to loop
+ var wrappedRequest = function(params) {
if (!that.started) {
if (!that._startPromise) {
console.warn('[BMA] Trying to get [{0}] before start(). Waiting...'.format(path));
}
- return that.ready().then(function() {
- return getRequestFn(params);
- });
+ return that.ready()
+ .then(function() {
+ return wrappedRequest(params);
+ });
}
+ // Create the request function, if not exists
var request = that.raw.getByPath[cacheKey];
if (!request) {
if (cacheTime) {
@@ -189,32 +194,75 @@ angular.module('cesium.bma.services', ['ngApi', 'cesium.http.services', 'cesium.
else {
request = csHttp.get(that.host, that.port, that.path + path, that.useSsl, forcedTimeout);
}
+
that.raw.getByPath[cacheKey] = request;
}
- var execCount = 1;
- return request(params)
- .catch(function(err){
- // If node return too many requests error
- if (err && err.ucode === exports.errorCodes.HTTP_LIMITATION) {
- // If max number of retry not reach
- if (execCount <= exports.constants.LIMIT_REQUEST_COUNT) {
- if (execCount === 1) {
- console.warn("[BMA] Too many HTTP requests: Will wait then retry...");
- // Update the loading message (if exists)
- UIUtils.loading.update({template: "COMMON.LOADING_WAIT"});
- }
- // Wait 1s then retry
- return $timeout(function() {
- execCount++;
- return request(params);
- }, exports.constants.LIMIT_REQUEST_DELAY);
- }
+
+ return request(params);
+ };
+
+ return wrappedRequest;
+ }
+
+ function incrementGetPathCount(path, limitRequestCount) {
+ limitRequestCount = limitRequestCount || constants.LIMIT_REQUEST_COUNT;
+ // Wait if too many requests on this path
+ if (that.raw.getCountByPath[path] >= limitRequestCount) {
+
+ // DEBUG
+ //console.debug("[BMA] Delaying request '{0}' to avoid a quota error...".format(path));
+
+ return $timeout(function() {
+ return incrementGetPathCount(path, limitRequestCount);
+ }, constants.LIMIT_REQUEST_DELAY);
+ }
+
+ that.raw.getCountByPath[path]++;
+ return $q.when();
+ }
+
+ function decrementGetPathCount(path, timeout) {
+ if (timeout > 0) {
+ $timeout(function() {
+ decrementGetPathCount(path);
+ }, timeout);
+ }
+ else {
+ that.raw.getCountByPath[path]--;
+ }
+ }
+
+ function get(path, cacheTime, forcedTimeout, limitRequestCount) {
+ limitRequestCount = limitRequestCount || constants.LIMIT_REQUEST_COUNT;
+
+ that.raw.getCountByPath[path] = that.raw.getCountByPath[path] || 0;
+ var request = getCacheable(path, cacheTime, forcedTimeout);
+
+ var wrappedRequest = function(params) {
+
+ var start = Date.now();
+ return incrementGetPathCount(path, limitRequestCount)
+ .then(function() {
+ return request(params);
+ })
+ .then(function(res) {
+ decrementGetPathCount(path, constants.LIMIT_REQUEST_DELAY - (Date.now() - start));
+ return res;
+ })
+ .catch(function (err) {
+ decrementGetPathCount(path, constants.LIMIT_REQUEST_DELAY - (Date.now() - start));
+ // When too many request, retry in 3s
+ if (err && err.ucode === errorCodes.HTTP_LIMITATION) {
+ // retry
+ return $timeout(function () {
+ return wrappedRequest(params);
+ }, constants.LIMIT_REQUEST_DELAY);
}
throw err;
});
};
- return getRequestFn;
+ return wrappedRequest;
}
function post(path) {
@@ -490,7 +538,7 @@ angular.module('cesium.bma.services', ['ngApi', 'cesium.http.services', 'cesium.
network: {
peering: {
self: get('/network/peering'),
- peers: get('/network/peering/peers')
+ peers: get('/network/peering/peers', null, null, 10)
},
peers: get('/network/peers'),
ws2p: {
@@ -880,10 +928,12 @@ angular.module('cesium.bma.services', ['ngApi', 'cesium.http.services', 'cesium.
/**
* Return all expected blocks
- * @param blockNumbers a rray of block number
+ * @param blockNumbers an array of block number
*/
exports.blockchain.blocks = function(blockNumbers){
- return exports.raw.getHttpRecursive(exports.blockchain.block, 'block', blockNumbers);
+ return $q.all(blockNumbers.map(function(block) {
+ return exports.blockchain.block({block: block})
+ }));
};
/**
@@ -891,7 +941,9 @@ angular.module('cesium.bma.services', ['ngApi', 'cesium.http.services', 'cesium.
* @param blockNumbers a rray of block number
*/
exports.network.peering.peersByLeaves = function(leaves){
- return exports.raw.getHttpRecursive(exports.network.peering.peers, 'leaf', leaves, 0, 10);
+ return $q.all(leaves.map(function(leaf) {
+ return exports.network.peering.peers({leaf: leaf})
+ }));
};
exports.raw.getHttpRecursive = function(httpGetRequest, paramName, paramValues, offset, size) {
@@ -900,7 +952,8 @@ angular.module('cesium.bma.services', ['ngApi', 'cesium.http.services', 'cesium.
return $q(function(resolve, reject) {
var result = [];
var jobs = [];
- _.each(paramValues.slice(offset, offset+size), function(paramValue) {
+ var chunk = paramValues.slice(offset, offset+size)
+ _.each(chunk, function(paramValue) {
var requestParams = {};
requestParams[paramName] = paramValue;
jobs.push(
@@ -945,17 +998,56 @@ angular.module('cesium.bma.services', ['ngApi', 'cesium.http.services', 'cesium.
});
};
- exports.raw.getHttpWithRetryIfLimitation = function(exec) {
- return exec()
- .catch(function(err){
- // When too many request, retry in 3s
- if (err && err.ucode == exports.errorCodes.HTTP_LIMITATION) {
- return $timeout(function() {
- // retry
- return exports.raw.getHttpWithRetryIfLimitation(exec);
- }, exports.constants.LIMIT_REQUEST_DELAY);
- }
+ function retryableSlices(getRequestFns, slices, offset, size) {
+ offset = angular.isDefined(offset) ? offset : 0;
+ size = size || exports.constants.LIMIT_REQUEST_COUNT;
+ return $q(function(resolve, reject) {
+ var result = [];
+ var jobs = [];
+ var chunk = slices.slice(offset, offset+size)
+ _.each(chunk, function(params) {
+ jobs.push(
+ getRequestFns(params)
+ .then(function(res){
+ if (!res) return;
+ result.push(res);
+ })
+ );
});
+
+ $q.all(jobs)
+ .then(function() {
+ if (offset < slices.length - 1) {
+ // Loop, with a new offset
+ $timeout(function() {
+ exports.raw.getHttpRecursive(getRequestFns, slices, offset+size, size)
+ .then(function(res) {
+ if (!res || !res.length) {
+ resolve(result);
+ return;
+ }
+
+ resolve(result.concat(res));
+ })
+ .catch(function(err) {
+ reject(err);
+ });
+ }, exports.constants.LIMIT_REQUEST_DELAY);
+ }
+ // End
+ else {
+ resolve(result);
+ }
+ })
+ .catch(function(err){
+ if (err && err.ucode === exports.errorCodes.HTTP_LIMITATION) {
+ resolve(result);
+ }
+ else {
+ reject(err);
+ }
+ });
+ });
};
exports.blockchain.lastUd = function() {
diff --git a/www/js/services/network-services.js b/www/js/services/network-services.js
index 8b50baad9..0569e4432 100644
--- a/www/js/services/network-services.js
+++ b/www/js/services/network-services.js
@@ -144,7 +144,7 @@ angular.module('cesium.network.services', ['ngApi', 'cesium.currency.services',
})
.catch(function(err) {
// When too many request, retry in 3s
- if (err && err.ucode == BMA.errorCodes.HTTP_LIMITATION) {
+ if (err && err.ucode === BMA.errorCodes.HTTP_LIMITATION) {
return $timeout(function() {
if (remainingTime() > 0) return loadW2spHeads();
}, 3000);
diff --git a/www/js/services/settings-services.js b/www/js/services/settings-services.js
index 7a37c0de4..7d9b60678 100644
--- a/www/js/services/settings-services.js
+++ b/www/js/services/settings-services.js
@@ -89,12 +89,13 @@ angular.module('cesium.settings.services', ['ngApi', 'cesium.config'])
defaultSettings = angular.merge({
timeout: -1, // -1 = auto
useRelative: false,
- useLocalStorage: !!$window.localStorage, // override to false if no device
+ useLocalStorage: !!$window.localStorage, // Overwritten to false if not a device
useLocalStorageEncryption: false,
persistCache: false, // disable by default (waiting resolution of issue #885)
- walletHistoryTimeSecond: 30 * 24 * 60 * 60 /*30 days*/,
- walletHistorySliceSecond: 5 * 24 * 60 * 60 /*download using 5 days slice*/,
- walletHistoryAutoRefresh: true, // override to false if device
+ walletHistoryTimeSecond: 30 * 24 * 60 * 60, // 30 days
+ walletHistorySliceSecond: 5 * 24 * 60 * 60, // download using 5 days slice - need for cache
+ walletHistoryScrollMaxTimeSecond: 2 * 30 * 24 * 60 * 60, // Limit TX load infinite scroll to 2 month
+ walletHistoryAutoRefresh: true, // Reload TX history on new block ? Overwritten to false if device
rememberMe: true,
keepAuthIdle: 10 * 60,
showUDHistory: true,
diff --git a/www/js/services/tx-services.js b/www/js/services/tx-services.js
index 88e1ee4a5..d98b85fd3 100644
--- a/www/js/services/tx-services.js
+++ b/www/js/services/tx-services.js
@@ -95,11 +95,12 @@ angular.module('cesium.tx.services', ['ngApi', 'cesium.bma.services',
var txPubkeys = amount > 0 ? otherIssuers : otherRecipients;
var time = tx.time || tx.blockstampTime;
- // Avoid duplicated tx, or tx to him self
- var txKey = (amount !== 0) && amount + ':' + tx.hash + ':' + time;
+ // Avoid duplicated tx, or tx to him self (if amount = 0)
+ var txKey = (amount !== 0) ? [amount, tx.hash, time].join(':') : undefined;
if (txKey && !processedTxMap[txKey]) {
processedTxMap[txKey] = true; // Mark as processed
var newTx = {
+ id: txKey,
time: time,
amount: amount,
pubkey: txPubkeys.length === 1 ? txPubkeys[0] : undefined,
@@ -139,6 +140,7 @@ angular.module('cesium.tx.services', ['ngApi', 'cesium.bma.services',
};
var processedTxMap = {};
+ var retryPendingCount = 0;
var jobs = [
// get current block
@@ -146,6 +148,15 @@ angular.module('cesium.tx.services', ['ngApi', 'cesium.bma.services',
// get pending tx
BMA.tx.history.pending({pubkey: pubkey})
+ .catch(function(err) {
+ if (err && err.ucode === BMA.errorCodes.HTTP_LIMITATION && retryPendingCount < 3) {
+ retryPendingCount++;
+ return $timeout(function() {
+ return BMA.tx.history.pending({pubkey: pubkey})
+ }, 2000 * retryPendingCount);
+ }
+ throw err;
+ })
.then(function (res) {
reduceTxAndPush(pubkey, res.history.sending, tx.pendings, processedTxMap, true /*allow pendings*/);
reduceTxAndPush(pubkey, res.history.pending, tx.pendings, processedTxMap, true /*allow pendings*/);
@@ -176,9 +187,9 @@ angular.module('cesium.tx.services', ['ngApi', 'cesium.bma.services',
// get TX from a given time
if (fromTime > 0) {
- jobs.push(slices.map(function(slice) {
- return BMA.tx.history.times(slice.params, slice.cache).then(reduceTxFn);
- }));
+ jobs.push($q.all(slices.map(function(slice) {
+ return BMA.tx.history.times(slice.params, slice.cache).then(reduceTxFn)
+ })));
}
// get all TX
@@ -194,6 +205,7 @@ angular.module('cesium.tx.services', ['ngApi', 'cesium.bma.services',
if (ud.time < fromTime) return res; // skip to old UD
var amount = powBase(ud.amount, ud.base);
tx.history.push({
+ id: [amount, 'ud', ud.time].join(':'),
time: ud.time,
amount: amount,
isUD: true,
@@ -203,10 +215,10 @@ angular.module('cesium.tx.services', ['ngApi', 'cesium.bma.services',
};
// get UD from a given time
- if ( fromTime > 0) {
- jobs.push(slices.map(function(slice) {
+ if (fromTime > 0) {
+ jobs.push($q.all(slices.map(function(slice) {
return BMA.ud.history.times(slice.params, slice.cache).then(reduceUdFn);
- }));
+ })));
}
// get all UD
else {
@@ -310,38 +322,48 @@ angular.module('cesium.tx.services', ['ngApi', 'cesium.bma.services',
function _processPendingTx(tx) {
var consumedSources = [];
- var valid = true;
- if (tx.amount > 0) { // do not check sources from received TX
- valid = false;
- // TODO get sources from the issuer ?
+ // do not check sources from received TX
+ // => move this tx in errors (even if not really)
+ if (tx.amount > 0) {
+ txErrors.push(tx);
}
else {
+ var validInputs = true;
_.find(tx.inputs, function(input) {
var inputKey = input.split(':').slice(2).join(':');
var srcIndex = data.sourcesIndexByKey[inputKey];
- if (angular.isDefined(srcIndex)) {
- consumedSources.push(data.sources[srcIndex]);
- }
- else {
- valid = false;
+
+ // The input source not exists: mark as invalid
+ if (!angular.isDefined(srcIndex)) {
+ validInputs = false;
return true; // break
}
+
+ // Mark input source as consumed
+ consumedSources.push(data.sources[srcIndex]);
});
- if (tx.sources) { // add source output
- addSources(data, tx.sources);
+
+ // Some input source not exist: mark as error
+ if (!validInputs) {
+ console.error("[tx] Pending TX '{}' use an unknown source as input: mark as error".format(tx.hash))
+ txErrors.push(tx);
+ }
+
+ // All tx inputs are valid
+ else {
+ // Add tx outputs has new sources
+ if (tx.sources) {
+ addSources(data, tx.sources);
+ }
+ delete tx.sources;
+ delete tx.inputs;
+
+ balanceWithPending += tx.amount; // update balance
+ txPendings.push(tx);
+ _.forEach(consumedSources, function(src) {
+ src.consumed = true;
+ });
}
- delete tx.sources;
- delete tx.inputs;
- }
- if (valid) {
- balanceWithPending += tx.amount; // update balance
- txPendings.push(tx);
- _.forEach(consumedSources, function(src) {
- src.consumed=true;
- });
- }
- else {
- txErrors.push(tx);
}
}
@@ -376,7 +398,9 @@ angular.module('cesium.tx.services', ['ngApi', 'cesium.bma.services',
var allTx = (data.tx.history || []).concat(data.tx.validating||[], data.tx.pendings||[], data.tx.errors||[]);
return csWot.extendAll(allTx, 'pubkey')
.then(function() {
- console.debug('[tx] TX and sources loaded in {0}ms'.format(Date.now() - now));
+ console.debug('[tx] Sources and {0}TX loaded in {1}ms'.format(
+ fromTime === 'pending' ? 'pending ' : '',
+ Date.now() - now));
return data;
});
})
diff --git a/www/templates/wallet/view_wallet_tx.html b/www/templates/wallet/view_wallet_tx.html
index f9d9480f2..419e89d98 100644
--- a/www/templates/wallet/view_wallet_tx.html
+++ b/www/templates/wallet/view_wallet_tx.html
@@ -152,15 +152,15 @@
<span class="gray">{{:locale:'ACCOUNT.NO_TX'|translate}}</span>
</span>
- <!-- Fix #780: do NOT use hash ash id
+ <!-- Fix #780: do NOT use hash as id
<div ng-repeat="tx in formData.tx.history track by tx.hash"
-->
- <div ng-repeat="tx in formData.tx.history track by tx.hash"
+ <div ng-repeat="tx in formData.tx.history track by tx.id"
class="item item-tx item-icon-left"
ng-include="::!tx.isUD ? 'templates/wallet/item_tx.html' : 'templates/wallet/item_ud.html'">
</div>
- <div class="item item-text-wrap text-center" ng-if="!formData.tx.canFetchMore && formData.tx.fromTime > 0">
+ <div class="item item-text-wrap text-center" ng-if="!canScrollMore && formData.tx.fromTime > 0">
<p>
<a ng-click="showMoreTx()">{{:locale:'ACCOUNT.SHOW_MORE_TX'|translate}}</a>
<span class="gray" translate="ACCOUNT.TX_FROM_DATE" translate-values="{fromTime: formData.tx.fromTime}"></span>
@@ -172,7 +172,7 @@
</div>
<ion-infinite-scroll
- ng-if="formData.tx.canFetchMore"
+ ng-if="canScrollMore"
spinner="android"
on-infinite="showMoreTx()"
distance="20%">
--
GitLab