From f4d865ba383a72eec3e7916132d0fcc839ab7b33 Mon Sep 17 00:00:00 2001
From: blavenie <benoit.lavenier@e-is.pro>
Date: Sat, 23 Sep 2017 20:49:22 +0200
Subject: [PATCH] [enh] Add statistics on documents stored in the ES node -
fix #547
---
www/index.html | 1 +
www/plugins/graph/i18n/locale-en.json | 24 ++
www/plugins/graph/i18n/locale-fr-FR.json | 24 ++
.../js/controllers/blockchain-controllers.js | 29 +--
.../js/controllers/docstats-controllers.js | 246 ++++++++++++++++++
www/plugins/graph/js/plugin.js | 3 +-
.../graph/js/services/color-services.js | 8 +-
.../graph/js/services/data-services.js | 145 ++++++++---
.../templates/docstats/graph_doc_stats.html | 22 ++
.../templates/docstats/view_doc_stats_lg.html | 33 +++
10 files changed, 476 insertions(+), 59 deletions(-)
create mode 100644 www/plugins/graph/js/controllers/docstats-controllers.js
create mode 100644 www/plugins/graph/templates/docstats/graph_doc_stats.html
create mode 100644 www/plugins/graph/templates/docstats/view_doc_stats_lg.html
diff --git a/www/index.html b/www/index.html
index 8cd93a1b..6d86e543 100644
--- a/www/index.html
+++ b/www/index.html
@@ -199,6 +199,7 @@
<script src="dist/dist_js/plugins/graph/js/controllers/network-controllers.js"></script>
<script src="dist/dist_js/plugins/graph/js/controllers/currency-controllers.js"></script>
<script src="dist/dist_js/plugins/graph/js/controllers/account-controllers.js"></script>
+<script src="dist/dist_js/plugins/graph/js/controllers/docstats-controllers.js"></script>
<!--endRemoveIf(ubuntu)-->
<!-- Map plugin -->
diff --git a/www/plugins/graph/i18n/locale-en.json b/www/plugins/graph/i18n/locale-en.json
index 85cb9b5c..5ecac739 100644
--- a/www/plugins/graph/i18n/locale-en.json
+++ b/www/plugins/graph/i18n/locale-en.json
@@ -57,6 +57,30 @@
"BLOCK_COUNT": "{{count}} blocks",
"NO_BLOCK": "No block"
}
+ },
+ "DOC_STATS": {
+ "TITLE": "Data storage statistitics",
+ "USER": {
+ "TITLE": "Number of documents linked to an account",
+ "USER_PROFILE": "User profiles utilisateur",
+ "USER_SETTINGS": "Saved settings",
+ },
+ "MESSAGE": {
+ "TITLE": "Number of documents related to the communication",
+ "MESSAGE_INBOX": "Messages in inbox",
+ "MESSAGE_OUTBOX": "Messages in outbox",
+ "INVITATION_CERTIFICATION": "Invitations to certify"
+ },
+ "SOCIAL": {
+ "TITLE": "Number of page or group",
+ "PAGE_COMMENT": "Comments",
+ "PAGE_RECORD": "Pages",
+ "GROUP_RECORD": "Groups",
+ },
+ "OTHER": {
+ "TITLE": "Other documents",
+ "HISTORY_DELETE": "Deletion of documents",
+ }
}
}
}
diff --git a/www/plugins/graph/i18n/locale-fr-FR.json b/www/plugins/graph/i18n/locale-fr-FR.json
index 37f8f92e..af9f1818 100644
--- a/www/plugins/graph/i18n/locale-fr-FR.json
+++ b/www/plugins/graph/i18n/locale-fr-FR.json
@@ -65,6 +65,30 @@
"BLOCK_COUNT": "{{count}} blocs",
"NO_BLOCK": "Aucun bloc"
}
+ },
+ "DOC_STATS": {
+ "TITLE": "Statistiques de stockage",
+ "USER": {
+ "TITLE": "Nombre de documents liés à un compte",
+ "USER_PROFILE": "Profils utilisateur",
+ "USER_SETTINGS": "Paramètres sauvegardés",
+ },
+ "MESSAGE": {
+ "TITLE": "Nombre de documents lié à la communication",
+ "MESSAGE_INBOX": "Messages en boite de réception",
+ "MESSAGE_OUTBOX": "Messages envoyés sauvegardés",
+ "INVITATION_CERTIFICATION": "Invitations à certifier"
+ },
+ "SOCIAL": {
+ "TITLE": "Nombre de pages ou groupes",
+ "PAGE_COMMENT": "Commentaires",
+ "PAGE_RECORD": "Pages",
+ "GROUP_RECORD": "Groupes",
+ },
+ "OTHER": {
+ "TITLE": "Autres documents",
+ "HISTORY_DELETE": "Suppressions de documents",
+ }
}
}
}
diff --git a/www/plugins/graph/js/controllers/blockchain-controllers.js b/www/plugins/graph/js/controllers/blockchain-controllers.js
index 601fabde..d15f2650 100644
--- a/www/plugins/graph/js/controllers/blockchain-controllers.js
+++ b/www/plugins/graph/js/controllers/blockchain-controllers.js
@@ -46,11 +46,6 @@ function GpBlockchainTxCountController($scope, $controller, $q, $state, $filter,
if (!$scope.formData.issuer && state && state.stateParams && state.stateParams.pubkey) { // Currency parameter
$scope.formData.issuer = state.stateParams.pubkey;
}
-
- // get the pubkey
- if (!$scope.formData.issuer && state && state.stateParams && state.stateParams.pubkey) { // Currency parameter
- $scope.formData.issuer = state.stateParams.pubkey;
- }
}
};
@@ -99,8 +94,7 @@ function GpBlockchainTxCountController($scope, $controller, $q, $state, $filter,
if ($scope.formData.rangeDuration != 'hour') {
$scope.data = [
result.amount,
- result.count/*,
- result.avgByBlock*/
+ result.count
];
}
else {
@@ -143,12 +137,7 @@ function GpBlockchainTxCountController($scope, $controller, $q, $state, $filter,
gridLines: {
drawOnChartArea: false
}
- }/*,
- {
- id: 'y-axis-avg',
- display: false,
- position: 'right'
- }*/
+ }
]
},
legend: {
@@ -193,19 +182,7 @@ function GpBlockchainTxCountController($scope, $controller, $q, $state, $filter,
pointHoverBackgroundColor: gpColor.rgba.gray(1),
pointHoverBorderColor: gpColor.rgba.translucent(),
pointRadius: 3
- }/*,
- {
- yAxisID: 'y-axis-avg',
- type: 'line',
- label: translations['GRAPH.BLOCKCHAIN.TX_AVG_BY_BLOCK'],
- fill: false,
- showLine: true,
- borderColor: 'rgba(0,0,0,0)',
- pointBackgroundColor: 'rgba(0,0,0,0)',
- pointBorderColor: 'rgba(0,0,0,0)',
- pointHoverBackgroundColor: 'rgba(0,0,0,0)',
- pointHoverBorderColor: 'rgba(0,0,0,0)'
- }*/
+ }
];
});
};
diff --git a/www/plugins/graph/js/controllers/docstats-controllers.js b/www/plugins/graph/js/controllers/docstats-controllers.js
new file mode 100644
index 00000000..f864345a
--- /dev/null
+++ b/www/plugins/graph/js/controllers/docstats-controllers.js
@@ -0,0 +1,246 @@
+
+angular.module('cesium.graph.docstats.controllers', ['chart.js', 'cesium.graph.services', 'cesium.graph.common.controllers'])
+
+ .config(function($stateProvider, PluginServiceProvider, csConfig) {
+ 'ngInject';
+
+ $stateProvider
+ .state('app.doc_stats_lg', {
+ url: "/data/stats?stepUnit&t&hide&scale",
+ views: {
+ 'menuContent': {
+ templateUrl: "plugins/graph/templates/docstats/view_doc_stats_lg.html"
+ }
+ }
+ });
+
+ var enable = csConfig.plugins && csConfig.plugins.es;
+ if (enable) {
+ // TODO: add buttons to link with doc stats
+ }
+ })
+
+
+ .controller('GpDocStatsCtrl', GpDocStatsController)
+;
+
+function GpDocStatsController($scope, $controller, $q, $translate, gpColor, gpData, $filter) {
+ 'ngInject';
+
+ // Initialize the super class and extend it.
+ angular.extend(this, $controller('GpCurrencyAbstractCtrl', {$scope: $scope}));
+
+ $scope.hiddenDatasets = [];
+
+ $scope.charts = [
+
+ // User count
+ {
+ id: 'user',
+ title: 'GRAPH.DOC_STATS.USER.TITLE',
+ series: [
+ {
+ key: 'user_profile',
+ label: 'GRAPH.DOC_STATS.USER.USER_PROFILE',
+ color: gpColor.rgba.royal(1),
+ pointHoverBackgroundColor: gpColor.rgba.gray(1)
+ },
+ {
+ key: 'user_settings',
+ label: 'GRAPH.DOC_STATS.USER.USER_SETTINGS',
+ color: gpColor.rgba.gray(0.5),
+ pointHoverBackgroundColor: gpColor.rgba.gray(1)
+ }
+ ]
+ },
+
+ // Message & Co.
+ {
+ id: 'message',
+ title: 'GRAPH.DOC_STATS.MESSAGE.TITLE',
+ series: [
+ {
+ key: 'message_inbox',
+ label: 'GRAPH.DOC_STATS.MESSAGE.MESSAGE_INBOX',
+ color: gpColor.rgba.royal(1),
+ pointHoverBackgroundColor: gpColor.rgba.royal(1)
+ },
+ {
+ key: 'message_outbox',
+ label: 'GRAPH.DOC_STATS.MESSAGE.MESSAGE_OUTBOX',
+ color: gpColor.rgba.calm(1),
+ pointHoverBackgroundColor: gpColor.rgba.calm(1)
+ },
+ {
+ key: 'invitation_certification',
+ label: 'GRAPH.DOC_STATS.MESSAGE.INVITATION_CERTIFICATION',
+ color: gpColor.rgba.gray(0.5),
+ pointHoverBackgroundColor: gpColor.rgba.gray(1)
+ }
+ ]
+ },
+
+ // Social Page & group
+ {
+ id: 'social',
+ title: 'GRAPH.DOC_STATS.SOCIAL.TITLE',
+ series: [
+ {
+ key: 'page_record',
+ label: 'GRAPH.DOC_STATS.SOCIAL.PAGE_RECORD',
+ color: gpColor.rgba.royal(1),
+ pointHoverBackgroundColor: gpColor.rgba.royal(1)
+ },
+ {
+ key: 'group_record',
+ label: 'GRAPH.DOC_STATS.SOCIAL.GROUP_RECORD',
+ color: gpColor.rgba.calm(1),
+ pointHoverBackgroundColor: gpColor.rgba.calm(1)
+ },
+ {
+ key: 'page_comment',
+ label: 'GRAPH.DOC_STATS.SOCIAL.PAGE_COMMENT',
+ color: gpColor.rgba.gray(0.5),
+ pointHoverBackgroundColor: gpColor.rgba.gray(1)
+ }
+ ]
+ },
+
+ // Other: deletion, doc, etc.
+ {
+ id: 'other',
+ title: 'GRAPH.DOC_STATS.OTHER.TITLE',
+ series: [
+ {
+ key: 'history_delete',
+ label: 'GRAPH.DOC_STATS.OTHER.HISTORY_DELETE',
+ color: gpColor.rgba.gray(0.5),
+ pointHoverBackgroundColor: gpColor.rgba.gray(1)
+ }
+ ]
+ }
+ ];
+
+ var formatInteger = $filter('formatInteger');
+
+ $scope.defaultChartOptions = {
+ responsive: true,
+ maintainAspectRatio: $scope.maintainAspectRatio,
+ title: {
+ display: true
+ },
+ legend: {
+ display: true,
+ onClick: $scope.onLegendClick
+ },
+ scales: {
+ yAxes: [
+ {
+ stacked: true,
+ id: 'y-axis'
+ }
+ ]
+ },
+ tooltips: {
+ enabled: true,
+ mode: 'index',
+ callbacks: {
+ label: function(tooltipItems, data) {
+ return data.datasets[tooltipItems.datasetIndex].label +
+ ': ' + formatInteger(tooltipItems.yLabel);
+ }
+ }
+ }
+ };
+
+ $scope.init = function(e, state) {
+ if (state && state.stateParams) {
+ // Manage URL parameters
+ }
+ };
+
+ $scope.load = function(updateTimePct) {
+
+
+ return $q.all([
+ // Get i18n keys (chart title, series labels, date patterns)
+ $translate($scope.charts.reduce(function(res, chart) {
+ return res.concat(chart.series.reduce(function(res, serie) {
+ return res.concat(serie.label);
+ }, [chart.title]));
+ }, [
+ 'COMMON.DATE_PATTERN',
+ 'COMMON.DATE_SHORT_PATTERN',
+ 'COMMON.DATE_MONTH_YEAR_PATTERN'
+ ])),
+
+ // get Data
+ gpData.docstat.get($scope.formData)
+ ])
+ .then(function(result) {
+ var translations = result[0];
+ var datePatterns = {
+ hour: translations['COMMON.DATE_PATTERN'],
+ day: translations['COMMON.DATE_SHORT_PATTERN'],
+ month: translations['COMMON.DATE_MONTH_YEAR_PATTERN']
+ };
+
+ result = result[1];
+ if (!result || !result.times) return; // no data
+ $scope.times = result.times;
+
+ // Labels
+ var labelPattern = datePatterns[$scope.formData.rangeDuration];
+ $scope.labels = result.times.reduce(function(res, time) {
+ return res.concat(moment.unix(time).local().format(labelPattern));
+ }, []);
+
+ // Update range options with received values
+ $scope.updateRange(result.times[0], result.times[result.times.length-1], updateTimePct);
+
+ $scope.setScale($scope.scale);
+
+ // For each chart
+ _.forEach($scope.charts, function(chart){
+
+ // Data
+ chart.data = [];
+ _.forEach(chart.series, function(serie){
+ chart.data.push(result[serie.key]||[]);
+ });
+
+ // Options (with title)
+ chart.options = angular.copy($scope.defaultChartOptions);
+ chart.options.title.text = translations[chart.title];
+
+ // Series datasets
+ chart.datasetOverride = chart.series.reduce(function(res, serie) {
+ return res.concat({
+ yAxisID: 'y-axis',
+ type: 'line',
+ label: translations[serie.label],
+ fill: true,
+ borderColor: serie.color,
+ borderWidth: 2,
+ backgroundColor: serie.color,
+ pointBackgroundColor: serie.color,
+ pointBorderColor: gpColor.rgba.white(),
+ pointHoverBackgroundColor: serie.pointHoverBackgroundColor||serie.color,
+ pointHoverBorderColor: gpColor.rgba.translucent(),
+ pointRadius: 3
+ });
+ }, []);
+ });
+ });
+
+ };
+
+ $scope.onChartClick = function(data, e, item) {
+ if (!item) return;
+ console.log('Click on item index='+ item._index);
+ var from = $scope.times[item._index];
+ var to = moment.unix(from).utc().add(1, $scope.formData.rangeDuration).unix();
+ };
+
+
+}
diff --git a/www/plugins/graph/js/plugin.js b/www/plugins/graph/js/plugin.js
index baa6ca01..d91d14b0 100644
--- a/www/plugins/graph/js/plugin.js
+++ b/www/plugins/graph/js/plugin.js
@@ -7,6 +7,7 @@ angular.module('cesium.graph.plugin', [
'cesium.graph.blockchain.controllers',
'cesium.graph.network.controllers',
'cesium.graph.currency.controllers',
- 'cesium.graph.account.controllers'
+ 'cesium.graph.account.controllers',
+ 'cesium.graph.docstats.controllers',
])
;
diff --git a/www/plugins/graph/js/services/color-services.js b/www/plugins/graph/js/services/color-services.js
index 5f7029d5..5116d296 100644
--- a/www/plugins/graph/js/services/color-services.js
+++ b/www/plugins/graph/js/services/color-services.js
@@ -46,15 +46,15 @@ angular.module('cesium.graph.color.services', [])
}
// From [0,1]
- opacity = opacity>0 && opacity|| '0.55';
+ opacity = opacity>0 && opacity|| 0.55;
var defaultStateSize = Math.round(count / 2.5/*=4 states max*/);
// Start color [r,v,b]
- var color = startColor ? angular.copy(startColor) : [255, 0, 0]; // Red
+ var color = startColor && startColor.length == 3 ? angular.copy(startColor) : [255, 0, 0]; // Red
// Colors state: 0=keep, 1=decrease, 2=keep, 3=increase
- var states = startState ? angular.copy(startState) : [0, 2, 3]; // R=keep, V=keep, B=increase
+ var states = startState && startState.length == 3 ? angular.copy(startState) : [0, 2, 3]; // R=keep, V=keep, B=increase
var steps = startColor ? [
Math.round(255 / defaultStateSize),
@@ -136,6 +136,8 @@ angular.module('cesium.graph.color.services', [])
return 'rgb(0,0,0,0)';
};
+ exports.constants = constants;
+
return exports;
})
diff --git a/www/plugins/graph/js/services/data-services.js b/www/plugins/graph/js/services/data-services.js
index 6da7ef75..c7f92796 100644
--- a/www/plugins/graph/js/services/data-services.js
+++ b/www/plugins/graph/js/services/data-services.js
@@ -1,6 +1,6 @@
angular.module('cesium.graph.data.services', ['cesium.wot.services', 'cesium.es.http.services'])
- .factory('gpData', function($rootScope, $q, $timeout, esHttp, BMA, csWot, csCache, csCurrency) {
+ .factory('gpData', function($rootScope, $q, $timeout, esHttp, BMA, csWot, csCache) {
'ngInject';
var
@@ -9,6 +9,7 @@ angular.module('cesium.graph.data.services', ['cesium.wot.services', 'cesium.es.
node: {},
wot: {},
blockchain: {},
+ docstat: {},
raw: {
block: {
search: esHttp.post('/:currency/block/_search')
@@ -21,37 +22,15 @@ angular.module('cesium.graph.data.services', ['cesium.wot.services', 'cesium.es.
},
user: {
event: esHttp.post('/user/event/_search?pretty')
+ },
+ docstat: {
+ search: esHttp.post('/docstat/record/_search')
}
},
regex: {
}
};
- function onCurrencyLoad(data, deferred) {
- deferred = deferred || $q.defer();
- var currency = data.currencies[data.currencies.length-1];
-
- if (currency.firstBlockTime) {
- deferred.resolve();
- return deferred.promise;
- }
- // Fill first block time value
- BMA.blockchain.block({block: 0})
- .then(function(block) {
- currency.firstBlockTime = block.medianTime;
- deferred.resolve();
- })
- .catch(function(err) {
- if (err && err.ucode == BMA.errorCodes.BLOCK_NOT_FOUND) {
- currency.firstBlockTime = esHttp.date.now();
- deferred.resolve();
- }
- deferred.reject(err);
- });
- return deferred.promise;
- }
-
-
function _powBase(amount, base) {
return base <= 0 ? amount : amount * Math.pow(10, base);
}
@@ -325,7 +304,7 @@ angular.module('cesium.graph.data.services', ['cesium.wot.services', 'cesium.es.
};
/**
- * Graph: "tx count"
+ * Graph: "block count"
* @param currency
* @returns {*}
*/
@@ -684,9 +663,117 @@ angular.module('cesium.graph.data.services', ['cesium.wot.services', 'cesium.es.
});
};
+ /**
+ * Graph: "statictics on ES documents"
+ * @param currency
+ * @returns {*}
+ */
+ exports.docstat.get = function(options) {
+
+ options = _initRangeOptions(options);
+
+ var jobs = [];
+
+ var from = moment.unix(options.startTime).utc().startOf(options.rangeDuration);
+ var to = moment.unix(options.endTime).utc().startOf(options.rangeDuration);
+ var ranges = [];
+ while(from.isBefore(to)) {
+
+ ranges.push({
+ from: from.unix(),
+ to: from.add(1, options.rangeDuration).unix()
+ });
+
+ // Flush if max range count, or just before loop condition end (fix #483)
+ var flush = (ranges.length === options.maxRangeSize) || !from.isBefore(to);
+ if (flush) {
+ var request = {
+ size: 0,
+ aggs: {
+ range: {
+ range: {
+ field: "time",
+ ranges: ranges
+ },
+ aggs: {
+ index : {
+ terms: {
+ field: "index",
+ size: 0
+ },
+ aggs: {
+ type: {
+ terms: {
+ field: "indexType",
+ size: 0
+ },
+ aggs: {
+ max: {
+ max: {
+ field : "count"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ };
- // register listener
-// csCurrency.api.data.on.load($rootScope, onCurrencyLoad, this);
+ // prepare next loop
+ ranges = [];
+ var indices = {};
+
+ if (jobs.length == 10) {
+ console.error('Too many parallel jobs!');
+ from = moment.unix(options.endTime).utc(); // stop while
+ }
+ else {
+ jobs.push(
+ exports.raw.docstat.search(request)
+ .then(function (res) {
+ var aggs = res.aggregations;
+ return (aggs.range && aggs.range.buckets || []).reduce(function (res, agg) {
+ var item = {
+ from: agg.from,
+ to: agg.to
+ };
+ _.forEach(agg.index && agg.index.buckets || [], function (agg) {
+ var index = agg.key;
+ _.forEach(agg.type && agg.type.buckets || [], function (agg) {
+ var key = (index + '_' + agg.key);
+ item[key] = agg.max.value;
+ if (!indices[key]) indices[key] = true;
+ });
+ });
+ return res.concat(item);
+ }, []);
+ })
+ );
+ }
+ }
+ } // loop
+
+ return $q.all(jobs)
+ .then(function(res) {
+ res = res.reduce(function(res, hits){
+ if (!hits || !hits.length) return res;
+ return res.concat(hits);
+ }, []);
+
+ res = _.sortBy(res, 'from');
+
+ return _.keys(indices).reduce(function(series, index) {
+ series[index] = _.pluck(res, index);
+ return series;
+ }, {
+ times: _.pluck(res, 'from')
+ });
+ });
+ };
return exports;
})
diff --git a/www/plugins/graph/templates/docstats/graph_doc_stats.html b/www/plugins/graph/templates/docstats/graph_doc_stats.html
new file mode 100644
index 00000000..724c4418
--- /dev/null
+++ b/www/plugins/graph/templates/docstats/graph_doc_stats.html
@@ -0,0 +1,22 @@
+
+ <!-- graphs button bar -->
+ <div class="button-bar-inline "
+ style="top: 33px; margin-top:-33px; position: relative;">
+ <button
+ class="button button-stable button-clear no-padding-xs pull-right"
+ ng-click="showActionsPopover($event)">
+ <i class="icon ion-navicon-round"></i>
+ </button>
+ </div>
+
+ <canvas id="docstats-chart-{{chart.id}}"
+ class="chart-line"
+ height="{{height}}"
+ width="{{width}}"
+ chart-data="chart.data"
+ chart-labels="labels"
+ chart-dataset-override="chart.datasetOverride"
+ chart-options="chart.options">
+ </canvas>
+
+ <ng-include src="'plugins/graph/templates/common/graph_range_bar.html'"></ng-include>
diff --git a/www/plugins/graph/templates/docstats/view_doc_stats_lg.html b/www/plugins/graph/templates/docstats/view_doc_stats_lg.html
new file mode 100644
index 00000000..e18218d7
--- /dev/null
+++ b/www/plugins/graph/templates/docstats/view_doc_stats_lg.html
@@ -0,0 +1,33 @@
+<ion-view left-buttons="leftButtons"
+ cache-view="false">
+ <ion-nav-title>
+ {{'GRAPH.DOC_STATS.TITLE' | translate}}
+ </ion-nav-title>
+
+ <ion-content scroll="true" class="padding" >
+
+
+
+ <div class="list" >
+
+ <!-- Doc stat -->
+ <ng-controller ng-controller="GpDocStatsCtrl" >
+
+ <div class="center padding" ng-if="loading">
+ <ion-spinner icon="android"></ion-spinner>
+ </div>
+
+ <div class="item no-padding-xs" ng-if="!loading"
+ ng-repeat="chart in charts"
+ ng-include="'plugins/graph/templates/docstats/graph_doc_stats.html'"
+ ng-init="setSize(250, 1000)">
+ </div>
+ </ng-controller>
+
+
+
+ </div>
+
+ </ion-content>
+
+</ion-view>
--
GitLab