[enh] Add statistics on documents stored in the ES node - fix #547

www/index.html

@@ -199,6 +199,7 @@
 <script src="dist/dist_js/plugins/graph/js/controllers/network-controllers.js"></script>
 <script src="dist/dist_js/plugins/graph/js/controllers/currency-controllers.js"></script>
 <script src="dist/dist_js/plugins/graph/js/controllers/account-controllers.js"></script>
+<script src="dist/dist_js/plugins/graph/js/controllers/docstats-controllers.js"></script>
 <!--endRemoveIf(ubuntu)-->
 
 <!-- Map plugin -->

www/plugins/graph/i18n/locale-en.json

@@ -57,6 +57,30 @@
         "BLOCK_COUNT": "{{count}} blocks",
         "NO_BLOCK": "No block"
       }
+    },
+    "DOC_STATS": {
+      "TITLE": "Data storage statistitics",
+      "USER": {
+        "TITLE": "Number of documents linked to an account",
+        "USER_PROFILE": "User profiles utilisateur",
+        "USER_SETTINGS": "Saved settings",
+      },
+      "MESSAGE": {
+        "TITLE": "Number of documents related to the communication",
+        "MESSAGE_INBOX": "Messages in inbox",
+        "MESSAGE_OUTBOX": "Messages in outbox",
+        "INVITATION_CERTIFICATION": "Invitations to certify"
+      },
+      "SOCIAL": {
+        "TITLE": "Number of page or group",
+        "PAGE_COMMENT": "Comments",
+        "PAGE_RECORD": "Pages",
+        "GROUP_RECORD": "Groups",
+      },
+      "OTHER": {
+        "TITLE": "Other documents",
+        "HISTORY_DELETE": "Deletion of documents",
+      }
     }
   }
 }

www/plugins/graph/i18n/locale-fr-FR.json

@@ -65,6 +65,30 @@
         "BLOCK_COUNT": "{{count}} blocs",
         "NO_BLOCK": "Aucun bloc"
       }
+    },
+    "DOC_STATS": {
+      "TITLE": "Statistiques de stockage",
+      "USER": {
+        "TITLE": "Nombre de documents liés à un compte",
+        "USER_PROFILE": "Profils utilisateur",
+        "USER_SETTINGS": "Paramètres sauvegardés",
+      },
+      "MESSAGE": {
+        "TITLE": "Nombre de documents lié à la communication",
+        "MESSAGE_INBOX": "Messages en boite de réception",
+        "MESSAGE_OUTBOX": "Messages envoyés sauvegardés",
+        "INVITATION_CERTIFICATION": "Invitations à certifier"
+      },
+      "SOCIAL": {
+        "TITLE": "Nombre de pages ou groupes",
+        "PAGE_COMMENT": "Commentaires",
+        "PAGE_RECORD": "Pages",
+        "GROUP_RECORD": "Groupes",
+      },
+      "OTHER": {
+        "TITLE": "Autres documents",
+        "HISTORY_DELETE": "Suppressions de documents",
+      }
     }
   }
 }

www/plugins/graph/js/controllers/blockchain-controllers.js

@@ -46,11 +46,6 @@ function GpBlockchainTxCountController($scope, $controller, $q, $state, $filter,
       if (!$scope.formData.issuer && state && state.stateParams && state.stateParams.pubkey) { // Currency parameter
         $scope.formData.issuer = state.stateParams.pubkey;
       }
-
-      // get the pubkey
-      if (!$scope.formData.issuer && state && state.stateParams && state.stateParams.pubkey) { // Currency parameter
-        $scope.formData.issuer = state.stateParams.pubkey;
-      }
     }
   };
 
@@ -99,8 +94,7 @@ function GpBlockchainTxCountController($scope, $controller, $q, $state, $filter,
         if ($scope.formData.rangeDuration != 'hour') {
           $scope.data = [
             result.amount,
-            result.count/*,
-            result.avgByBlock*/
+            result.count
           ];
         }
         else {
@@ -143,12 +137,7 @@ function GpBlockchainTxCountController($scope, $controller, $q, $state, $filter,
                 gridLines: {
                   drawOnChartArea: false
                 }
-              }/*,
-              {
-                id: 'y-axis-avg',
-                display: false,
-                position: 'right'
-              }*/
+              }
             ]
           },
           legend: {
@@ -193,19 +182,7 @@ function GpBlockchainTxCountController($scope, $controller, $q, $state, $filter,
             pointHoverBackgroundColor: gpColor.rgba.gray(1),
             pointHoverBorderColor: gpColor.rgba.translucent(),
             pointRadius: 3
-          }/*,
-          {
-            yAxisID: 'y-axis-avg',
-            type: 'line',
-            label: translations['GRAPH.BLOCKCHAIN.TX_AVG_BY_BLOCK'],
-            fill: false,
-            showLine: true,
-            borderColor: 'rgba(0,0,0,0)',
-            pointBackgroundColor: 'rgba(0,0,0,0)',
-            pointBorderColor: 'rgba(0,0,0,0)',
-            pointHoverBackgroundColor: 'rgba(0,0,0,0)',
-            pointHoverBorderColor: 'rgba(0,0,0,0)'
-          }*/
+          }
         ];
       });
   };

www/plugins/graph/js/controllers/docstats-controllers.js

+
+angular.module('cesium.graph.docstats.controllers', ['chart.js', 'cesium.graph.services', 'cesium.graph.common.controllers'])
+
+  .config(function($stateProvider, PluginServiceProvider, csConfig) {
+    'ngInject';
+
+    $stateProvider
+      .state('app.doc_stats_lg', {
+        url: "/data/stats?stepUnit&t&hide&scale",
+        views: {
+          'menuContent': {
+            templateUrl: "plugins/graph/templates/docstats/view_doc_stats_lg.html"
+          }
+        }
+      });
+
+    var enable = csConfig.plugins && csConfig.plugins.es;
+    if (enable) {
+      // TODO: add buttons to link with doc stats
+    }
+  })
+
+
+  .controller('GpDocStatsCtrl', GpDocStatsController)
+;
+
+function GpDocStatsController($scope, $controller, $q, $translate, gpColor, gpData, $filter) {
+  'ngInject';
+
+  // Initialize the super class and extend it.
+  angular.extend(this, $controller('GpCurrencyAbstractCtrl', {$scope: $scope}));
+
+  $scope.hiddenDatasets = [];
+
+  $scope.charts = [
+
+    // User count
+    {
+      id: 'user',
+      title: 'GRAPH.DOC_STATS.USER.TITLE',
+      series: [
+        {
+          key: 'user_profile',
+          label: 'GRAPH.DOC_STATS.USER.USER_PROFILE',
+          color: gpColor.rgba.royal(1),
+          pointHoverBackgroundColor: gpColor.rgba.gray(1)
+        },
+        {
+          key: 'user_settings',
+          label: 'GRAPH.DOC_STATS.USER.USER_SETTINGS',
+          color: gpColor.rgba.gray(0.5),
+          pointHoverBackgroundColor: gpColor.rgba.gray(1)
+        }
+      ]
+    },
+
+    // Message & Co.
+    {
+      id: 'message',
+      title: 'GRAPH.DOC_STATS.MESSAGE.TITLE',
+      series: [
+        {
+          key: 'message_inbox',
+          label: 'GRAPH.DOC_STATS.MESSAGE.MESSAGE_INBOX',
+          color: gpColor.rgba.royal(1),
+          pointHoverBackgroundColor: gpColor.rgba.royal(1)
+        },
+        {
+          key: 'message_outbox',
+          label: 'GRAPH.DOC_STATS.MESSAGE.MESSAGE_OUTBOX',
+          color: gpColor.rgba.calm(1),
+          pointHoverBackgroundColor: gpColor.rgba.calm(1)
+        },
+        {
+          key: 'invitation_certification',
+          label: 'GRAPH.DOC_STATS.MESSAGE.INVITATION_CERTIFICATION',
+          color: gpColor.rgba.gray(0.5),
+          pointHoverBackgroundColor: gpColor.rgba.gray(1)
+        }
+      ]
+    },
+
+    // Social Page & group
+    {
+      id: 'social',
+      title: 'GRAPH.DOC_STATS.SOCIAL.TITLE',
+      series: [
+        {
+          key: 'page_record',
+          label: 'GRAPH.DOC_STATS.SOCIAL.PAGE_RECORD',
+          color: gpColor.rgba.royal(1),
+          pointHoverBackgroundColor: gpColor.rgba.royal(1)
+        },
+        {
+          key: 'group_record',
+          label: 'GRAPH.DOC_STATS.SOCIAL.GROUP_RECORD',
+          color: gpColor.rgba.calm(1),
+          pointHoverBackgroundColor: gpColor.rgba.calm(1)
+        },
+        {
+          key: 'page_comment',
+          label: 'GRAPH.DOC_STATS.SOCIAL.PAGE_COMMENT',
+          color: gpColor.rgba.gray(0.5),
+          pointHoverBackgroundColor: gpColor.rgba.gray(1)
+        }
+      ]
+    },
+
+    // Other: deletion, doc, etc.
+    {
+      id: 'other',
+      title: 'GRAPH.DOC_STATS.OTHER.TITLE',
+      series: [
+        {
+          key: 'history_delete',
+          label: 'GRAPH.DOC_STATS.OTHER.HISTORY_DELETE',
+          color: gpColor.rgba.gray(0.5),
+          pointHoverBackgroundColor: gpColor.rgba.gray(1)
+        }
+      ]
+    }
+  ];
+
+  var formatInteger = $filter('formatInteger');
+
+  $scope.defaultChartOptions = {
+    responsive: true,
+    maintainAspectRatio: $scope.maintainAspectRatio,
+    title: {
+      display: true
+    },
+    legend: {
+      display: true,
+      onClick: $scope.onLegendClick
+    },
+    scales: {
+      yAxes: [
+        {
+          stacked: true,
+          id: 'y-axis'
+        }
+      ]
+    },
+    tooltips: {
+      enabled: true,
+      mode: 'index',
+      callbacks: {
+        label: function(tooltipItems, data) {
+          return data.datasets[tooltipItems.datasetIndex].label +
+            ': ' + formatInteger(tooltipItems.yLabel);
+        }
+      }
+    }
+  };
+
+  $scope.init = function(e, state) {
+    if (state && state.stateParams) {
+      // Manage URL parameters
+    }
+  };
+
+  $scope.load = function(updateTimePct) {
+
+
+    return $q.all([
+      // Get i18n keys (chart title, series labels, date patterns)
+      $translate($scope.charts.reduce(function(res, chart) {
+        return res.concat(chart.series.reduce(function(res, serie) {
+          return res.concat(serie.label);
+        }, [chart.title]));
+      }, [
+        'COMMON.DATE_PATTERN',
+        'COMMON.DATE_SHORT_PATTERN',
+        'COMMON.DATE_MONTH_YEAR_PATTERN'
+      ])),
+
+      // get Data
+      gpData.docstat.get($scope.formData)
+    ])
+    .then(function(result) {
+      var translations = result[0];
+      var datePatterns = {
+        hour: translations['COMMON.DATE_PATTERN'],
+        day: translations['COMMON.DATE_SHORT_PATTERN'],
+        month: translations['COMMON.DATE_MONTH_YEAR_PATTERN']
+      };
+
+      result = result[1];
+      if (!result || !result.times) return; // no data
+      $scope.times = result.times;
+
+      // Labels
+      var labelPattern = datePatterns[$scope.formData.rangeDuration];
+      $scope.labels = result.times.reduce(function(res, time) {
+        return res.concat(moment.unix(time).local().format(labelPattern));
+      }, []);
+
+      // Update range options with received values
+      $scope.updateRange(result.times[0], result.times[result.times.length-1], updateTimePct);
+
+      $scope.setScale($scope.scale);
+
+      // For each chart
+      _.forEach($scope.charts, function(chart){
+
+        // Data
+        chart.data = [];
+        _.forEach(chart.series, function(serie){
+          chart.data.push(result[serie.key]||[]);
+        });
+
+        // Options (with title)
+        chart.options = angular.copy($scope.defaultChartOptions);
+        chart.options.title.text = translations[chart.title];
+
+        // Series datasets
+        chart.datasetOverride = chart.series.reduce(function(res, serie) {
+          return res.concat({
+            yAxisID: 'y-axis',
+            type: 'line',
+            label: translations[serie.label],
+            fill: true,
+            borderColor: serie.color,
+            borderWidth: 2,
+            backgroundColor: serie.color,
+            pointBackgroundColor: serie.color,
+            pointBorderColor: gpColor.rgba.white(),
+            pointHoverBackgroundColor: serie.pointHoverBackgroundColor||serie.color,
+            pointHoverBorderColor: gpColor.rgba.translucent(),
+            pointRadius: 3
+          });
+        }, []);
+      });
+    });
+
+  };
+
+  $scope.onChartClick = function(data, e, item) {
+    if (!item) return;
+    console.log('Click on item index='+ item._index);
+    var from = $scope.times[item._index];
+    var to = moment.unix(from).utc().add(1, $scope.formData.rangeDuration).unix();
+  };
+
+
+}

www/plugins/graph/js/plugin.js

@@ -7,6 +7,7 @@ angular.module('cesium.graph.plugin', [
     'cesium.graph.blockchain.controllers',
     'cesium.graph.network.controllers',
     'cesium.graph.currency.controllers',
-    'cesium.graph.account.controllers'
+    'cesium.graph.account.controllers',
+    'cesium.graph.docstats.controllers',
   ])
 ;

www/plugins/graph/js/services/color-services.js

@@ -46,15 +46,15 @@ angular.module('cesium.graph.color.services', [])
       }
 
       // From [0,1]
-      opacity = opacity>0 && opacity|| '0.55';
+      opacity = opacity>0 && opacity|| 0.55;
 
       var defaultStateSize = Math.round(count / 2.5/*=4 states max*/);
 
       // Start color [r,v,b]
-      var color = startColor ? angular.copy(startColor) : [255, 0, 0]; // Red
+      var color = startColor && startColor.length == 3 ? angular.copy(startColor) : [255, 0, 0]; // Red
 
       // Colors state: 0=keep, 1=decrease, 2=keep, 3=increase
-      var states = startState ? angular.copy(startState) : [0, 2, 3]; // R=keep, V=keep, B=increase
+      var states = startState && startState.length == 3 ? angular.copy(startState) : [0, 2, 3]; // R=keep, V=keep, B=increase
 
       var steps = startColor ? [
         Math.round(255 / defaultStateSize),
@@ -136,6 +136,8 @@ angular.module('cesium.graph.color.services', [])
       return 'rgb(0,0,0,0)';
     };
 
+    exports.constants = constants;
+
     return exports;
   })
 

www/plugins/graph/js/services/data-services.js

 angular.module('cesium.graph.data.services', ['cesium.wot.services', 'cesium.es.http.services'])
 
-  .factory('gpData', function($rootScope, $q, $timeout, esHttp, BMA, csWot, csCache, csCurrency) {
+  .factory('gpData', function($rootScope, $q, $timeout, esHttp, BMA, csWot, csCache) {
     'ngInject';
 
     var
@@ -9,6 +9,7 @@ angular.module('cesium.graph.data.services', ['cesium.wot.services', 'cesium.es.
         node: {},
         wot: {},
         blockchain: {},
+        docstat: {},
         raw: {
           block: {
             search: esHttp.post('/:currency/block/_search')
@@ -21,37 +22,15 @@ angular.module('cesium.graph.data.services', ['cesium.wot.services', 'cesium.es.
           },
           user: {
             event: esHttp.post('/user/event/_search?pretty')
+          },
+          docstat: {
+            search: esHttp.post('/docstat/record/_search')
           }
         },
         regex: {
         }
       };
 
-      function onCurrencyLoad(data, deferred) {
-        deferred = deferred || $q.defer();
-        var currency = data.currencies[data.currencies.length-1];
-
-        if (currency.firstBlockTime) {
-          deferred.resolve();
-          return deferred.promise;
-        }
-        // Fill first block time value
-        BMA.blockchain.block({block: 0})
-            .then(function(block) {
-              currency.firstBlockTime = block.medianTime;
-              deferred.resolve();
-            })
-            .catch(function(err) {
-              if (err && err.ucode == BMA.errorCodes.BLOCK_NOT_FOUND) {
-                currency.firstBlockTime = esHttp.date.now();
-                deferred.resolve();
-              }
-              deferred.reject(err);
-            });
-        return deferred.promise;
-      }
-
-
     function _powBase(amount, base) {
       return base <= 0 ? amount : amount * Math.pow(10, base);
     }
@@ -325,7 +304,7 @@ angular.module('cesium.graph.data.services', ['cesium.wot.services', 'cesium.es.
     };
 
     /**
-     * Graph: "tx count"
+     * Graph: "block count"
      * @param currency
      * @returns {*}
      */
@@ -684,9 +663,117 @@ angular.module('cesium.graph.data.services', ['cesium.wot.services', 'cesium.es.
         });
     };
 
+    /**
+     * Graph: "statictics on ES documents"
+     * @param currency
+     * @returns {*}
+     */
+    exports.docstat.get = function(options) {
+
+      options = _initRangeOptions(options);
 
-    // register listener
-//    csCurrency.api.data.on.load($rootScope, onCurrencyLoad, this);
+      var jobs = [];
+
+      var from = moment.unix(options.startTime).utc().startOf(options.rangeDuration);
+      var to = moment.unix(options.endTime).utc().startOf(options.rangeDuration);
+      var ranges = [];
+      while(from.isBefore(to)) {
+
+        ranges.push({
+          from: from.unix(),
+          to: from.add(1, options.rangeDuration).unix()
+        });
+
+        // Flush if max range count, or just before loop condition end (fix #483)
+        var flush = (ranges.length === options.maxRangeSize) || !from.isBefore(to);
+        if (flush) {
+          var request = {
+            size: 0,
+            aggs: {
+              range: {
+                range: {
+                  field: "time",
+                  ranges: ranges
+                },
+                aggs: {
+                  index : {
+                    terms: {
+                      field: "index",
+                      size: 0
+                    },
+                    aggs: {
+                      type: {
+                        terms: {
+                          field: "indexType",
+                          size: 0
+                        },
+                        aggs: {
+                          max: {
+                            max: {
+                              field : "count"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+
+          };
+
+          // prepare next loop
+          ranges = [];
+          var indices = {};
+
+          if (jobs.length == 10) {
+            console.error('Too many parallel jobs!');
+            from = moment.unix(options.endTime).utc(); // stop while
+          }
+          else {
+            jobs.push(
+              exports.raw.docstat.search(request)
+                .then(function (res) {
+                  var aggs = res.aggregations;
+                  return (aggs.range && aggs.range.buckets || []).reduce(function (res, agg) {
+                    var item = {
+                      from: agg.from,
+                      to: agg.to
+                    };
+                    _.forEach(agg.index && agg.index.buckets || [], function (agg) {
+                      var index = agg.key;
+                      _.forEach(agg.type && agg.type.buckets || [], function (agg) {
+                        var key = (index + '_' + agg.key);
+                        item[key] = agg.max.value;
+                        if (!indices[key]) indices[key] = true;
+                      });
+                    });
+                    return res.concat(item);
+                  }, []);
+                })
+            );
+          }
+        }
+      } // loop
+
+      return $q.all(jobs)
+        .then(function(res) {
+          res = res.reduce(function(res, hits){
+            if (!hits || !hits.length) return res;
+            return res.concat(hits);
+          }, []);
+
+          res = _.sortBy(res, 'from');
+
+          return _.keys(indices).reduce(function(series, index) {
+            series[index] = _.pluck(res, index);
+            return series;
+          }, {
+            times: _.pluck(res, 'from')
+          });
+        });
+    };
 
     return exports;
   })

www/plugins/graph/templates/docstats/graph_doc_stats.html

+
+  <!-- graphs button bar -->
+  <div class="button-bar-inline "
+       style="top: 33px; margin-top:-33px; position: relative;">
+    <button
+      class="button button-stable button-clear no-padding-xs pull-right"
+      ng-click="showActionsPopover($event)">
+      <i class="icon ion-navicon-round"></i>
+    </button>
+  </div>
+
+  <canvas id="docstats-chart-{{chart.id}}"
+          class="chart-line"
+          height="{{height}}"
+          width="{{width}}"
+          chart-data="chart.data"
+          chart-labels="labels"
+          chart-dataset-override="chart.datasetOverride"
+          chart-options="chart.options">
+  </canvas>
+
+  <ng-include src="'plugins/graph/templates/common/graph_range_bar.html'"></ng-include>

www/plugins/graph/templates/docstats/view_doc_stats_lg.html

+<ion-view left-buttons="leftButtons"
+          cache-view="false">
+  <ion-nav-title>
+    {{'GRAPH.DOC_STATS.TITLE' | translate}}
+  </ion-nav-title>
+
+  <ion-content scroll="true" class="padding" >
+
+
+
+    <div class="list" >
+
+      <!-- Doc stat -->
+      <ng-controller ng-controller="GpDocStatsCtrl" >
+
+        <div class="center padding" ng-if="loading">
+          <ion-spinner icon="android"></ion-spinner>
+        </div>
+
+        <div class="item no-padding-xs" ng-if="!loading"
+             ng-repeat="chart in charts"
+             ng-include="'plugins/graph/templates/docstats/graph_doc_stats.html'"
+             ng-init="setSize(250, 1000)">
+        </div>
+      </ng-controller>
+
+
+
+    </div>
+
+  </ion-content>
+
+</ion-view>