Skip to content
Snippets Groups Projects
Commit 1c60544d authored by Benoit Lavenier's avatar Benoit Lavenier
Browse files

[fix] Fix Wot map loading (was exceed 10000 profiles, so force load using geo slice)

parent 5ecea0e8
No related branches found
No related tags found
No related merge requests found
Pipeline #15577 failed
Stage: build
    Stage: alt_build
      Hide whitespace changes
      Inline Side-by-side
      www/plugins/map/js/controllers/wot-controllers.js
      + 18
      − 24
      View file @ 1c60544d
      ...@@ -156,8 +156,10 @@ function MapWotViewController($scope, $filter, $templateCache, $interpolate, $ti ...@@ -156,8 +156,10 @@ function MapWotViewController($scope, $filter, $templateCache, $interpolate, $ti
      $scope.map.center.zoom = parseInt(cPart[2]); $scope.map.center.zoom = parseInt(cPart[2]);
      } }
      var lastLocationUpdate = Date.now();
      $scope.$watch("map.center", function() { $scope.$watch("map.center", function() {
      if (!$scope.loading) { if (!$scope.loading && (Date.now() - lastLocationUpdate) > 5000) {
      lastLocationUpdate = Date.now();
      return $timeout(function() { return $timeout(function() {
      $scope.updateLocationHref(); $scope.updateLocationHref();
      }, 300); }, 300);
      ...@@ -279,22 +281,19 @@ function MapWotViewController($scope, $filter, $templateCache, $interpolate, $ti ...@@ -279,22 +281,19 @@ function MapWotViewController($scope, $filter, $templateCache, $interpolate, $ti
      .then($scope.load); .then($scope.load);
      } }
      console.info('[map] Leaflet map is ready: starting loading data...');
      $scope.loading = true; $scope.loading = true;
      // Show loading indicator // Show loading indicator
      map.fire('dataloading'); map.fire('dataloading');
      // add bounding box
      var options = { var options = {
      fields: { bounds: angular.copy($scope.map.bounds)
      description: $scope.enableDescription
      }
      }; };
      delete options.bounds.options;
      // add bounding box var formatPubkey = $filter('formatPubkey');
      if ($scope.map.bounds) { var markerTemplate = $templateCache.get('plugins/map/templates/wot/popup_marker.html');
      // FIXME - this is not working well
      //options.bounds = angular.copy($scope.map.bounds);
      //delete options.bounds.options;
      }
      // Load wot data, from service // Load wot data, from service
      return mapWot.load(options) return mapWot.load(options)
      ...@@ -302,26 +301,24 @@ function MapWotViewController($scope, $filter, $templateCache, $interpolate, $ti ...@@ -302,26 +301,24 @@ function MapWotViewController($scope, $filter, $templateCache, $interpolate, $ti
      .then(function(res) { .then(function(res) {
      var markers = {}; var markers = {};
      if (res && res.length) { if (res && res.length) {
      var formatPubkey = $filter('formatPubkey');
      var markerTemplate = $templateCache.get('plugins/map/templates/wot/popup_marker.html');
      _.forEach(res, function (hit) { _.forEach(res, function (hit) {
      var type = hit.pending ? 'pending' : (hit.uid ? 'member' : 'wallet'); var type = hit.pending ? 'pending' : (hit.uid ? 'member' : 'wallet');
      var id = hit.index + '_' + (hit.id || (hit.uid ? (hit.uid + ':' + hit.pubkey) : hit.pubkey)).replace(/-/g, '_'); var id = hit.index + '_' + (hit.id || (hit.uid ? (hit.uid + ':' + hit.pubkey) : hit.pubkey)).replace(/-/g, '_');
      var title = hit.name + ' | ' + formatPubkey(hit.pubkey); var title = hit.name + ' | ' + formatPubkey(hit.pubkey);
      var marker = { var marker = {
      id: id,
      message: markerTemplate,
      title: title,
      layer: type, layer: type,
      icon: icons[type], icon: icons[type],
      opacity: hit.uid ? 1 : 0.7, opacity: hit.uid ? 1 : 0.7,
      title: title,
      lat: hit.geoPoint.lat, lat: hit.geoPoint.lat,
      lng: hit.geoPoint.lon, lng: hit.geoPoint.lon,
      focus: false,
      getMessageScope: function () { getMessageScope: function () {
      //console.debug('[map] Loading marker ' + title + "..."); console.debug('[map] Loading marker ' + title + "...");
      var markerScope = $scope.$new(); var markerScope = $scope.$new();
      markerScope.loadingMarker = true; markerScope.loadingMarker = true;
      markerScope.formData = {}; markerScope.formData = {};
      ...@@ -331,15 +328,12 @@ function MapWotViewController($scope, $filter, $templateCache, $interpolate, $ti ...@@ -331,15 +328,12 @@ function MapWotViewController($scope, $filter, $templateCache, $interpolate, $ti
      uid: hit.uid, uid: hit.uid,
      name: title, name: title,
      profile: hit, profile: hit,
      isMember: type == 'member' isMember: type === 'member'
      }; };
      markerScope.loadingMarker = false; markerScope.loadingMarker = false;
      }); });
      return markerScope; return markerScope;
      }, }
      focus: false,
      message: markerTemplate,
      id: id
      }; };
      markers[id] = marker; markers[id] = marker;
      }); });
      ...@@ -369,10 +363,10 @@ function MapWotViewController($scope, $filter, $templateCache, $interpolate, $ti ...@@ -369,10 +363,10 @@ function MapWotViewController($scope, $filter, $templateCache, $interpolate, $ti
      $scope.updateLocationHref = function(centerHash) { $scope.updateLocationHref = function(centerHash) {
      // removeIf(device) // removeIf(device)
      var params = $location.search() || {}; var params = $location.search() || {};
      if (!params.c || !MapUtils.center.isDefault($scope.map.center)) { /*if (!params.c || !MapUtils.center.isDefault($scope.map.center)) {
      centerHash = centerHash || '{0}:{1}:{2}'.format($scope.map.center.lat.toFixed(4), $scope.map.center.lng.toFixed(4), $scope.map.center.zoom); centerHash = centerHash || '{0}:{1}:{2}'.format($scope.map.center.lat.toFixed(4), $scope.map.center.lng.toFixed(4), $scope.map.center.zoom);
      $location.search({c: centerHash}).replace(); $location.search({c: centerHash}).replace();
      } }*/
      // endRemoveIf(device) // endRemoveIf(device)
      }; };
      ......
      www/plugins/map/js/services/utils-services.js
      + 1
      − 1
      View file @ 1c60544d
      ...@@ -13,7 +13,7 @@ angular.module('cesium.map.utils.services', ['cesium.services', 'ui-leaflet']) ...@@ -13,7 +13,7 @@ angular.module('cesium.map.utils.services', ['cesium.services', 'ui-leaflet'])
      lat: 46.5588603, lng: 4.229736328124999, zoom: 6 lat: 46.5588603, lng: 4.229736328124999, zoom: 6
      } }
      }, },
      LOCALIZE_ZOOM: 15 LOCALIZE_ZOOM: 16
      }, },
      data = { data = {
      cache: {} cache: {}
      ......
      www/plugins/map/js/services/wot-services.js
      + 112
      − 106
      View file @ 1c60544d
      ...@@ -22,75 +22,92 @@ angular.module('cesium.map.wot.services', ['cesium.services']) ...@@ -22,75 +22,92 @@ angular.module('cesium.map.wot.services', ['cesium.services'])
      } }
      }; };
      /**
      * Convert Leaflet BBox into ES BBox query
      * see https://www.elastic.co/guide/en/elasticsearch/reference/2.4/geo-point.html
      * @param bounds
      */
      function createFilterQuery(options) { function createFilterQuery(options) {
      options = options || {}; var bounds = options && options.bounds;
      var query = { if (!bounds) throw new Error('Missing options.bounds!');
      bool: {}
      };
      // Limit to profile with geo point var minLon = Math.min(bounds.northEast.lng, bounds.southWest.lng),
      if (options.searchAddress) { minLat = Math.min(bounds.northEast.lat, bounds.southWest.lat),
      query.bool.should = [ maxLon = Math.max(bounds.northEast.lng, bounds.southWest.lng),
      maxLat = Math.max(bounds.northEast.lat, bounds.southWest.lat);
      return {constant_score: {
      filter: [
      {exists: {field: "geoPoint"}}, {exists: {field: "geoPoint"}},
      {exists: {field: "city"}} {geo_bounding_box: {geoPoint: {
      ]; top_left: {
      } lat: maxLat,
      else { lon: minLon
      query.bool.must= [ },
      {exists: {field: "geoPoint"}} bottom_right: {
      ]; lat: minLat,
      } lon: maxLon
      }
      }
      }}
      ]
      }};
      }
      // Filter on bounding box function createSliceQueries(options, total) {
      // see https://www.elastic.co/guide/en/elasticsearch/reference/2.4/geo-point.html var bounds = options && options.bounds;
      if (options.bounds && options.bounds.northEast && options.bounds.southWest) { if (!bounds) throw new Error('Missing options.bounds!');
      var boundingBox = {
      "geoPoint" : { var minLon = Math.min(bounds.northEast.lng, bounds.southWest.lng),
      "top_left" : { minLat = Math.min(bounds.northEast.lat, bounds.southWest.lat),
      "lat" : Math.max(Math.min(options.bounds.northEast.lat, 90), -90), maxLon = Math.max(bounds.northEast.lng, bounds.southWest.lng),
      "lon" : Math.max(Math.min(options.bounds.southWest.lng, 180), -180) maxLat = Math.max(bounds.northEast.lat, bounds.southWest.lat);
      },
      "bottom_right" : { var queries = [];
      "lat" : Math.max(Math.min(options.bounds.southWest.lat, 90), -90), var lonStep = (maxLon - minLon) / 5;
      "lon" : Math.max(Math.min(options.bounds.northEast.lng, 180), -180) var latStep = (maxLat - minLat) / 5;
      } for (var lon = minLon; lon < maxLon; lon += lonStep) {
      } for (var lat = minLat; lat < maxLat; lat += latStep) {
      }; queries.push({constant_score: {
      console.debug("[map] [wot] Filtering on bounds: ", options.bounds); filter: [
      query.bool.must = query.bool.must || []; {exists: {field: "geoPoint"}},
      query.bool.must.push({geo_bounding_box: boundingBox}); {geo_bounding_box: {
      geoPoint: {
      top_left: {
      lat: lat + latStep,
      lon: lon
      },
      bottom_right: {
      lat: lat,
      lon: lon + lonStep
      }
      }
      }}
      ]
      }});
      }
      } }
      return query; return queries;
      } }
      function load(options) { function load(options) {
      options = options || {}; options = options || {};
      options.from = options.from || 0; options.from = options.from || 0;
      options.size = options.size || constants.DEFAULT_LOAD_SIZE; options.size = options.size || constants.DEFAULT_LOAD_SIZE;
      options.searchAddress = esGeo.google.isEnable() && (angular.isDefined(options.searchAddress) ? options.searchAddress : true);
      options.fields = options.fields || {}; options.fields = options.fields || {};
      options.fields.description = angular.isDefined(options.fields.description) ? options.fields.description : false;
      var request = { var countRequest = {
      query: createFilterQuery(options), query: createFilterQuery(options),
      from: 0, size: 0
      size: options.size,
      _source: options.fields.description ? fields.profile.concat("description") : fields.profile
      }; };
      var mixedSearch = false; var mixedSearch = false;
      /*var mixedSearch = esSettings.wot.isMixedSearchEnable();
      if (mixedSearch) {
      // add special fields for page and group
      request._source = request._source.concat(["type", "pubkey", "issuer", "category"]);
      console.debug("[ES] [map] Mixed search: enable");
      }*/
      var search = mixedSearch ? that.raw.profile.mixedSearch : that.raw.profile.search; var search = mixedSearch ? that.raw.profile.mixedSearch : that.raw.profile.search;
      return $q.all([ return $q.all([
      search(request), search(countRequest),
      BMA.wot.member.uids(), BMA.wot.member.uids(),
      BMA.wot.member.pending() BMA.wot.member.pending()
      .then(function(res) { .then(function(res) {
      ...@@ -98,14 +115,18 @@ angular.module('cesium.map.wot.services', ['cesium.services']) ...@@ -98,14 +115,18 @@ angular.module('cesium.map.wot.services', ['cesium.services'])
      }) })
      ]) ])
      .then(function(res) { .then(function(res) {
      var total = res[0].hits && res[0].hits.total || 0;
      var uids = res[1]; var uids = res[1];
      var memberships = res[2]; var memberships = res[2];
      res = res[0];
      if (!res.hits || !res.hits.total) return []; if (!total) return []; // No data
      var now = Date.now();
      console.info('[map] [wot] Loading {0} profiles...'.format(total));
      // Transform pending MS into a map by pubkey // Transform pending MS into a map by pubkey
      memberships = memberships.reduce(function(res, ms){ memberships = memberships.reduce(function(res, ms){
      if (ms.membership == 'IN' && !uids[ms.pubkey]) { if (ms.membership === 'IN' && !uids[ms.pubkey]) {
      var idty = { var idty = {
      uid: ms.uid, uid: ms.uid,
      pubkey: ms.pubkey, pubkey: ms.pubkey,
      ...@@ -122,26 +143,47 @@ angular.module('cesium.map.wot.services', ['cesium.services']) ...@@ -122,26 +143,47 @@ angular.module('cesium.map.wot.services', ['cesium.services'])
      return res; return res;
      }, {}); }, {});
      var jobs = [ var searchRecursive = function(request, result) {
      processLoadHits(options, uids, memberships, res) request.from = request.from || 0;
      ]; request.size = request.size || constants.DEFAULT_LOAD_SIZE;
      result = result || {hits: {hits: []}};
      // Additional slice requests
      request.from += request.size; // DEBUG
      //console.debug('Searching... ' + request.from);
      return search(request).then(function(res) {
      if (!res.hits || !res.hits.hits.length) return result;
      result.hits.total = res.hits.total
      result.hits.hits = result.hits.hits.concat(res.hits.hits);
      if (result.hits.hits.length < result.hits.total) {
      request.from += request.size;
      if (request.from >= 10000) {
      console.error("Cannot load more than 10000 profiles in a slice. Please reduce slice size!");
      return result; // Skip if too large
      }
      return searchRecursive(request, result);
      }
      return result;
      })
      }
      var processRequestResultFn = function(subRes) { var processRequestResultFn = function(subRes) {
      if (!subRes.hits || !subRes.hits.hits.length) return []; if (!subRes.hits || !subRes.hits.hits.length) return [];
      return processLoadHits(options, uids, memberships, subRes); return processLoadHits(options, uids, memberships, subRes);
      }; };
      while (request.from < res.hits.total) {
      var searchRequest = search(angular.copy(request)).then(processRequestResultFn);
      jobs.push(searchRequest); return $q.all(createSliceQueries(options, total)
      request.from += request.size; .reduce(function(res, query) {
      } var request = {query: query, _source: fields.profile};
      return $q.all(jobs) return res.concat(searchRecursive(request).then(processRequestResultFn));
      }, []))
      .then(function(res){ .then(function(res){
      return res.reduce(function(res, items) { var result = res.reduce(function(res, items) {
      return res.concat(items); return res.concat(items);
      }, []); }, []);
      console.info('[map] [wot] Loaded {0} profiles in {1}ms'.format(result.length, Date.now() - now));
      return result;
      }); });
      }); });
      } }
      ...@@ -149,8 +191,7 @@ angular.module('cesium.map.wot.services', ['cesium.services']) ...@@ -149,8 +191,7 @@ angular.module('cesium.map.wot.services', ['cesium.services'])
      function processLoadHits(options, uids, memberships, res) { function processLoadHits(options, uids, memberships, res) {
      // Transform profile hits // Transform profile hits
      var commaRegexp = new RegExp('[,]'); var commaRegexp = new RegExp(',');
      var searchAddressItems = [];
      var items = res.hits.hits.reduce(function(res, hit) { var items = res.hits.hits.reduce(function(res, hit) {
      var pubkey = hit._id; var pubkey = hit._id;
      var uid = uids[pubkey]; var uid = uids[pubkey];
      ...@@ -164,19 +205,12 @@ angular.module('cesium.map.wot.services', ['cesium.services']) ...@@ -164,19 +205,12 @@ angular.module('cesium.map.wot.services', ['cesium.services'])
      // Set geo point // Set geo point
      item.geoPoint = hit._source.geoPoint; item.geoPoint = hit._source.geoPoint;
      if (!item.geoPoint || !item.geoPoint.lat || !item.geoPoint.lon) { // Convert lat/lon to float (if need)
      if (!options.searchAddress || !item.city) return res; // no city: exclude this item if (item.geoPoint.lat && typeof item.geoPoint.lat === 'string') {
      item.searchAddress = item.city && ((hit._source.address ? hit._source.address+ ', ' : '') + item.city); item.geoPoint.lat = parseFloat(item.geoPoint.lat.replace(commaRegexp, '.'));
      searchAddressItems.push(item);
      } }
      else { if (item.geoPoint.lon && typeof item.geoPoint.lon === 'string') {
      // Convert lat/lon to float (if need) item.geoPoint.lon = parseFloat(item.geoPoint.lon.replace(commaRegexp, '.'));
      if (item.geoPoint.lat && typeof item.geoPoint.lat === 'string') {
      item.geoPoint.lat = parseFloat(item.geoPoint.lat.replace(commaRegexp, '.'));
      }
      if (item.geoPoint.lon && typeof item.geoPoint.lon === 'string') {
      item.geoPoint.lon = parseFloat(item.geoPoint.lon.replace(commaRegexp, '.'));
      }
      } }
      // Avatar // Avatar
      ...@@ -190,39 +224,11 @@ angular.module('cesium.map.wot.services', ['cesium.services']) ...@@ -190,39 +224,11 @@ angular.module('cesium.map.wot.services', ['cesium.services'])
      } }
      // Description // Description
      item.description = hit._source.description && esHttp.util.parseAsHtml(hit._source.description); //item.description = hit._source.description && esHttp.util.parseAsHtml(hit._source.description);
      return item.geoPoint ? res.concat(item) : res; return item.geoPoint ? res.concat(item) : res;
      }, []); }, []);
      // Resolve missing positions by addresses (only if google API enable)
      if (searchAddressItems.length) {
      var now = Date.now();
      console.debug('[map] [wot] Search positions of {0} addresses...'.format(searchAddressItems.length));
      var counter = 0;
      return $q.all(searchAddressItems.reduce(function(res, item) {
      return !item.city ? res : res.concat(esGeo.google.searchByAddress(item.searchAddress)
      .then(function(res) {
      if (!res || !res.length) return;
      item.geoPoint = res[0];
      // If search on city, add a randomized delta to avoid superposition
      if (item.city == item.searchAddress) {
      item.geoPoint.lon += Math.random() / 1000;
      item.geoPoint.lat += Math.random() / 1000;
      }
      delete item.searchAddress; // not need anymore
      items.push(item);
      counter++;
      })
      .catch(function() {/*silent*/}));
      }, []))
      .then(function(){
      console.debug('[map] [wot] Resolved {0}/{1} addresses in {2}ms'.format(counter, searchAddressItems.length, Date.now()-now));
      return items;
      });
      }
      return $q.when(items); return $q.when(items);
      } }
      ......
      0% or .
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment