Optimisations for /wot/certifiers-of and /wot/certified-by (#197)

app/controllers/wot.js

@@ -108,80 +108,58 @@ function WOTBinding (server) {
 
   this.certifiersOf = function (req, res) {
     res.type('application/json');
-    async.waterfall([
-      function (next){
-        ParametersService.getSearch(req, next);
-      },
-      function (search, next){
-        IdentityService.findMember(search, next);
-      },
-      function (idty, next){
-        async.waterfall([
-          function (next){
-            server.dal.certsToTarget(idty.getTargetHash()).then(_.partial(next, null)).catch(next);
-          },
-          function (certs, next){
-            idty.certs = [];
-            async.forEach(certs, function (cert, callback) {
-              async.waterfall([
-                function (next) {
-                  server.dal.getWritten(cert.from, next);
-                },
-                function (idty, next) {
-                  if (!idty) {
-                    next('Not a member');
-                    return;
-                  }
-                  cert.uid = idty.uid;
-                  cert.isMember = idty.member;
-                  cert.wasMember = idty.wasMember;
-                  server.dal.getBlock(cert.block_number, next);
-                },
-                function (block, next) {
-                  cert.cert_time = {
-                    block: block.number,
-                    medianTime: block.medianTime
-                  };
-                  idty.certs.push(cert);
-                  next();
-                }
-              ], function () {
-                callback();
-              });
-            }, next);
-          },
-          function (next) {
-            next(null, idty);
+    co(function *() {
+      try {
+        let search = yield ParametersService.getSearchP(req);
+        let idty = yield IdentityService.findMemberWithoutMemberships(search);
+        let excluding = yield BlockchainService.getCertificationsExludingBlock();
+        let certs = yield server.dal.certsToTarget(idty.getTargetHash());
+        idty.certs = [];
+        for (let i = 0; i < certs.length; i++) {
+          let cert = certs[i];
+          if (!(excluding && cert.block <= excluding.number)) {
+            let certifier = yield server.dal.getWrittenIdtyByPubkey(cert.from);
+            if (certifier) {
+              cert.uid = certifier.uid;
+              cert.isMember = certifier.member;
+              cert.wasMember = true; // As we checked if(certified)
+              if (!cert.cert_time) {
+                // TODO: would be more efficient to save medianTime on certification reception
+                let certBlock = yield server.dal.getBlock(cert.block_number);
+                cert.cert_time = {
+                  block: certBlock.number,
+                  medianTime: certBlock.medianTime
+                };
+              }
+              idty.certs.push(cert);
+            }
           }
-        ], next);
-      }
-    ], function (err, idty) {
-      if(err){
+        }
+        var json = {
+          pubkey: idty.pubkey,
+          uid: idty.uid,
+          isMember: idty.member,
+          certifications: []
+        };
+        idty.certs.forEach(function(cert){
+          json.certifications.push({
+            pubkey: cert.from,
+            uid: cert.uid,
+            isMember: cert.isMember,
+            wasMember: cert.wasMember,
+            cert_time: cert.cert_time,
+            written: cert.linked,
+            signature: cert.sig
+          });
+        });
+        res.send(200, JSON.stringify(json, null, "  "));
+      } catch (err) {
         if (err == 'No member matching this pubkey or uid') {
           res.send(404, err);
           return;
         }
         res.send(400, err);
-        return;
       }
-      var json = {
-        pubkey: idty.pubkey,
-        uid: idty.uid,
-        isMember: idty.member,
-        certifications: []
-      };
-      idty.certs.forEach(function(cert){
-        json.certifications.push({
-          pubkey: cert.from,
-          uid: cert.uid,
-          isMember: cert.isMember,
-          wasMember: cert.wasMember,
-          cert_time: cert.cert_time,
-          written: cert.linked,
-          signature: cert.sig
-        });
-      });
-      res.send(200, JSON.stringify(json, null, "  "));
     });
   };
 
@@ -236,80 +214,58 @@ function WOTBinding (server) {
 
   this.certifiedBy = function (req, res) {
     res.type('application/json');
-    async.waterfall([
-      function (next){
-        ParametersService.getSearch(req, next);
-      },
-      function (search, next){
-        IdentityService.findMember(search, next);
-      },
-      function (idty, next){
-        async.waterfall([
-          function (next){
-            server.dal.certsFrom(idty.pubkey).then(_.partial(next, null)).catch(next);
-          },
-          function (certs, next){
-            idty.certs = [];
-            async.forEach(certs, function (cert, callback) {
-              async.waterfall([
-                function (next) {
-                  server.dal.getWritten(cert.to, next);
-                },
-                function (idty, next) {
-                  if (!idty) {
-                    next('Not a member');
-                    return;
-                  }
-                  cert.uid = idty.uid;
-                  cert.isMember = idty.member;
-                  cert.wasMember = idty.wasMember;
-                  server.dal.getBlock(cert.block_number, next);
-                },
-                function (block, next) {
-                  cert.cert_time = {
-                    block: block.number,
-                    medianTime: block.medianTime
-                  };
-                  idty.certs.push(cert);
-                  next();
-                }
-              ], function () {
-                callback();
-              });
-            }, next);
-          },
-          function (next) {
-            next(null, idty);
+    co(function *() {
+      try {
+        let search = yield ParametersService.getSearchP(req);
+        let idty = yield IdentityService.findMemberWithoutMemberships(search);
+        let excluding = yield BlockchainService.getCertificationsExludingBlock();
+        let certs = yield server.dal.certsFrom(idty.pubkey);
+        idty.certs = [];
+        for (let i = 0; i < certs.length; i++) {
+          let cert = certs[i];
+          if (!(excluding && cert.block <= excluding.number)) {
+            let certified = yield server.dal.getWrittenIdtyByPubkey(cert.to);
+            if (certified) {
+              cert.uid = certified.uid;
+              cert.isMember = certified.member;
+              cert.wasMember = true; // As we checked if(certified)
+              if (!cert.cert_time) {
+                // TODO: would be more efficient to save medianTime on certification reception
+                let certBlock = yield server.dal.getBlock(cert.block_number);
+                cert.cert_time = {
+                  block: certBlock.number,
+                  medianTime: certBlock.medianTime
+                };
+              }
+              idty.certs.push(cert);
+            }
           }
-        ], next);
-      }
-    ], function (err, idty) {
-      if(err){
+        }
+        var json = {
+          pubkey: idty.pubkey,
+          uid: idty.uid,
+          isMember: idty.member,
+          certifications: []
+        };
+        idty.certs.forEach(function(cert){
+          json.certifications.push({
+            pubkey: cert.from,
+            uid: cert.uid,
+            isMember: cert.isMember,
+            wasMember: cert.wasMember,
+            cert_time: cert.cert_time,
+            written: cert.linked,
+            signature: cert.sig
+          });
+        });
+        res.send(200, JSON.stringify(json, null, "  "));
+      } catch (err) {
         if (err == 'No member matching this pubkey or uid') {
           res.send(404, err);
           return;
         }
         res.send(400, err);
-        return;
       }
-      var json = {
-        pubkey: idty.pubkey,
-        uid: idty.uid,
-        isMember: idty.member,
-        certifications: []
-      };
-      idty.certs.forEach(function(cert){
-        json.certifications.push({
-          pubkey: cert.to,
-          uid: cert.uid,
-          cert_time: cert.cert_time,
-          isMember: cert.isMember,
-          wasMember: cert.wasMember,
-          written: cert.linked,
-          signature: cert.sig
-        });
-      });
-      res.send(200, JSON.stringify(json, null, "  "));
     });
   };
 

app/lib/blockchainContext.js

@@ -413,7 +413,7 @@ function BlockchainContext(conf, dal) {
         }
       })
       .then(function(){
-        return dal.getCertificationExcludingBlock(block, conf.sigValidity);
+        return dal.getCertificationExcludingBlock(block, conf.sigValidity, conf.sigDelay);
       });
   }
 

app/lib/dal/fileDAL.js

@@ -746,21 +746,48 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, core, lo
         }
       } else {
         var start = currentExcluding.number;
-        var nextPotential;
+        let newExcluding;
+        let top = current.number, bottom = start;
+        // Binary tree search
         do {
-          nextPotential = yield that.getBlock(start + 1);
-          var delaySinceNextOfExcluding = current.medianTime - nextPotential.medianTime;
-          if (delaySinceNextOfExcluding > msValidtyTime) {
-            yield that.indicatorsDAL.writeCurrentExcluding(nextPotential).then(() => nextPotential);
-            start++;
+          let middle = top - bottom;
+          if (middle % 2 != 0) {
+            middle = middle + 1;
           }
-        } while (delaySinceNextOfExcluding > msValidtyTime);
-        return nextPotential;
+          middle /= 2;
+          middle += bottom;
+          if (middle == top) {
+            middle--;
+            bottom--; // Helps not being stuck looking at 'top'
+          }
+          let middleBlock = yield that.getBlock(middle);
+          let middleNextB = yield that.getBlock(middle + 1);
+          var delaySinceMiddle = current.medianTime - middleBlock.medianTime;
+          var delaySinceNextB = current.medianTime - middleNextB.medianTime;
+          let isValidPeriod = delaySinceMiddle <= msValidtyTime;
+          let isValidPeriodB = delaySinceNextB <= msValidtyTime;
+          let isExcludin = !isValidPeriod && isValidPeriodB;
+          //console.log('MS: Search between %s and %s: %s => %s,%s', bottom, top, middle, isValidPeriod ? 'DOWN' : 'UP', isValidPeriodB ? 'DOWN' : 'UP');
+          if (isExcludin) {
+            // Found
+            yield that.indicatorsDAL.writeCurrentExcluding(middleBlock);
+            newExcluding = middleBlock;
+          }
+          else if (isValidPeriod) {
+            // Look down in the blockchain
+            top = middle;
+          }
+          else {
+            // Look up in the blockchain
+            bottom = middle;
+          }
+        } while (!newExcluding);
+        return newExcluding;
       }
     });
   };
 
-  this.getCertificationExcludingBlock = function(current, certValidtyTime) {
+  this.getCertificationExcludingBlock = function(current, certValidtyTime, certDelay) {
     return co(function *() {
       var currentExcluding;
       if (current.number > 0) {
@@ -773,25 +800,67 @@ function FileDAL(profile, home, localDir, myFS, parentFileDAL, dalName, core, lo
       if (!currentExcluding) {
         var root = yield that.getRootBlock();
         var delaySinceStart = current.medianTime - root.medianTime;
-        if (delaySinceStart > certValidtyTime) {
+        if (delaySinceStart > certValidtyTime + certDelay) {
           return that.indicatorsDAL.writeCurrentExcludingForCert(root).then(() => root);
         }
       } else {
-        var start = currentExcluding.number;
-        var nextPotential;
-        do {
-          nextPotential = yield that.getBlock(start + 1);
-          var delaySinceNextOfExcluding = current.medianTime - nextPotential.medianTime;
-          if (delaySinceNextOfExcluding > certValidtyTime) {
-            yield that.indicatorsDAL.writeCurrentExcludingForCert(nextPotential).then(() => nextPotential);
-            start++;
-          }
-        } while (delaySinceNextOfExcluding > certValidtyTime);
-        return nextPotential;
+        // Check current position
+        let currentNextBlock = yield that.getBlock(currentExcluding.number + 1);
+        if (isExcluding(current, currentExcluding, currentNextBlock, certValidtyTime, certDelay)) {
+          return currentExcluding;
+        } else {
+          // Have to look for new one
+          var start = currentExcluding.number;
+          let newExcluding;
+          let top = current.number, bottom = start;
+          // Binary tree search
+          do {
+            let middle = top - bottom;
+            if (middle % 2 != 0) {
+              middle = middle + 1;
+            }
+            middle /= 2;
+            middle += bottom;
+            if (middle == top) {
+              middle--;
+              bottom--; // Helps not being stuck looking at 'top'
+            }
+            let middleBlock = yield that.getBlock(middle);
+            let middleNextB = yield that.getBlock(middle + 1);
+            var delaySinceMiddle = current.medianTime - middleBlock.medianTime;
+            var delaySinceNextB = current.medianTime - middleNextB.medianTime;
+            let isValidPeriod = delaySinceMiddle <= certValidtyTime + certDelay;
+            let isValidPeriodB = delaySinceNextB <= certValidtyTime + certDelay;
+            let isExcludin = !isValidPeriod && isValidPeriodB;
+            console.log('CRT: Search between %s and %s: %s => %s,%s', bottom, top, middle, isValidPeriod ? 'DOWN' : 'UP', isValidPeriodB ? 'DOWN' : 'UP');
+            if (isExcludin) {
+              // Found
+              yield that.indicatorsDAL.writeCurrentExcludingForCert(middleBlock);
+              newExcluding = middleBlock;
+            }
+            else if (isValidPeriod) {
+              // Look down in the blockchain
+              top = middle;
+            }
+            else {
+              // Look up in the blockchain
+              bottom = middle;
+            }
+          } while (!newExcluding);
+          return newExcluding;
+        }
       }
     });
   };
 
+  function isExcluding(current, excluding, nextBlock, certValidtyTime, certDelay) {
+    var delaySinceMiddle = current.medianTime - excluding.medianTime;
+    var delaySinceNextB = current.medianTime - nextBlock.medianTime;
+    let isValidPeriod = delaySinceMiddle <= certValidtyTime + certDelay;
+    let isValidPeriodB = delaySinceNextB <= certValidtyTime + certDelay;
+    return !isValidPeriod && isValidPeriodB;
+  }
+
   this.kickWithOutdatedMemberships = function(maxNumber) {
     return that.getMembers()
       .then(function(members){

app/service/BlockchainService.js

@@ -1506,7 +1506,7 @@ function BlockchainService (conf, mainDAL, pair) {
   this.getCertificationsExludingBlock = function() {
     return that.currentDal.getCurrent()
       .then(function(current){
-        return that.currentDal.getCertificationExcludingBlock(current, conf.sigValidity);
+        return that.currentDal.getCertificationExcludingBlock(current, conf.sigValidity, conf.sigDelay);
       })
       .catch(function(){
         return { number: -1 };

app/service/IdentityService.js

@@ -60,8 +60,22 @@ function IdentityService (conf, dal) {
   })
   .catch(done);
 
+  this.findMemberWithoutMemberships = (search) => co(function *() {
+    let idty = null;
+    if (search.match(constants.PUBLIC_KEY)) {
+      idty = yield dal.getWrittenIdtyByPubkey(search);
+    }
+    else {
+      idty = yield dal.getWrittenIdtyByUID(search);
+    }
+    if (!idty) {
+      throw 'No member matching this pubkey or uid';
+    }
+    return new Identity(idty);
+  });
+
   this.getWrittenByPubkey = function(pubkey) {
-    return dal.getWritten(pubkey);
+    return dal.getWrittenIdtyByPubkey(pubkey);
   };
 
   this.getPendingFromPubkey = function(pubkey) {

app/service/PeeringService.js

@@ -287,7 +287,7 @@ function PeeringService(server, pair, dal) {
                                     tx.issuers = tx.signatories;
                                     tx.hash = ("" + sha1(rawer.getTransaction(tx))).toUpperCase();
                                   });
-                                  logger.info("Downloaded block #%s from peer ", block.number, p.getNamedURL());
+                                  logger.info("Downloaded block #%s from peer %s", block.number, p.getNamedURL());
                                   server.BlockchainService.submitBlock(block, true)
                                     .then(function(block){
                                       return next(null, block);

test/integration/tools/node.js

@@ -100,7 +100,7 @@ function Node (dbName, options) {
         BlockchainService.prove(block, sigFunc, difficulty, next);
       },
       function (provenBlock, next){
-        provenBlock && provenBlock.getRawSigned && logger.debug(provenBlock.getRawSigned());
+        logger.debug(provenBlock.getRawSigned());
         post('/blockchain/block', {
           "block": provenBlock.getRawSigned()
         }, next);