Crawl Loop: sitecawl

crawl loop

 function  runCrawler(obs) {
    var self = this;
    self.crawling = true;
    Rx.Observable.While(
      function() {return self.crawling},
      Rx.Observable.If(
        function(){return self.crawlQueue.length > 0;},
        Rx.Observable.Defer(function (){
          var nextCrawlStep = self.crawlQueue.pop();
          return self.selectForCrawlLinks(nextCrawlStep.Delay(self.delay))}),
        Rx.Observable.Return('nop').Delay(self.maxCrawlTime))
    ).Subscribe(function(crawlResult){
        if (crawlResult != 'nop') {
          obs.OnNext(crawlResult);
        } else {
          if (self.crawlQueue.length ==0) {
            // we have waited maxCrawlTime for new pages
            // and the queue is empty, signal complete
            obs.OnCompleted();
          }
        }
      },
      function(exn){
        obs.OnError(exn);
      });
      
  }(obs) {
    var self = this;
    self.crawling = true;
    Rx.Observable.While(
      function() {return self.crawling},
      Rx.Observable.If(
        function(){return self.crawlQueue.length > 0;},
        Rx.Observable.Defer(function (){
          var nextCrawlStep = self.crawlQueue.pop();
          return self.selectForCrawlLinks(nextCrawlStep.Delay(self.delay))}),
        Rx.Observable.Return('nop').Delay(self.maxCrawlTime))
    ).Subscribe(function(crawlResult){
        if (crawlResult != 'nop') {
          obs.OnNext(crawlResult);
        } else {
          if (self.crawlQueue.length ==0) {
            // we have waited maxCrawlTime for new pages
            // and the queue is empty, signal complete
            obs.OnCompleted();
          }
        }
      },
      function(exn){
        obs.OnError(exn);
      });
      
 }