It is a simple crawler implemented with Node crawler package .

crawler-project.js

//require('events').EventEmitter.defaultMaxListeners = 0


var Crawler = require("crawler");


var baseURLS = ['http://www.google.com/','http://www.yahoo.com', 'http://www.amazon.com', 'http://www.sharif.ir/home'];
var depth = 3

var urls = new Set()


var c = new Crawler({
    maxConnections : 100000,
    retries: 2,
    skipDuplicates: true,
    // preRequest: function(options, done) {
    //
    //     // console.log("This request: ", options.uri)
    //     done();
    // },
    callback : function (error, res, done) {
        if(error){
            console.log(error);
            done()
        }else {
            var $ = res.$;
            console.log(res.request.uri.href + " " + urls.size);

            if($) {
                var tags = $("a");
                // console.log("inja" + " " + res.body)
                // console.log($)
                for (var a = 0; a < tags.length; a++) {
                    // console.log(res.request.uri.href+ " " + tags.length)

                    if (tags[a].attribs.href) {
                        // console.log(res.request.uri.href)

                        if (tags[a].attribs.href.startsWith("www") || tags[a].attribs.href.startsWith("http") || tags[a].attribs.href.startsWith("https")) {
                            // console.log(res.request.uri.href)
                            urls.add(tags[a].attribs.href)
                        }
                    }
                }
            }
            else {
                console.log("hereee")
            }
            done();
        }
    }
});

c.queue(baseURLS);
// c.queue("http://www.sharif.ir/home");
c.on('drain',function(){
    depth--;
    if (depth<=0) {
        console.log(urls);
        console.log("size: ", urls.size)
    }
    else {
        console.log("Draiinnnnnnnnnnn, "+ depth);
        // c.queue("http://www.sharif.ir/home")
        c.queue(Array.from(urls))
    }
});

For testing after initializing new node project, use npm install crawler then node crawler-project.js.
baseURLS should contain the start points you want to crawl.
the depth variable shows depth you want to crawl from start points.

Sometimes javascript heap memory will finish, a naive solution is using this option for node :
--max_old_space_size=2000000