Last active
August 29, 2015 14:05
-
-
Save vkobel/8e44810f683ff153b4b8 to your computer and use it in GitHub Desktop.
Retrieve links (ajax created) on a website for further processing using PhantomJS (http://phantomjs.org/)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var url = 'http://cert.europa.eu/cert/filteredition/en/CERT-LatestNews.html?&printpreview=all'; | |
var querySelector = 'a'; // All 'a' HTML elements (could be any selector) | |
var page = require('webpage').create(); | |
page.open(url, function (status) { | |
if(status === 'fail') { | |
console.log("ERROR"); | |
} else { | |
// get a map for each link (we filter later) | |
var links = page.evaluate(function() { | |
// Important part: extract information | |
// 'a' is the querySelector (could be any 'CSS' selector) -> select all links in page | |
return [].map.call(document.querySelectorAll('a'), function(link) { | |
return { "txt": link.innerText, "href": link.getAttribute('href') }; // Object with text and link value | |
}); | |
}); | |
// Here we can filter the results | |
links.forEach(function(l){ | |
// Take only if href begin with a slash / | |
if(l.href.trim() != "" && l.href.indexOf("/") == 0) | |
console.log(l.txt + " -> " + l.href); // print it | |
}); | |
} | |
page.close(); | |
phantom.exit(); | |
}); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
> phantomjs.exe links.js | |
Hardware; -> /cert/alertedition/en/VulnerabilitiesHardware.html | |
Cyber Crime; -> /cert/alertedition/en/ThreatsCybercrime.html | |
Cyber Crime; -> /cert/alertedition/en/ThreatsCybercrime.html | |
Symantec; -> /cert/alertedition/en/SymantecNews.html | |
Malware; -> /cert/alertedition/en/Malware.html | |
All products; -> /cert/alertedition/en/VulnerabilitiesAll.html | |
Cyber Crime; -> /cert/alertedition/en/ThreatsCybercrime.html | |
Malware; -> /cert/alertedition/en/Malware.html | |
Kaspersky; -> /cert/alertedition/en/KasperskyNews.html | |
Advanced Persistent Threats; -> /cert/alertedition/en/APTFilter.html | |
Cyber Crime; -> /cert/alertedition/en/ThreatsCybercrime.html | |
Economic; -> /cert/alertedition/en/ThreatsEconomic.html | |
Strategic; -> /cert/alertedition/en/ThreatsStrategic.html | |
Malware; -> /cert/alertedition/en/Malware.html | |
Cyber Crime; -> /cert/alertedition/en/ThreatsCybercrime.html | |
Malware; -> /cert/alertedition/en/Malware.html | |
Cyber Crime; -> /cert/alertedition/en/ThreatsCybercrime.html | |
Malware; -> /cert/alertedition/en/Malware.html | |
McAfee; -> /cert/alertedition/en/McAfeeNews.html | |
Cyber Crime; -> /cert/alertedition/en/ThreatsCybercrime.html | |
Economic; -> /cert/alertedition/en/ThreatsEconomic.html | |
Malware; -> /cert/alertedition/en/Malware.html | |
Cyber Crime; -> /cert/alertedition/en/ThreatsCybercrime.html | |
Social Engineering; -> /cert/alertedition/en/SocialEngineering.html | |
Cyber Crime; -> /cert/alertedition/en/ThreatsCybercrime.html | |
Strategic; -> /cert/alertedition/en/ThreatsStrategic.html |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment