Skip to content

Instantly share code, notes, and snippets.

@mitchellporter
Created June 10, 2017 20:23
Show Gist options
  • Select an option

  • Save mitchellporter/98769b54d9c61871e0ec53730ddbe7f0 to your computer and use it in GitHub Desktop.

Select an option

Save mitchellporter/98769b54d9c61871e0ec53730ddbe7f0 to your computer and use it in GitHub Desktop.
CapserJS - Scraping google search results
const casper = require('casper').create();
const fs = require('fs');
var data;
casper.on('remote.message', function(msg) {
console.log('remote message is: ' + msg);
});
casper.start('http://www.google.com/', function() {
var title = this.getCurrentUrl();
this.fill('form', { q: 'hello world!' }, true);
});
casper.wait(1000, function() {
console.log('CasperJS user agent: ' + window.navigator.userAgent);
data = this.evaluate(function() {
var targetElements = document.querySelectorAll('.g > .r > a');
var data = [];
console.log('target elements length: ' + targetElements.length);
for (var index = 0; index < targetElements.length; index++) {
var currentEl = targetElements[index];
var currentLink = currentEl.getAttribute('href');
var currentTitle = currentEl.text;
var currentItem = {
link: currentLink,
title: currentTitle,
}
data.push(currentItem);
}
return data;
});
fs.write('./output.json',JSON.stringify(data, null, '\t'));
this.exit();
});
casper.run();
[
{
"link": "/url?q=https://en.wikipedia.org/wiki/%2522Hello,_World!%2522_program&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFggUMAA&usg=AFQjCNFxhrqP7FDEfkLooqdtT-E-o0NWvg",
"title": "\"Hello, World!\" program - Wikipedia"
},
{
"link": "/url?q=http://www.helloworld.com/&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFggfMAE&usg=AFQjCNHZ2Dbo1Yd01VAHpeU3PSYCsiHJyA",
"title": "HelloWorld: Digital, Social, Mobile Marketing"
},
{
"link": "/url?q=https://www.learnpython.org/en/Hello%252C_World%2521&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFggqMAI&usg=AFQjCNEi8wR7Sxoe_x0UwW2zKxcpTlw89Q",
"title": "Hello, World! - Learn Python - Free Interactive Python Tutorial"
},
{
"link": "/url?q=https://en.wikibooks.org/wiki/Computer_Programming/Hello_world&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFggvMAM&usg=AFQjCNEYcteeST6uYcrY_De0-M45gX6p5A",
"title": "Computer Programming/Hello world - Wikibooks, open books for an ..."
},
{
"link": "/url?q=http://wiki.c2.com/%3FHelloWorldInManyProgrammingLanguages&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFgg1MAQ&usg=AFQjCNH0hNdpgrWWDNSwCZaz6F4TjyTB8A",
"title": "Hello World In Many Programming Languages - C2 Wiki"
},
{
"link": "/url?q=https://www.raspberrypi.org/helloworld&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFgg6MAU&usg=AFQjCNFtVw0d0Np2ODYIKxLsiy3DpGUavQ",
"title": "Hello World - Raspberry Pi"
},
{
"link": "/url?q=https://thehelloworldprogram.com/&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFgg_MAY&usg=AFQjCNFYaWWOhpjew3RnVuA9CL_SroHpVA",
"title": "The Hello World Program: Hands-on Computer Science"
},
{
"link": "/url?q=https://gobyexample.com/hello-world&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFghEMAc&usg=AFQjCNFXYM5bvBdoGb-0TzuoIfH9u61Rtg",
"title": "Go by Example: Hello World"
},
{
"link": "/url?q=https://expressjs.com/starter/hello-world.html&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFghKMAg&usg=AFQjCNETNpTyrqN2NuDjicQTb3SnyO_Ysg",
"title": "Express \"Hello World\" example"
},
{
"link": "/url?q=http://docs.oracle.com/javase/tutorial/getStarted/cupojava/&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFghPMAk&usg=AFQjCNEjr8OGnt5S19LfZxvlicQENTIv0A",
"title": "Lesson: The \"Hello World!\" Application (The Java™ Tutorials ..."
}
]
{
"name": "automation-tools",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start": "./node_modules/.bin/casperjs main.js",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"casperjs": "1.1.0-beta5",
"phantomjs-prebuilt": "2.1.7"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment