Created
June 10, 2017 20:23
-
-
Save mitchellporter/98769b54d9c61871e0ec53730ddbe7f0 to your computer and use it in GitHub Desktop.
CapserJS - Scraping google search results
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| const casper = require('casper').create(); | |
| const fs = require('fs'); | |
| var data; | |
| casper.on('remote.message', function(msg) { | |
| console.log('remote message is: ' + msg); | |
| }); | |
| casper.start('http://www.google.com/', function() { | |
| var title = this.getCurrentUrl(); | |
| this.fill('form', { q: 'hello world!' }, true); | |
| }); | |
| casper.wait(1000, function() { | |
| console.log('CasperJS user agent: ' + window.navigator.userAgent); | |
| data = this.evaluate(function() { | |
| var targetElements = document.querySelectorAll('.g > .r > a'); | |
| var data = []; | |
| console.log('target elements length: ' + targetElements.length); | |
| for (var index = 0; index < targetElements.length; index++) { | |
| var currentEl = targetElements[index]; | |
| var currentLink = currentEl.getAttribute('href'); | |
| var currentTitle = currentEl.text; | |
| var currentItem = { | |
| link: currentLink, | |
| title: currentTitle, | |
| } | |
| data.push(currentItem); | |
| } | |
| return data; | |
| }); | |
| fs.write('./output.json',JSON.stringify(data, null, '\t')); | |
| this.exit(); | |
| }); | |
| casper.run(); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "link": "/url?q=https://en.wikipedia.org/wiki/%2522Hello,_World!%2522_program&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFggUMAA&usg=AFQjCNFxhrqP7FDEfkLooqdtT-E-o0NWvg", | |
| "title": "\"Hello, World!\" program - Wikipedia" | |
| }, | |
| { | |
| "link": "/url?q=http://www.helloworld.com/&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFggfMAE&usg=AFQjCNHZ2Dbo1Yd01VAHpeU3PSYCsiHJyA", | |
| "title": "HelloWorld: Digital, Social, Mobile Marketing" | |
| }, | |
| { | |
| "link": "/url?q=https://www.learnpython.org/en/Hello%252C_World%2521&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFggqMAI&usg=AFQjCNEi8wR7Sxoe_x0UwW2zKxcpTlw89Q", | |
| "title": "Hello, World! - Learn Python - Free Interactive Python Tutorial" | |
| }, | |
| { | |
| "link": "/url?q=https://en.wikibooks.org/wiki/Computer_Programming/Hello_world&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFggvMAM&usg=AFQjCNEYcteeST6uYcrY_De0-M45gX6p5A", | |
| "title": "Computer Programming/Hello world - Wikibooks, open books for an ..." | |
| }, | |
| { | |
| "link": "/url?q=http://wiki.c2.com/%3FHelloWorldInManyProgrammingLanguages&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFgg1MAQ&usg=AFQjCNH0hNdpgrWWDNSwCZaz6F4TjyTB8A", | |
| "title": "Hello World In Many Programming Languages - C2 Wiki" | |
| }, | |
| { | |
| "link": "/url?q=https://www.raspberrypi.org/helloworld&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFgg6MAU&usg=AFQjCNFtVw0d0Np2ODYIKxLsiy3DpGUavQ", | |
| "title": "Hello World - Raspberry Pi" | |
| }, | |
| { | |
| "link": "/url?q=https://thehelloworldprogram.com/&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFgg_MAY&usg=AFQjCNFYaWWOhpjew3RnVuA9CL_SroHpVA", | |
| "title": "The Hello World Program: Hands-on Computer Science" | |
| }, | |
| { | |
| "link": "/url?q=https://gobyexample.com/hello-world&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFghEMAc&usg=AFQjCNFXYM5bvBdoGb-0TzuoIfH9u61Rtg", | |
| "title": "Go by Example: Hello World" | |
| }, | |
| { | |
| "link": "/url?q=https://expressjs.com/starter/hello-world.html&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFghKMAg&usg=AFQjCNETNpTyrqN2NuDjicQTb3SnyO_Ysg", | |
| "title": "Express \"Hello World\" example" | |
| }, | |
| { | |
| "link": "/url?q=http://docs.oracle.com/javase/tutorial/getStarted/cupojava/&sa=U&ved=0ahUKEwjn9arCiLTUAhUH2GMKHTmUBYwQFghPMAk&usg=AFQjCNEjr8OGnt5S19LfZxvlicQENTIv0A", | |
| "title": "Lesson: The \"Hello World!\" Application (The Java™ Tutorials ..." | |
| } | |
| ] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "name": "automation-tools", | |
| "version": "1.0.0", | |
| "description": "", | |
| "main": "index.js", | |
| "scripts": { | |
| "start": "./node_modules/.bin/casperjs main.js", | |
| "test": "echo \"Error: no test specified\" && exit 1" | |
| }, | |
| "keywords": [], | |
| "author": "", | |
| "license": "ISC", | |
| "dependencies": { | |
| "casperjs": "1.1.0-beta5", | |
| "phantomjs-prebuilt": "2.1.7" | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment