Last active
January 20, 2024 15:57
-
-
Save MattSandy/91c28069291b86f9a617ab0f608c9bdc to your computer and use it in GitHub Desktop.
Tweet Scraper in Node.JS
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var Twitter = require('node-tweet-stream') | |
, t = new Twitter({ | |
consumer_key: '', | |
consumer_secret: '', | |
token: '', | |
token_secret: '' | |
}); | |
var watch = [ | |
"SB52", | |
"SBLII", | |
"SuperBowl", | |
"sb52", | |
"Superbowl", | |
"SuperBowlLII", | |
"SuperBowlLll", | |
"sblii", | |
"SuperBowl2018", | |
"superbowl", | |
"sb2018"]; | |
var fs = require('fs'); | |
var ndjson = require('ndjson'); | |
var tweet_count = 0; | |
var serialize = ndjson.serialize(); | |
serialize.on('data', function(line) { | |
fs.appendFile('data.json', line, function (err) { | |
tweet_count = tweet_count + 1; | |
console.log(tweet_count) | |
}); | |
}); | |
t.on('tweet', function (tweet) { | |
serialize.write(tweet); | |
}); | |
watch.forEach(function(tag) { | |
t.track(tag); | |
console.log(tag); | |
}); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "stream-tweets-to-ndjson", | |
"version": "1.0.0", | |
"description": "Stream tweets matching an array for hashtags to an ndjson file.", | |
"main": "app.js", | |
"dependencies": { | |
"fs": "^0.0.1-security", | |
"ndjson": "^1.5.0", | |
"node-tweet-stream": "^2.0.3" | |
}, | |
"keywords": [ | |
"ndjson", | |
"twitter", | |
"tweets" | |
], | |
"author": "Matt Sandy", | |
"license": "ISC" | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment