Last active
March 28, 2016 17:56
-
-
Save ukd1/3e5560891e17a8ca635e to your computer and use it in GitHub Desktop.
use webtask.io + twilio to find things on PCA fast.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var request = require('request'); | |
return function (context, callback) { | |
var required_params = ['TWILIO_AUTH_TOKEN', 'TWILIO_ACCOUNT_SID', 'TWILIO_NUMBER', 'SEARCH', 'TO']; | |
for (var p in required_params) | |
if (!context.data[required_params[p]]) | |
return callback(new Error('The `' + required_params[p] + '` parameter must be provided.')); | |
function message(to, message) { | |
request({ | |
url: 'https://api.twilio.com/2010-04-01/Accounts/' + context.data.TWILIO_ACCOUNT_SID + '/Messages', | |
method: 'POST', | |
auth: { | |
user: context.data.TWILIO_ACCOUNT_SID, | |
pass: context.data.TWILIO_AUTH_TOKEN | |
}, | |
form: { | |
From: context.data.TWILIO_NUMBER, | |
To: to, | |
Body: message | |
} | |
}, function (error, res, body) { | |
if (error) return callback(error); | |
}); | |
} | |
function check_pca(id, nxt) { | |
request({ | |
url: 'https://www.pca.org/classified-ad/' + id | |
}, function (error, res, body) { | |
if (error) { | |
return callback(error); | |
} else { | |
if (res.statusCode == 404) { | |
count404 += 1; | |
} | |
if (res.statusCode == 200 || res.statusCode == 403) { | |
nxt(id, re.exec(body)); | |
} | |
} | |
}); | |
} | |
var re = new RegExp('<title>(.+' + context.data.SEARCH + '.+?) Date Created.+<\/title>', 'igm'); | |
var count404 = 0; | |
var countDone = 0; | |
context.storage.get(function (error, data) { | |
if (error) return cb(error); | |
data = data || { counter: 313596 }; | |
while (count404 < 3 && countDone < 10) { | |
data['counter'] += 1; | |
countDone += 1 | |
id = data['counter'] | |
check_pca(id, function(id, match) { | |
if (match !== null) { | |
message(context.data.TO, match[1] + " https://www.pca.org/classified-ad/" + id); | |
} | |
}); | |
} | |
// save, minus the seen 404's | |
data['counter'] -= count404; | |
context.storage.set(data, function (error) { | |
if (error) return callback(error); | |
}); | |
callback(null, "Done: " + data['counter']) | |
}); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment