Skip to content

Instantly share code, notes, and snippets.

@brian-lc
Last active August 29, 2015 14:23
Show Gist options
  • Select an option

  • Save brian-lc/4a8ee95e73e236a2c491 to your computer and use it in GitHub Desktop.

Select an option

Save brian-lc/4a8ee95e73e236a2c491 to your computer and use it in GitHub Desktop.
Scrapes the PG&E website for energy and billing data using your login/password
# Call this from casperjs cli like this
# casperjs pge_scrape.coffee --user=<your_username> --password=<your_password>
# Works on Phantom 1.98 and Casper 1.1 beta
casper = require('casper').create
viewportSize: {width: 800, height: 600},
pageSettings: {webSecurityEnabled: false}
user = casper.cli.get('user')
password = casper.cli.get('pass')
casper.start "http://www.pge.com", ->
# Login to the PG&E portal
login =
USER: user,
PASSWORD: password
@fill "form#login-form", login, true
casper.then ->
# Update status and grab screen
@echo("Logged in!").capture("pge_account.png")
casper.then ->
# Click the My Usage tab
@click("li#primaryNav3 a")
casper.then ->
# Update status and grab screen
@echo("Viewing usage").capture("pge_myusage.png")
casper.then ->
# Click the Green Button icon
@click("a[data-trigger-dialog='href']")
casper.then ->
# TODO: make event based on load of content in dialog
# TODO: Look at waitFor()
@echo("Waiting for the dialog animation").wait(2000)
casper.then ->
@echo("Green Button dialog open").capture("pge_greenbutton.png")
casper.then ->
#select the option to export interval data
@click("input[name='exportFormat'][value='ESPI_AMI']")
@click("input[value='Export']")
casper.then ->
@echo("Waiting for download to finish").wait(2000)
casper.then ->
@reload()
casper.then ->
# Click the Green Button icon
@click("a[data-trigger-dialog='href']")
casper.then ->
# TODO: make event based on load of content in dialog
# TODO: Look at waitFor()
@echo("Waiting for the dialog animation").wait(2000)
casper.then ->
@echo("Green Button dialog open").capture("pge_billing.png")
casper.then ->
# Select the option to export all the bills
@click("input[name='exportFormat'][value='ESPI_BILLING']")
@click("input[value='Export']")
casper.then ->
@echo("Waiting for download to finish").wait(2000)
casper.on 'page.resource.received', (resource) ->
@echo(resource.url)
if resource.stage != "end"
return
if resource.url.indexOf('ESPI_BILLING') > -1
@echo("Downloading bills...")
@download(resource.url, 'pge_billing_data.zip')
if resource.url.indexOf('ESPI_AMI') > -1
@echo("Downloading interval...")
@download(resource.url, 'pge_interval_data.zip')
casper.run ->
@echo("Done!").exit()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment