IanHopkinson

# Some experiments in R to handle twitter profile data
# Ian Hopkinson 2013-03-28

# First we get the data in JSON format and then convert to a data.frame
library(rjson)
TwitterRecordsFile = 'http://box.scraperwiki.com/bdaw7xa/a6e8e71d96b2449/sql?q=select+*+from+twitter_followers'
TwitterRecords = fromJSON(file=TwitterRecordsFile, method='C')
Twitterdf=data.frame(t(sapply(TwitterRecords,c)))
# smoothScatter is in base R
smoothScatter(Twitterdf$followers_count,Twitterdf$following_count,xlim=c(0,5000),ylim=c(0,5000),main="Followers vs following count for a subset of twitter users")

IanHopkinson

def threshold_above(hist, threshold_value):
    """
    >>> threshold_above(collections.Counter({518: 10, 520: 20, 530: 20, 525: 17}), 15)
    [520, 530, 525]
    """
    if not isinstance(hist,collections.Counter):
        raise ValueError("requires collections.Counter")
   
    above = [k for k, v in hist.items() if v > threshold_value]
    return above

IanHopkinson

SET storage_engine=MyISAM;
CREATE SCHEMA IF NOT EXISTS `MOT` ;
USE `MOT` ;

DROP TABLE IF EXISTS TESTRESULT;

CREATE TABLE TESTRESULT (
  TESTID INT UNSIGNED
	,VEHICLEID INT UNSIGNED
	,TESTDATE DATE

IanHopkinson

USE MOT;

LOAD DATA LOCAL INFILE '[yourpath \\ as separator in windows]failure_location.txt'
INTO TABLE failure_location
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
;

LOAD DATA LOCAL INFILE '[yourpath \\ as separator in windows]item_detail.txt'
INTO TABLE testitem_detail

IanHopkinson

USE MOT;

ALTER TABLE testitem DISABLE KEYS;

LOAD DATA LOCAL INFILE '[yourpath \\ separated in Windows]UAT_test_item_2011.txt'
INTO TABLE testitem
FIELDS TERMINATED BY '|'
LINES TERMINATED BY '\n'
;

IanHopkinson

[Results2008Cohort.csv]
CharacterSet=65001
Format=Delimited(|)
ColNameHeader=False
Col1="TESTID" Integer
Col2="VEHICLEID" Integer
Col3="TESTDATE" Char 
Col4="TESTCLASSID" Char
Col5="TESTTYPE" Char
Col6="TESTRESULT" Char

IanHopkinson

function save_api_stub(){
  scraperwiki.exec('echo "' + scraperwiki.readSettings().target.url + '" > ~/tool/dataset_url.txt; ')
}

function run_once_install_packages(){
  scraperwiki.exec('run-one tool/runonce.R &> tool/log.txt &')
}

$(function(){
   save_api_stub();

IanHopkinson

#!/usr/bin/Rscript
# Script to knit a file 2013-08-08
# Ian Hopkinson
library(knitr)
.libPaths('/home/tool/R/libraries')
render_html()
knit("/home/tool/view.Rhtml",output="/home/tool/http/index.html")

IanHopkinson

#!/usr/bin/Rscript
# Script to create r-view 2013-08-14
# Ian Hopkinson

source('scraperwiki_utils.R')

NumberOfTweets<-function(){
  query = 'select count(*) from tweets'
  number = ScraperWikiSQL(query)
  return(number)

IanHopkinson

#!/usr/bin/env python
"""
    Save stock ticker data from Yahoo! Finance to sqlite.
"""
import datetime as d
import sqlite3

import pandas.io.data as web
import pandas.io.sql as sql