randyzwitch

#Installing BreakoutDetection package
install.packages("devtools")
devtools::install_github("twitter/BreakoutDetection")
library(BreakoutDetection)

library("RSiteCatalyst")
SCAuth("company", "secret")

#Get pageviews for each day in 2014
pageviews_2014 <- QueueOvertime('report-suite', 

randyzwitch

#Example
{
  "timestamp": 1324830675.076,
  "status": "404",
  "short_message": "File does not exist: /var/www/no-such-file",
  "host": "ord1.product.api0",
  "facility": "httpd",
  "errno": "ENOENT",
  "remote_host": "50.57.61.4",
  "remote_port": "40100",

randyzwitch

#Multi-page pathing
library("d3Network")
library("RSiteCatalyst")

#### Authentication
SCAuth("name", "secret")

#### Get All Possible Paths with ("::anything::", "::anything::")
pathpattern <- c("::anything::", "::anything::")
next_page <- QueuePathing("zwitchdev",

randyzwitch

library("RSiteCatalyst")
library("d3Network")

#### Authentication
SCAuth("key", "secret")

#### Get Pathing data: Single page, then ::anything:: pattern
pathpattern <- c("http://randyzwitch.com/big-data-hadoop-amazon-ec2-cloudera-part-1", "::anything::")
next_page <- QueuePathing("zwitchdev",
                          "2014-01-01",

randyzwitch

import pandas as pd
from matplotlib import pyplot as plt
import matplotlib as mpl
import seaborn as sns
%matplotlib inline

#Read in data & create total column
stacked_bar_data = pd.read_csv("C:\stacked_bar.csv")
stacked_bar_data["total"] = stacked_bar_data.Series1 + stacked_bar_data.Series2

randyzwitch

#### Force directed network

#Limit to more than 5 occurence like in simple network
fd_graph_links <- subset(graph_links, count > 5)

#Get unique values of page name to create nodes df
#Create an index value, starting at 0
fd_nodes <- as.data.frame(unique(c(fd_graph_links$step.1, fd_graph_links$step.2)))
names(fd_nodes) <- "name"
fd_nodes$nodevalue <- as.numeric(row.names(fd_nodes)) - 1

randyzwitch

#### Second pass: thin the spaghetti blob!
#Require path to happen more than some number of times (count > x)
#What constitutes "low volume" will depend on your level of traffic
simpleoutput2 = "C:/Users/rzwitc200/Desktop/simpleoutput2.html"
d3SimpleNetwork(subset(graph_links, count > 5), Source = "step.1", Target = "step.2", height = 600,
                width = 750, fontsize = 12, linkDistance = 50, charge = -100,
                linkColour = "#666", nodeColour = "#3182bd",
                nodeClickColour = "#E34A33", textColour = "#3182bd", opacity = 0.6,
                standAlone = TRUE, file = simpleoutput2)

randyzwitch

#Optional step: Cleaning my pagename URLs to remove to domain for graph clarity
queue_pathing_pages$step.1 <- sub("http://randyzwitch.com/","", 
                                  queue_pathing_pages$step.1, ignore.case = TRUE)
queue_pathing_pages$step.2 <- sub("http://randyzwitch.com/","", 
                                  queue_pathing_pages$step.2, ignore.case = TRUE)

#### Remove Enter and Exit site values
#This information is important for analysis, but not related to website structure
graph_links <- subset(queue_pathing_pages, step.1 != "Entered Site" & step.2 != "Exited Site")

randyzwitch

library("RSiteCatalyst")
library("d3Network")

#### Authentication
SCAuth("username", "secret")

#### Get Pathing data using ::anything:: wildcards
# Results are limited by the API to 50000
pathpattern <- c("::anything::", "::anything::")

randyzwitch

# Rectangles
eo = make_euler_object(labels, data, [EulerSpec(:rectangle), EulerSpec(:rectangle, [.5, .5, .4], [0, 0, 0]), 
    EulerSpec(:rectangle)],
    sizesum=.3)


(minf,minx,ret) = optimize_iteratively(eo, random_state(eo), ftol=-1, xtol=0.0025, maxtime=5, pop=100)
println("phase 1: got $minf at $minx (returned $ret)")
(minf,minx,ret) = optimize(eo, minx, ftol=-1, xtol=0.001, maxtime=30, pop=100)
println("phase 2: got $minf at $minx (returned $ret)")