thekensta

/*
# https://cloud.google.com/bigquery/audit-logs#sample_audit_queries_in_bigquery

select   
  protopayload_auditlog.authenticationInfo.principalEmail as user_email, 
  protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query, 
--   protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.state,
--   protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.code,
--   protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatus.error.message,
  sum(protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobStatistics.totalBilledBytes) billedBytes, 

thekensta

thekensta

--
-- Week Daily for non-Orthogonal Date dimensions e.g. day of week
--
with Calendar as (
  select 
    this_year,  
    date_add(this_year, interval - 364 day) as last_year
  from 
    unnest(GENERATE_DATE_ARRAY('2017-01-01', '2017-12-31')) as this_year
),

thekensta

#!/usr/bin/env bash

DATASET="<YOUR DATASET HERE>"
TABLE="fun_with_time"
SCHEMA_FILE="/tmp/ts-schema.json"
DATA_FILE="/tmp/ts-data.ndjson"
SQL_FILE="/tmp/query.sql"
BUCKET="<YOUR BUCKET HERE>"

cat <<EOM >${SCHEMA_FILE}

thekensta

"""
Script to scrape Gigaclear status for each postcode (in this case in GL* areas) 
Writes a Json file with lat-lon, postcode and response for display and
later reference
"""

import requests
import csv
import io
import time

thekensta

thekensta

thekensta

with X as (
--  select STRUCT<x int64, y string>(1, 'a') as R
   select * 
--   from unnest(split("I am the bad man, I am the bad man", " ")) as w with offset as c
  from 
  unnest([
  -- Will be named and explictly typed
--           STRUCT<x int64, y string>(1, 'a'), 
--           STRUCT<x int64, y string>(2, 'b'),
--           STRUCT<x int64, y string>(3, 'c')

thekensta

x = pd.DataFrame({"Date": np.repeat(pd.date_range("2015-01-01", "2015-01-04"), 2), 
                  "Class": [1, 2, 1, 2, 1, 2, 1, 2], 
                  "Value": np.random.random(size=8)})

p1 = pd.pivot_table(x, index="Date", columns="Class")
p2 = pd.pivot_table(x, index="Date", columns="Class", values="Value")

p1
#               Value          
# Class              1         2

thekensta

"""Really simple example of reading aws Key and Secret from credentials file. """

import os 
from ConfigParser import SafeConfigParser # Python2 

def main(profile="default"):
  config = SafeConfigParser()
  config.read(os.path.join(os.getenv("HOME"), ".aws/credentials"))
  
  secret = config.get(profile, "aws_secret_access_key")