itochu0523

-- INSERT INTO id_mapping -- presto
With ssuid_table as 
(
select 
  td_global_id
 ,max(ssuid) as ssuid
from tmp_id_mapping
group by td_global_id
), 
 cluid_table as 

itochu0523

#!/bin/sh
#
# td command option
# -d : database name
# -w : waiting
# --output : output file name to local
# -f : file format
# -c : add column name (top line)

mode=$1

itochu0523

(function() {
  // td tag
  !function(t,e){if(void 0===e[t]){e[t]=function(){e[t].clients.push(this),this._init=[Array.prototype.slice.call(arguments)]},e[t].clients=[];for(var r=function(t){return function(){return this["_"+t]=this["_"+t]||[],this["_"+t].push(Array.prototype.slice.call(arguments)),this}},s=["addRecord","set","trackEvent","trackPageview","trackClicks","ready"],a=0;a<s.length;a++){var c=s[a];e[t].prototype[c]=r(c)}var n=document.createElement("script");n.type="text/javascript",n.async=!0,n.src=("https:"===document.location.protocol?"https:":"http:")+"//cdn.treasuredata.com/sdk/1.6.0/td.min.js";var i=document.getElementsByTagName("script")[0];i.parentNode.insertBefore(n,i)}}("Treasure",this);

    var td = new Treasure({
      host: "in.treasuredata.com",
      writeKey: "TD_API_KEY",
      database: "TD_DB_NAME"
    });

itochu0523

// Intimate Merger
!function(a,b,c,d){if(!b[a]){b[a]=function(){b[a].instances.push(this),this.init&&this.init()},b[a].instances=[];for(var e=0,f=["td_send","td_send_imid"];e<f.length;e++)b[a].prototype[f[e]]=function(a){return function(){this["tmp_"+a]=this["tmp_"+a]||[],this["tmp_"+a].push(Array.prototype.slice.call(arguments))}}(f[e]);var g=c.getElementsByTagName(d)[0],h=c.createElement(d);h.async=!0,h.src="//cf.im-apps.net/sdk/tdim-1.1.0.min.js",g.parentNode.insertBefore(h,g)}}("TDIM",window,document,"script");


!function(t,e){if(void 0===e[t]){e[t]=function(){e[t].clients.push(this),this._init=[Array.prototype.slice.call(arguments)]},e[t].clients=[];for(var r=function(t){return function(){return this["_"+t]=this["_"+t]||[],this["_"+t].push(Array.prototype.slice.call(arguments)),this}},n=["addRecord","set","trackEvent","trackPageview","ready"],s=0;s<n.length;s++){var i=n[s];e[t].prototype[i]=r(i)}var a=document.createElement("script");a.type="text/javascript",a.async=!0,a.src=("https:"===document.locatio

itochu0523

in:
  type: s3
  access_key_id: XXXXXXXXXX
  secret_access_key: YYYYYYYYYY
  bucket: sample_bucket
  path_prefix: path/to/sample_file
  parser:
    charset: UTF-8
    newline: CRLF
    type: csv

itochu0523

in:
  type: postgresql
  host: 00.00.00.00
  user: tank_user
  password: "XXXXXXXXXXXXXXXX"
  database: datatank
  query: |
    SELECT os,device,flag,count 
    FROM device_master
out:

itochu0523

in:
  type: s3
  bucket: td-test-data
  path_prefix: test/example.log
  access_key_id: TTTTTTTTTTTTTTTTTT
  secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXX
  parser:
    type: fluent-s3-log
    columns:
    - {name: uid, type: string}

itochu0523

ito@ito:~/embulk$ embulk preview config/s3_fluent_load.yml 
2015-12-02 00:28:28.930 -0800: Embulk v0.7.1
2015-12-02 00:28:30.186 -0800 [INFO] (preview): Loaded plugin embulk-input-s3 (0.2.3)
2015-12-02 00:28:30.229 -0800 [INFO] (preview): Loaded plugin embulk-parser-fluent-s3-log (0.0.1)
java.lang.IllegalArgumentException: Multiple entries with same key: material_id=org.embulk.spi.util.dynamic.StringColumnSetter@5a82bc58 and material_id=org.embulk.spi.util.dynamic.StringColumnSetter@4aab7195
        at com.google.common.collect.ImmutableMap.checkNoConflict(com/google/common/collect/com/google/common/collect/com/google/common/collect/ImmutableMap.java:150)
        at com.google.common.collect.RegularImmutableMap.checkNoConflictInBucket(com/google/common/collect/com/google/common/collect/com/google/common/collect/RegularImmutableMap.java:104)
        at com.google.common.collect.RegularImmutableMap.<init>(com/google/common/collect/com/google/common/collect/com/google/common/collect/RegularImmutableMap.java:70)

itochu0523

SELECT
  TD_TIME_FORMAT(TD_TIME_PARSE(session_start_time),'yyyy-MM-dd') as date
 ,count(distinct session_id) as session_cnt
FROM
 session_summary
GROUP BY
 TD_TIME_FORMAT(TD_TIME_PARSE(session_start_time),'yyyy-MM-dd')
ORDER BY 
 TD_TIME_FORMAT(TD_TIME_PARSE(session_start_time),'yyyy-MM-dd')

itochu0523

--hive
SELECT 
  TD_SESSIONIZE(time, 86400, td_ip) as session_id
, time
, td_ip
, td_path
, td_client_id
, td_title
, td_browser
, td_color