Matt Abrams abramsm
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| output.std-tree:{ | |
| root:[ | |
| {const:"root"} | |
| {branch:[ | |
| [ | |
| // record segment hits in case we want to do a quick sanity check | |
| {const:"seg_stats", filter:{from:"SRC", require:"SEG"}} | |
| {field:"DATE_YMD"} | |
| // TODO: create a field for 'SEGMENTIDS and keep track of the unique count of UIDs | |
| // and the top 20 UIDs for each SEGMENTIDS |
abramsm
/ hydra-field-day-filterout-template.hocon
Created
January 29, 2016 22:57
Hydra Field Day Filter Out
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| map.filterOut:[ | |
| {time-to-date-ymd {}} | |
| // only the most wellformed of UIDs are counted. | |
| {from:"UID", require.match:["[0-9a-f]{16}"]} | |
| // filter segment data | |
| {safely:[ | |
| {is-not-empty:"RED"} | |
| // filter segments based on the provided list | |
| {from:"SEGMENTIDS", list-apply.elementFilter.require:[%[segment-ids]%]} | |
| {is-not-empty:"SEGMENTIDS"} |
abramsm
/ hydra-field-day-fields-template.hocon
Created
January 29, 2016 22:53
Hydra Field Data - Fields Template
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| map.fields:[ | |
| // fields common to all sources | |
| "TIME" | |
| "UID" | |
| // fields specific to segment data | |
| "SEGMENTIDS" | |
| // fields specific to usync data | |
| {from:"PUID_%[pid]%", to:"PUID"} | |
| // injected source names | |
| "RED" |
abramsm
/ hydra-field-day-source-template.hocon
Created
January 29, 2016 22:49
Source Template for Hydra Field Day Tutorial
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| include "extra-filters" | |
| %{user-filters}% | |
| source:[ | |
| {mesh2:{ | |
| markDir:"pixelatorDataMarks" | |
| mesh:{ | |
| startDate:"%[segment-start-date:{{now-1}}]%" | |
| endDate:"%[segment-end-date:{{now-1}}]%" | |
| dateFormat:"yyMMdd" |
abramsm
/ gist:9237098
Created
February 26, 2014 19:52
Configuration for fast start embedded Zookeeper, useful for unit tests
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| InstanceSpec spec = new InstanceSpec(null, -1, -1, -1, true, -1, 2000, 10); | |
| System.setProperty("zookeeper.serverCnxnFactory", "org.apache.zookeeper.server.NettyServerCnxnFactory"); | |
| myKeeper = new TestingServer(spec); | |
| String keeperPort = String.valueOf(spec.getPort()); | |
| System.setProperty("zk.servers", "localhost:" + keeperPort); | |
| zkClient = CuratorFrameworkFactory.builder() | |
| .sessionTimeoutMs(60000) | |
| .connectionTimeoutMs(10000) | |
| .connectString("localhost:" + keeperPort) | |
| .retryPolicy(new RetryOneTime(1000)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| $ vagrant init precise32 http://files.vagrantup.com/precise32.box | |
| $ vagrant up | |
| $ vagrant ssh | |
| $ sudo apt-get update | |
| $ sudo apt-get install python-software-properties | |
| $ sudo add-apt-repository ppa:webupd8team/java | |
| $ sudo apt-get update | |
| $ sudo apt-get install oracle-java7-installer | |
| $ sudo apt-get install rabbitmq-server | |
| $ sudo apt-get install maven |
abramsm
/ gist:9136422
Last active
August 29, 2015 13:56
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"op":"chain", "filter":[ | |
| {"op":"field", "from":"UID"}, | |
| {"op":"equals", "left":"FIELD_ONE", "right":"FIELD_TWO", "not":true}, | |
| {"op":"concat", "in":["FOO", "BAR"], "out":"OUTPUT", "join":":"}, | |
| {"op":"num", "columns":["END", "START", "WALL"], "define":"c0,c1,sub, | |
| v1000,ddiv,toint,v2,set"}, | |
| ]} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 3.535, 5214d63bab95687d, 166.144.203.186, "the then good" | |
| 3.568, 5dbd9451948ad895, 88.120.153.226, "know boys" | |
| 4.206, 5dbd9451948ad895, 88.120.153.226, "to" | |
| 4.673, b967d99cad0b3e60, 88.120.153.226, "seven" | |
| 4.900, bd0d760fbb338955, 166.144.203.186, "did local if to" | |
| 6.166, ef909223e4873178, 166.144.203.186, "every to" | |
| 7.050, ff1fda5a8c6361fe, 166.144.203.186, "talking from wore" | |
| 8.114, 90fbf36695d3a2d, 176.205.174.108, "was i favorite papa" | |
| 8.732, 3ef5a81b79e149a6, 166.144.203.186, "us pile we it" | |
| 9.697, 8a9d23755e58f66, 88.120.153.226, "make to" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // -Dpagedb.kvstore.type=1 | |
| // -Deps.gz.type=3 | |
| // -Deps.cache.pages=100000 | |
| // -Xmx2G | |
| { | |
| "type":"map", | |
| taskthreads:2, | |
| "source":{ | |
| "type":"mesh2", |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| "output":{ | |
| "type":"tree", | |
| "root":{"path":"TREE"}, | |
| "paths":{ | |
| "TREE":[ | |
| {"type":"const", "value":"root", "data":{ | |
| "topterms":{"type":"key.top","key":"TERMS","size":1000}, | |
| "topuids":{"type":"key.top", "key":"UID", "size":1000}, | |
| "topip":{"type":"key.top", "key":"IP", "size":1000}, |
NewerOlder