abramsm

$ vagrant init precise32 http://files.vagrantup.com/precise32.box
$ vagrant up
$ vagrant ssh
$ sudo apt-get update 
$ sudo apt-get install python-software-properties
$ sudo add-apt-repository ppa:webupd8team/java
$ sudo apt-get update
$ sudo apt-get install oracle-java7-installer
$ sudo apt-get install rabbitmq-server
$ sudo apt-get install maven

abramsm

        InstanceSpec spec = new InstanceSpec(null, -1, -1, -1, true, -1, 2000, 10);
        System.setProperty("zookeeper.serverCnxnFactory", "org.apache.zookeeper.server.NettyServerCnxnFactory");
        myKeeper = new TestingServer(spec);
        String keeperPort = String.valueOf(spec.getPort());
        System.setProperty("zk.servers", "localhost:" + keeperPort);
        zkClient = CuratorFrameworkFactory.builder()
                .sessionTimeoutMs(60000)
                .connectionTimeoutMs(10000)
                .connectString("localhost:" + keeperPort)
                .retryPolicy(new RetryOneTime(1000))

abramsm

include "extra-filters"
%{user-filters}%

source:[
    {mesh2:{
        markDir:"pixelatorDataMarks"
        mesh:{
            startDate:"%[segment-start-date:{{now-1}}]%"
            endDate:"%[segment-end-date:{{now-1}}]%"
            dateFormat:"yyMMdd"

abramsm

map.fields:[
    // fields common to all sources
    "TIME"
    "UID"
    // fields specific to segment data
    "SEGMENTIDS"
    // fields specific to usync data
    {from:"PUID_%[pid]%", to:"PUID"}
    // injected source names
    "RED"

abramsm

map.filterOut:[
    {time-to-date-ymd {}}
    // only the most wellformed of UIDs are counted.
    {from:"UID", require.match:["[0-9a-f]{16}"]}
    // filter segment data
    {safely:[
        {is-not-empty:"RED"}
        // filter segments based on the provided list
        {from:"SEGMENTIDS", list-apply.elementFilter.require:[%[segment-ids]%]}
        {is-not-empty:"SEGMENTIDS"}

abramsm

output.std-tree:{
    root:[
        {const:"root"}
        {branch:[
            [
                // record segment hits in case we want to do a quick sanity check
                {const:"seg_stats", filter:{from:"SRC", require:"SEG"}}
                {field:"DATE_YMD"}
                // TODO:  create a field for 'SEGMENTIDS and keep track of the unique count of UIDs
                // and the top 20 UIDs for each SEGMENTIDS