ebernhardson

            // The LatencyStat object is a POJO with three fields: bucket, percentile and latency
            // The goal is to get a new List<LatencyStat> with one object per (bucket, percentile) combination
            // averaging the latency value between them.
            Map<String, Map<Double, List<SearchLatencyProbe.LatencyStat>>> foo = nodes.stream()
                    .flatMap(n -> n.statDetails.latencies.stream())
                    .collect(Collectors.groupingBy(LatencyStat::getBucket,
                            Collectors.groupingBy(LatencyStat::getPercentile)));
            List<LatencyStat> allNodes = new ArrayList<>();
            for (Map.Entry<String, Map<Double, List<LatencyStat>>> bucketEntry : foo.entrySet()) {
                for (Map.Entry<Double, List<LatencyStat>> percentileEntry : bucketEntry.getValue().entrySet()) {

ebernhardson

import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.rdd.RDD
import scala.collection.mutable.ArrayBuffer
import scala.util.Random

def randomVec(r: Random, size: Int): Vector = {
  val feats = for (i <- 0 to size) yield r.nextDouble
  Vectors.dense(feats.toArray)
}

ebernhardson

import argparse
import logging
import os
import re

from tempfile import TemporaryFile

import boto3
import botocore

ebernhardson

FROM docker-registry.wikimedia.org/wikimedia-jessie

ENTRYPOINT ["/bin/bash"]

COPY cloudera.list /etc/apt/sources.list.d/cloudera.list
COPY cloudera.pref /etc/apt/preferences.d/cloudera.pref
COPY archive.key /root/archive.key

ENV HADDOP_CONF=/etc/hadoop/conf

ebernhardson

ebernhardson

@startuml

== click log generation ==
oozie -> oozie: schedule label generation

note left
  arrow signify initiator
  of communication, not
  data flow
end note

ebernhardson

ebernhardson

public Dataset<Row> buildPairsForM0Prep(Dataset<Row> df, Dataset<Row> dfOld, GlentParams params) {
        dfOld = dfOld
                .where(col("part").equalTo(params.glentDfM0PrepPartOld))  // limit to previous portion of M0Prep dataframe
                .drop(col("part"));

        Column oldTsCondition = null;
        if (dfOld.isEmpty()) {
            oldTsCondition = lit(true);
        } else {
            Row[] oldTsRows = dfOld.agg(max("q1_ts").alias("tsmax")).collect();

ebernhardson

ebernhardson