ebernhardson

Notebook and output data for https://phabricator.wikimedia.org/T263781

ebernhardson

ebernhardson

ebernhardson

public Dataset<Row> buildPairsForM0Prep(Dataset<Row> df, Dataset<Row> dfOld, GlentParams params) {
        dfOld = dfOld
                .where(col("part").equalTo(params.glentDfM0PrepPartOld))  // limit to previous portion of M0Prep dataframe
                .drop(col("part"));

        Column oldTsCondition = null;
        if (dfOld.isEmpty()) {
            oldTsCondition = lit(true);
        } else {
            Row[] oldTsRows = dfOld.agg(max("q1_ts").alias("tsmax")).collect();

ebernhardson

ebernhardson

@startuml

== click log generation ==
oozie -> oozie: schedule label generation

note left
  arrow signify initiator
  of communication, not
  data flow
end note

ebernhardson

ebernhardson

FROM docker-registry.wikimedia.org/wikimedia-jessie

ENTRYPOINT ["/bin/bash"]

COPY cloudera.list /etc/apt/sources.list.d/cloudera.list
COPY cloudera.pref /etc/apt/preferences.d/cloudera.pref
COPY archive.key /root/archive.key

ENV HADDOP_CONF=/etc/hadoop/conf

ebernhardson

import argparse
import logging
import os
import re

from tempfile import TemporaryFile

import boto3
import botocore

ebernhardson

import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.feature.LabeledPoint
import org.apache.spark.rdd.RDD
import scala.collection.mutable.ArrayBuffer
import scala.util.Random

def randomVec(r: Random, size: Int): Vector = {
  val feats = for (i <- 0 to size) yield r.nextDouble
  Vectors.dense(feats.toArray)
}