acmiyaguchi

% ./dataproc.sh            
Copying file://test.py [Content-Type=text/x-python]...
/ [1 files][  346.0 B/  346.0 B]                                                
Operation completed over 1 objects/346.0 B.                                      
Waiting on operation [projects/amiyaguchi-dev/regions/us-west1/operations/38876a55-8a25-30f3-9a33-b9bd960ea881].
Waiting for cluster creation operation...done.                                                                                                                                                                                 
Created [https://dataproc.googleapis.com/v1beta2/projects/amiyaguchi-dev/regions/us-west1/clusters/test-3364] Cluster placed in zone [us-west1-a].
Job [3b00983a98754c8b8a7d22aed68931eb] submitted.
Waiting for job output...

acmiyaguchi

from decimal import Decimal
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt


# counts_test = [("20191119", 3693149), ("20191118", 3645259)]
# sums_test = [
#     ("20191119", Decimal("20186565809991403910"), Decimal("4709045032")),
#     ("20191118", Decimal("19764958396836226602"), Decimal("4688191083")),

acmiyaguchi

docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -v `pwd`:/app -it prio:dev bash
pip3 install -r requirements.txt
source dev-env
python3 -m prio random-shares --n-data 8 --batch-id testing --output-B b --output-A a

acmiyaguchi

$ jsonschema-transpiler --type bigquery --resolve drop struct.json
[
  {
    "fields": [
      {
        "mode": "NULLABLE",
        "name": "type",
        "type": "STRING"
 }

acmiyaguchi

#!/usr/bin/env python3

import json
import sys
from collections import deque

def main():
    # traverse a json document, finding path to leaves that match exactly with the term
    document = json.load(sys.stdin)
    

acmiyaguchi

/// Scrape the resource metrics from the top-level summary page
function scrapeResourceMetrics() {
    $("#p6n-dax-graph-container").click();
    let resource_metric_names = [
        "daxJobSectionCtrl.getJobName()",
        "$ctrl.getCurrentCpus()",
        "$ctrl.getTotalCpuInHours()",
        "$ctrl.getCurrentMemInBytes()",
        "$ctrl.getCurrentPdInBytes()",

acmiyaguchi

FROM python:3.7-stretch

RUN apt-get update && apt-get install -y libsnappy-dev openjdk-8-jre-headless
RUN pip install pyspark

# in an interactive session, run
# export SPARK_HOME=$(python -c "import pyspark; print(pyspark.__path__[0])")
ENV SPARK_HOME=/usr/local/lib/python3.7/site-packages/pyspark
RUN wget --directory-prefix $SPARK_HOME/jars/ https://storage.googleapis.com/spark-lib/bigquery/spark-bigquery-latest.jar
RUN wget --directory-prefix $SPARK_HOME/jars/ https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-latest.jar

acmiyaguchi

diff -r /var/folders/q6/m7bqqf2n05l5dvng72_xvvlw0000gn/T/tmp.XXXXXXXXXX.LkWNd1Nk/avro-no-tuple-control/activity-stream.impression-stats.1.schema.json /var/folders/q6/m7bqqf2n05l5dvng72_xvvlw0000gn/T/tmp.XXXXXXXXXX.LkWNd1Nk/avro-no-tuple/activity-stream.impression-stats.1.schema.json
159c159
<           "name": "items",
---
>           "name": "list",
diff -r /var/folders/q6/m7bqqf2n05l5dvng72_xvvlw0000gn/T/tmp.XXXXXXXXXX.LkWNd1Nk/avro-no-tuple-control/activity-stream.spoc-fills.1.schema.json /var/folders/q6/m7bqqf2n05l5dvng72_xvvlw0000gn/T/tmp.XXXXXXXXXX.LkWNd1Nk/avro-no-tuple/activity-stream.spoc-fills.1.schema.json
69c69
<           "name": "items",
---
>           "name": "list",

acmiyaguchi

diff -r /var/folders/q6/m7bqqf2n05l5dvng72_xvvlw0000gn/T/tmp.XXXXXXXXXX.QVNkpA7g/avro-no-tuple/firefox-launcher-process.launcher-process-failure.1.schema.json /var/folders/q6/m7bqqf2n05l5dvng72_xvvlw0000gn/T/tmp.XXXXXXXXXX.QVNkpA7g/avro-tuple/firefox-launcher-process.launcher-process-failure.1.schema.json
179a180,216
>       "default": null,
>       "name": "modules",
>       "type": [
>         {
>           "type": "null"
>         },
>         {
>           "type": "map",

acmiyaguchi

[
  {
    "fields": [
      {
        "mode": "NULLABLE",
        "name": "architecture",
        "type": "STRING"
      },
      {
        "mode": "NULLABLE",