bindiego

SELECT
    timestamp AS Date,
    resource.labels.project_id AS ProjectId,
    protopayload_auditlog.serviceName AS ServiceName,
    protopayload_auditlog.methodName AS MethodName,
    protopayload_auditlog.status.code AS StatusCode,
    protopayload_auditlog.status.message AS StatusMessage,
    protopayload_auditlog.authenticationInfo.principalEmail AS UserId,
    protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobName.jobId AS JobId,
    protopayload_auditlog.servicedata_v1_bigquery.jobCompletedEvent.job.jobConfiguration.query.query AS Query,

bindiego

#      ___             ___ __      __
#     /   | __  ______/ (_) /_____/ /
#    / /| |/ / / / __  / / __/ __  / 
#   / ___ / /_/ / /_/ / / /_/ /_/ /  
#  /_/  |_\__,_/\__,_/_/\__/\__,_/   
#                                    
# Linux Audit Daemon - Best Practice Configuration 
# /etc/audit/audit.rules

# Remove any existing rules

bindiego

package main

import (
    "fmt"
    "os"
    "os/exec"
    "strings"
    "bytes"
    //"time"
)

bindiego

#!/bin/bash -ex

gsutil cp gs://cloud-training-demos/sandiego/sensor_obs2008.csv.gz .

bindiego

bindiego

with n_dups as
(
  SELECT event_name, event_timestamp, user_pseudo_id, count(1)-1 as n_duplicates
  FROM `project.dataset.events_20190610`
  group by event_name, event_timestamp, user_pseudo_id
)
select n_duplicates, count(1) as n_cases
from n_dups
group by n_duplicates
order by n_cases desc

bindiego

Check storage performance with dd

Tests

One process reading from $MY_DISK

dd if=$MY_DISK of=/dev/null bs=1MiB count=200 iflag=nocache

bindiego

#!/bin/bash

apt-get update || true

ROLE=$(/usr/share/google/get_metadata_value attributes/dataproc-role)

if [[ "${ROLE}" == 'Master' ]]; then
  apt-get install -y vim
else
  # something that goes only on worker nodes

bindiego

#!/bin/bash

pwd=`pwd`

bq mk --table bindiego:click_stream.events $pwd/schema.json

bindiego

Server

Edit file /etc/rsyslog.conf

 # provides UDP syslog reception
 module(load="imudp")
 input(type="imudp" port="514")

 # provides TCP syslog reception