Vadim Solovey spark2ignite
spark2ignite
/ load-multiple-files-in-one-job.go
Last active
March 20, 2018 18:59
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "context" | |
| "fmt" | |
| "os" | |
| "cloud.google.com/go/bigquery" | |
| ) |
spark2ignite
/ delete-older-gcloud-app-versions.sh
Created
February 21, 2018 21:09
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| VERSIONS=$(gcloud app versions list --service $1 --sort-by '~version' --format 'value(version.id)') | |
| COUNT=0 | |
| echo "Keeping the $2 latest versions of the $1 service" | |
| for VERSION in $VERSIONS | |
| do | |
| ((COUNT++)) | |
| if [ $COUNT -gt $2 ] | |
| then |
spark2ignite
/ k8s-setup-federated-clusters.sh
Created
September 13, 2017 14:16
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ## Requires these variables to be set | |
| # export PROJECT=your-projectname | |
| # export DNS_ZONE=your.domain.com. | |
| [[ -z “$PROJECT” ]] && { echo “Env var PROJECT is missing, please export PROJECT=your-project” ; exit 1; } | |
| [[ -z “$DNS_ZONE” ]] && { echo “Env var DNS_ZONE is missing, please export DNS_ZONE=your.domain.com.” ; exit 1; } | |
| # Create Clusters | |
| gcloud container clusters create west-cluster — zone us-west1-a — scopes “cloud-platform,storage-ro,logging-write,monitoring-write,service-control,service-management,https://www.googleapis.com/auth/ndev.clouddns.readwrite" | |
| gcloud container clusters create east-cluster — zone us-east1-b — scopes “cloud-platform,storage-ro,logging-write,monitoring-write,service-control,service-management,https://www.googleapis.com/auth/ndev.clouddns.readwrite" | |
| # Make Aliases | |
| kubectl config set-context east — cluster=gke_${PROJECT}_us-east1-b_east-cluster — user=gke_${PROJECT}_us-east1-b_east-cluster |
spark2ignite
/ mapping.java
Last active
August 14, 2017 11:41
How we saved over $240K per year by replacing Mixpanel with Google BigQuery, Pub/Sub, Dataflow & Kubernetes (code snippet #4)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package com.jellybtn; | |
| import com.google.api.services.bigquery.model.TableRow; | |
| import com.google.cloud.dataflow.sdk.values.TupleTag; | |
| import org.json.JSONObject; | |
| public class MapPeople extends BaseMap { | |
| MapPeople(TupleTag < TableRow > errorsTag) { | |
| super(errorsTag); | |
| } | |
| @Override | |
| String getType() { |
spark2ignite
/ pipeline.java
Last active
August 14, 2017 11:38
How we saved over $240K per year by replacing Mixpanel with Google BigQuery, Pub/Sub, Dataflow & Kubernetes (code snippet #3)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package com.jellybtn; | |
| import com.google.api.services.bigquery.model.TableReference; | |
| import com.google.api.services.bigquery.model.TableRow; | |
| import com.google.api.services.bigquery.model.TableSchema; | |
| import com.google.cloud.dataflow.sdk.Pipeline; | |
| import com.google.cloud.dataflow.sdk.io.BigQueryIO; | |
| import com.google.cloud.dataflow.sdk.io.PubsubIO; | |
| import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory; | |
| import com.google.cloud.dataflow.sdk.transforms.Filter; | |
| import com.google.cloud.dataflow.sdk.transforms.Flatten; |
spark2ignite
/ autoscaler.yaml
Created
August 14, 2017 11:27
How we saved over $240K per year by replacing Mixpanel with Google BigQuery, Pub/Sub, Dataflow & Kubernetes (code snippet #2)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "apiVersion": "extensions/v1beta1", | |
| "kind": "HorizontalPodAutoscaler", | |
| "metadata": { | |
| "name": "api" | |
| }, | |
| "spec": { | |
| "cpuUtilization": { | |
| "targetPercentage": 40 | |
| }, |
spark2ignite
/ api.js
Last active
August 14, 2017 11:36
How we saved over $240K per year by replacing Mixpanel with Google BigQuery, Pub/Sub, Dataflow & Kubernetes (code snippet #1)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| "use strict"; | |
| require('@google/cloud-trace').start(); | |
| const fs = require('fs'); | |
| const stream = require('stream'); | |
| const PubSub = require('@google-cloud/pubsub'); | |
| const util = require("util"); | |
| const request = require("request"); | |
| const bodyParser = require("body-parser"); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| FROM ubuntu:14.04.4 | |
| ENV DEBIAN_FRONTEND=noninteractive | |
| RUN apt-get update && apt-get install -y tomcat7 mysql-client vim curl | |
| EXPOSE 8080 | |
| # Workaround for https://bugs.launchpad.net/ubuntu/+source/tomcat7/+bug/1232258 | |
| RUN ln -s /var/lib/tomcat7/common/ /usr/share/tomcat7/common && \ | |
| ln -s /var/lib/tomcat7/server/ /usr/share/tomcat7/server && \ | |
| ln -s /var/lib/tomcat7/shared/ /usr/share/tomcat7/shared && \ |
spark2ignite
/ sample-list-bigquery-jobs.py
Created
April 7, 2016 08:46
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import httplib2 | |
| import pprint | |
| import sys | |
| from apiclient.discovery import build | |
| from apiclient.errors import HttpError | |
| from oauth2client.client import AccessTokenRefreshError | |
| from oauth2client.client import OAuth2WebServerFlow | |
| from oauth2client.client import flow_from_clientsecrets |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "canIpForward": false, | |
| "cpuPlatform": "Intel Sandy Bridge", | |
| "creationTimestamp": "2015-06-03T01:51:33.713-07:00", | |
| "description": "", | |
| "disks": [ | |
| { | |
| "autoDelete": true, | |
| "boot": true, |
NewerOlder