justinnaldzin’s gists

justinnaldzin / git_merge_master.sh

Created September 5, 2018 15:58

Merge the master branch into the current checked out branch

	#!/bin/bash -x

	# Merge the master branch into the current checked out branch

	CURRENT=`git branch \| grep "*" \| awk '{print $2}'`
	git checkout master
	git fetch
	git merge origin/master
	git checkout ${CURRENT}
	git merge master ${CURRENT}

justinnaldzin / get_folders_in_s3_bucket.py

Created August 31, 2018 14:14

List all folder names in S3 bucket under a prefix

	import boto3

	bucket_name = 'my_bucket_name'
	prefix = 'some/path/'

	print("Getting list of all folder names in S3 bucket {} under prefix {}".format(bucket_name, prefix))
	folders_list = []
	client = boto3.client('s3')
	results = client.list_objects(Bucket=bucket_name, Prefix=prefix, Delimiter='/')
	for folder in results.get('CommonPrefixes'):

justinnaldzin / get_ec2_instance_name.py

Created August 27, 2018 15:19

Returns the EC2 instance name

	import logging

	import boto3
	from botocore.exceptions import ClientError, BotoCoreError
	import requests
	from requests import RequestException


	def get_instance_name():
	try:

justinnaldzin / gcloud_storage_delete.sh

Created August 6, 2018 13:00

Google Cloud Storage delete operations

justinnaldzin / gcloud_kms_iam.sh

Created August 6, 2018 12:58

Google Cloud KMS using IAM

	# https://cloud.google.com/kms/docs/iam

	# Add IAM policy binding to a specific KMS keyring with the cryptoKeyEncrypterDecrypter role
	KEYRING=my_keyring_name
	USER_EMAIL=serviceAccount:[email protected]
	gcloud kms keyrings add-iam-policy-binding $KEYRING --location global --member user:$USER_EMAIL --role roles/cloudkms.cryptoKeyEncrypterDecrypter

	# Add IAM policy binding to a specific KMS key with the cryptoKeyEncrypterDecrypter role
	KEY=my_key_name
	KEYRING=my_keyring_name

justinnaldzin / spark_dataframe_size_estimator.py

Created July 18, 2018 19:29

Estimate size of Spark DataFrame in bytes

	# Function to convert python object to Java objects
	def _to_java_object_rdd(rdd):
	""" Return a JavaRDD of Object by unpickling
	It will convert each Python object into Java object by Pyrolite, whenever the
	RDD is serialized in batch or not.
	"""
	rdd = rdd._reserialize(AutoBatchedSerializer(PickleSerializer()))
	return rdd.ctx._jvm.org.apache.spark.mllib.api.python.SerDe.pythonToJava(rdd._jrdd, True)

	# Convert DataFrame to an RDD

justinnaldzin / delete_tables_in_bigquery_dataset.py

Created July 13, 2018 05:01

Delete all tables within a BigQuery dataset

	# Delete all tables within a BigQuery dataset
	from google.cloud import bigquery

	bigquery_client = bigquery.Client()
	bq_dataset = 'my_dataset'
	dataset_ref = bigquery_client.dataset(bq_dataset)
	tables = list(bigquery_client.list_dataset_tables(dataset_ref))
	for table in tables:
	bigquery_client.delete_table(table)

justinnaldzin / replace_dictionary_key_chars.py

Last active April 27, 2018 02:25

Recursively replace characters from keys of a Python dictionary

	################################################################################
	# The following demonstrates how to recursively replace specific characters
	# from within keys of a nested dictionary or JSON object, leaving values as is.
	################################################################################

	# Given the following example dictionary with characters needing to be replaced:
	example = [
	{
	"_id": "5ae2821988fc6a16af73aeb0",
	"index": 0,

justinnaldzin / gcs_bucket.py

Created April 25, 2018 03:04

Define Google Cloud Storage bucket (create bucket if it doesn't exist)

	from google.cloud import storage

	# Define Google Cloud Storage bucket
	storage_client = storage.Client()
	bucket = storage_client.lookup_bucket(bucket_name)
	if bucket:
	print("Using bucket '{}'.".format(bucket.name))
	else:
	# Create Google Cloud Storage bucket if it doesn't exist
	print("Bucket '{}' doesn't exist. Creating bucket...".format(bucket_name))

justinnaldzin / bootstrap_social_buttons.html

Created April 4, 2017 14:57

HTML example using Bootstrap Social Buttons

	<!DOCTYPE html>
	<html>
	<head>
	<title>Bootstrap Social Buttons</title>
	<meta name="viewport" content="width=device-width, initial-scale=1">
	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css">
	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
	<style>
	/*
	* Social Buttons for Bootstrap

Justin Naldzin justinnaldzin