mavencode01

#!/bin/bash

# Get directory of script
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

if [[ $# -ne 1 ]]
then
    echo "ERROR: This script expects the namespace name to be given as an argument"
    echo "e.g. ./ecr-cred-updater.sh my-namespace"
    exit 1

mavencode01

package main
import (
	"fmt"
	"io/ioutil"
	"net/http"
)
func main() {
	url := "https://api.ipify.org?format=text"	// we are using a pulib IP API, we're using ipify here, below are some others
                                              // https://www.ipify.org
                                              // http://myexternalip.com

mavencode01

#!/bin/bash -e

mkdir -p /usr/local/phoenix

PHOENIX_VERSION=4.9.0
HBASE_VERSION=1.1
HDINSIGHT_VERSION=2.6.2.25-1

PHOENIX_HOME=/usr/local/phoenix/apache-phoenix-$PHOENIX_VERSION-HBase-$HBASE_VERSION-bin

mavencode01

#!/usr/bin/env bash

# Deploy a new image to an ECS service by creating a new task revision 
# specifying a container repoository tag, and updating the service to use the new revision.
#
# Note: Your application's container MUST be the first container in the task revision.

#The tag to deploy.  Specify as the first cli argument
TAG=$1

mavencode01

#!/usr/bin/env bash

# install docker
# https://docs.docker.com/engine/installation/linux/ubuntulinux/

# install docker-compose
# https://docs.docker.com/compose/install/

# install letsencrypt
# https://www.digitalocean.com/community/tutorials/how-to-secure-nginx-with-let-s-encrypt-on-ubuntu-16-04

mavencode01

#!/usr/bin/env bash

# install docker
# https://docs.docker.com/engine/installation/linux/ubuntulinux/

# install docker-compose
# https://docs.docker.com/compose/install/

# install letsencrypt
# https://www.digitalocean.com/community/tutorials/how-to-secure-nginx-with-let-s-encrypt-on-ubuntu-16-04

mavencode01

package com.databricks.spark.jira

import scala.io.Source

import org.apache.spark.rdd.RDD

import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.sources.{TableScan, BaseRelation, RelationProvider}

mavencode01

package com.databricks.spark.jira

import scala.io.Source

import org.apache.spark.rdd.RDD

import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.sources.{TableScan, BaseRelation, RelationProvider}

mavencode01

package com.mavencode.clustering


import java.util.Properties

import com.typesafe.config.ConfigFactory
import org.apache.log4j.{Level, LogManager}
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{SaveMode, SparkSession}

mavencode01

1. Issue with Spark scratch space growing too much and running out of disk space eventually lead to failed job 

Solution:
Removing "org.apache.spark.serializer.KryoSerializer" seems to solve the problem
//.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")


## Your executor needs memory, 
One of the reason you get OOM exception is because the partition data your executor needs to process is
more than what you have provided.