aaronsteers

import org.apache.spark.sql.jdbc.{JdbcDialects, JdbcType, JdbcDialect}
import org.apache.spark.sql.jdbc.JdbcType
val SQLServerDialect = new JdbcDialect {
  override def canHandle(url: String): Boolean = url.startsWith("jdbc:jtds:sqlserver") || url.contains("sqlserver")

  override def getJDBCType(dt: DataType): Option[JdbcType] = dt match {
    case StringType => Some(JdbcType("VARCHAR(5000)", java.sql.Types.VARCHAR))
    case BooleanType => Some(JdbcType("BIT(1)", java.sql.Types.BIT))
    case IntegerType => Some(JdbcType("INTEGER", java.sql.Types.INTEGER))
    case LongType => Some(JdbcType("BIGINT", java.sql.Types.BIGINT))

aaronsteers

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;

public class HiveMetastoreJDBCTest {

aaronsteers

## This moves execution into the script's parent directory.
## See also: http://stackoverflow.com/questions/59895/can-a-bash-script-tell-which-directory-it-is-stored-in
ORIG_DIR=$(pwd)
SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
SCRIPT_PARENT_DIR="$(dirname "$SCRIPT_DIR")"
cd $SCRIPT_DIR
##
## Do stuff...
##
cd $ORIG_DIR

aaronsteers

from pyspark import SparkContext, SparkConf
from pyspark.sql import HiveContext, SQLContext
import pandas as pd

# sc: Spark context
# file_name: csv file_name
# table_name: output table name
# sep: csv file separator
# infer_limit: pandas type inference nb rows
def read_csv(sc, file_name, table_name, sep=",", infer_limit=10000):

aaronsteers

# Step 1: Create a CalendarGenerator() function and paste the contents below into the Advanced Editor
let CreateDateTable = (StartDate as date, EndDate as date, optional Culture as nullable text) as table =>
  let
    DayCount = Duration.Days(Duration.From(EndDate - StartDate)),
    Source = List.Dates(StartDate,DayCount,#duration(1,0,0,0)),
    TableFromList = Table.FromList(Source, Splitter.SplitByNothing()),
    ChangedType = Table.TransformColumnTypes(TableFromList,{{"Column1", type date}}),
    RenamedColumns = Table.RenameColumns(ChangedType,{{"Column1", "Date"}}),
    InsertYear = Table.AddColumn(RenamedColumns, "Year", each Date.Year([Date])),
    InsertQuarter = Table.AddColumn(InsertYear, "Quarter Number", each Date.QuarterOfYear([Date])),

aaronsteers

## A simple CI script that builds the data warehouse and tests that any dependent views are still able to compile
##
##
## Schema Names:
##    dw    production schema
##    dev   shared development schema


build_on_master:
  stage: build

aaronsteers

### This script will execute any T-SQL scripts located in a named subfolder (taken as the first script argument).
### Any references to the templated schema name ("DWCI_Test") will be replaced with the value that is provided
### as the 2nd command line argument.

Param(
    [string]$subFolder,
    [string]$deploymentSchema
)

$ErrorActionPreference = "Stop"

aaronsteers

{
  "name": "Sample Dark Theme",
  "background": "#000000",
  "backgroundLight": "#9b9b9b",
  "backgroundNeutral": "#535353",
  "foreground": "#ffffff",
  "foregroundNeutralSecondary": "#ffffff",
  "foregroundNeutralTertiary": "#ffffff",
  "tableAccent": "#0c62fb",
  "dataColors": [

aaronsteers

// To view the default settings, hold "alt" while clicking on the "Settings" button.
// For documentation on these settings, see: https://aka.ms/terminal-documentation

{
    "$schema": "https://aka.ms/terminal-profiles-schema",
    "defaultProfile": "{2c4de342-38b7-51cf-b940-2309a097f518}",
    "profiles":
    [
        {

aaronsteers

#!/bin/bash

# Echo this entire script to a logfile.txt
exec > >(tee -a $HOME/logfile.txt) 2>&1

echo "Initializing hostname (needed for ECS)..."

echo "Detecting 'eth1' interface..."
DETECTED_IP=$(ifconfig -a | grep -A2 eth1 | grep inet | awk '{print $2}' | sed 's#/.*##g' | grep "\.")
if [[ -z $DETECTED_IP ]]; then