This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from sklearn import metrics | |
def get_classification_as_df(y_test: list, y_pred: list, sort_by: list =[])-> pd.DataFrame: | |
''' Get the classification report as a DataFrame''' | |
report = metrics.classification_report(y_test, y_pred, output_dict=True) | |
df_classification_report = pd.DataFrame(report).transpose() | |
df_classification_report = df_classification_report.sort_values( | |
by=sort_by, ascending=False) | |
return df_classification_report |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# A simple cheat sheet of Spark Dataframe syntax | |
# Current for Spark 1.6.1 | |
# import statements | |
from pyspark.sql import SQLContext | |
from pyspark.sql.types import * | |
from pyspark.sql.functions import * | |
#creating dataframes | |
df = sqlContext.createDataFrame([(1, 4), (2, 5), (3, 6)], ["A", "B"]) # from manual data |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import contextlib | |
import os | |
import tempfile | |
half_lambda_memory = 10**6 * ( | |
int(os.getenv('AWS_LAMBDA_FUNCITON_MEMORY_SIZE', '0')) / 2) | |
@contextlib.contextmanager |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | |
AWSTemplateFormatVersion: '2010-09-09' | |
Description: Simple S3 Bucket with SNS Trigger | |
Parameters: | |
BucketName: | |
Type: String | |
Description: The name of the S3 Bucket to create |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
set -e | |
docker run -dit --rm --name test tensorflow/tensorflow:latest-gpu-py3-jupyter | |
TF_VERSION=`docker exec -it test bash -c "pip freeze | grep tensorflow-gpu | cut -d'=' -f3"` | |
TF_VERSION=`echo $TF_VERSION | tr -d '\r'` | |
docker stop test |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
lastversion() { | |
basename $(curl -Ls -o /dev/null -w %{url_effective} https://github.com/$1/releases/latest) | |
} | |
lastversion $1 | |
# usage e.g. ./last_release_version.sh tensorflow/tensorflow |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
get_latest_release() { | |
curl --silent "https://api.github.com/repos/$1/releases/latest" | # Get latest release from GitHub api | |
grep '"tag_name":' | # Get tag line | |
sed -E 's/.*"([^"]+)".*/\1/' # Pluck JSON value | |
} | |
# Usage | |
# $ get_latest_release "creationix/nvm" | |
# v0.31.4 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import dateutil.parser | |
import optparse | |
import csv | |
import tempodb | |
from threading import Thread | |
from Queue import Queue | |
class Worker(Thread): | |
"""Thread executing tasks from a given tasks queue""" |
NewerOlder