This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
line_count_query=""" | |
SELECT COUNT(*) FROM BREATHE.{table_name} | |
""" | |
for table in tables: | |
res = client.query(line_count_query.format(table_name=table)).to_dataframe().values[0, 0] | |
lines_counts.append((table, res)) | |
lines_counts = pd.DataFrame(lines_counts, columns=['table', 'rows']) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import matplotlib.pyplot as plt | |
from keras.models import Sequential | |
from keras.layers import Dense | |
from keras.optimizers import Adam | |
model = Sequential() | |
model.add(Dense(10, input_dim=X_train.shape[1], activation='sigmoid')) | |
model.add(Dense(1, activation='sigmoid')) | |
model.compile(optimizer=Adam(lr=0.01), | |
loss='binary_crossentropy', |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def nn(X, y, hidden_dim=10, learning_rate=0.01, epochs=100, debug=False): | |
input_dim = X.shape[1] | |
output_dim = y.shape[1] | |
# Make our model | |
model = dict( | |
w0 = np.random.randn(input_dim, hidden_dim), | |
w1 = np.random.randn(hidden_dim, output_dim) | |
) | |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import matplotlib.pyplot as plt | |
import numpy as np | |
import pandas as pd | |
np.random.seed(1) | |
# train comes from the titantic dataset provided by | |
# kaggle (https://www.kaggle.com/c/titanic/data) | |
data = pd.read_csv('./train.csv') | |
# Preprocess data |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! $HOME/anaconda/bin/python | |
# swap $HOME with your absolute path if this doesn't work | |
import sys | |
import io | |
import os | |
from nbformat import read, write, current_nbformat | |
from argparse import ArgumentParser | |
from glob import glob | |
from shutil import rmtree |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Usage: python remove_output.py notebook.ipynb [-o] | |
""" | |
import sys | |
import io | |
import os | |
from nbformat import read, write, current_nbformat | |
from argparse import ArgumentParser | |
def remove_outputs(nb): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
## Some global parameters that you can change | |
VERSION=2.3.0 | |
SPARK=spark-$VERSION-bin-hadoop2.7 | |
INSTALL_PATH=$HOME/spark | |
URL=http://mirrors.ocf.berkeley.edu/apache/spark/spark-$VERSION/$SPARK.tgz | |
BASH_PROFILE=$HOME/.bash_profile | |
SPARK_PATH=$INSTALL_PATH/$SPARK | |
BIN_PATH=$SPARK_PATH/bin |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
cat in.json | sed 's/\(},\)/\1\'$'\n/g' | sed 's/\(\[\)/\1\'$'\n/g' | sed 's/\(\}\)\]/\1\'$'\n]/g' > out.json |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#based on http://stackoverflow.com/questions/2720014/upgrading-all-packages-with-pip | |
conda list | grep '<pip>' | cut -d ' ' -f 1 | xargs -n1 pip install -U |
NewerOlder