bitcoin_cleaning.py

#data storage
import h5py
#matrix math
import numpy as np
#data preprocessing
import pandas as pd

#ETL - Extract, Transform, and Load Data Class
class ETL:

	def clean_data(self, filepath, batch_size, x_window_size, y_window_size, y_col, filter_cols, normalise):
		"""Cleans and Normalises the data in batches `batch_size` at a time"""
		data = pd.read_csv(filepath, index_col=0)

		if(filter_cols):
			#Remove any columns from data that we don't need by getting the difference between cols and filter list
			rm_cols = set(data.columns) - set(filter_cols)
			for col in rm_cols:
				del data[col]

		#Convert y-predict column name to numerical index
		y_col = list(data.columns).index(y_col)

		num_rows = len(data)
		x_data = []
		y_data = []
		i = 0
		while((i+x_window_size+y_window_size) <= num_rows):
			x_window_data = data[i:(i+x_window_size)]
			y_window_data = data[(i+x_window_size):(i+x_window_size+y_window_size)]

			#Remove any windows that contain NaN
			if(x_window_data.isnull().values.any() or y_window_data.isnull().values.any()):
				i += 1
				continue
			
			if(normalise):
				abs_base, x_window_data = self.zero_base_standardise(x_window_data)
				_, y_window_data = self.zero_base_standardise(y_window_data, abs_base=abs_base)

			#Average of the desired predicter y column
			y_average = np.average(y_window_data.values[:, y_col])
			x_data.append(x_window_data.values)
			y_data.append(y_average)
			i += 1

			#Restrict yielding until we have enough in our batch. Then clear x, y data for next batch
			if(i % batch_size == 0):
				#Convert from list to 3 dimensional numpy array [windows, window_val, val_dimension]
				x_np_arr = np.array(x_data)
				y_np_arr = np.array(y_data)
				x_data = []
				y_data = []
				yield (x_np_arr, y_np_arr)