loading_data.py

import pandas as pd
import modin.pandas as pd_modin
import cudf as pd_cudf

results_loading = []

### Read in the data with Pandas
for run in range(0,30):
	s = time.time()
	df = pd.read_csv("../inep/dados/microdados_educacao_superior_2018//microdados_ed_superior_2018/dados/DM_ALUNO.CSV")
	e = time.time()
	results_loading.append({"lib":"Pandas","time":float("{}".format(e-s))})
	print("Pandas Loading Time = {}".format(e-s))

### Read in the data with Modin
for run in range(0,30):
	s = time.time()
	df = pd_modin.read_csv("../inep/dados/microdados_educacao_superior_2018//microdados_ed_superior_2018/dados/DM_ALUNO.CSV")
	e = time.time()
	results_loading.append({"lib":"Modin","time":float("{}".format(e-s))})
	print("Modin Loading Time = {}".format(e-s))

### Read in the data with cudf
for run in range(0,30):
	s = time.time()
	df = pd_cudf.read_csv("../inep/dados/microdados_educacao_superior_2018//microdados_ed_superior_2018/dados/DM_ALUNO.CSV")
	e = time.time()
	results_loading.append({"lib":"Cudf","time":float("{}".format(e-s))})
	print("Cudf Loading Time = {}".format(e-s))