Dask Example.py

#!/usr/bin/env python
# coding: utf-8

# In[1]:


get_ipython().system('python -m pip install dask[dataframe] --upgrade')
get_ipython().system('python -m pip install Flask')


# In[13]:


import dask.dataframe as dd
from datetime import timedelta
import datetime
import pandas as pd


# In[14]:


def shape(df):
    shape = df.shape
    print(str(shape[0].compute()) + ',' + str(shape[1]))


# In[15]:


df = dd.read_csv("*.csv", assume_missing=True)
shape(df)


# In[16]:


df['Time'] = dd.to_datetime(df['Time'], format="%m/%d/%Y %H:%M").dt.year  # %m/%d/%Y %H:%M
shape(df)


# In[17]:


df.set_index('Time', sorted=True)
shape(df)


# In[36]:


year = 2019


# In[37]:


df_new = df[df.Time == year]
shape(df_new)


# In[38]:


df_new.compute().to_json()