Load and merge data from Rescuetime

load_rescuetime_data.py

import requests
import datetime
import pandas as pd
import json
import time


# Read rescuetime key
with open('key.txt', 'r') as f:
	key = f.read()

# Get current date
d = datetime.datetime.now()
url = 'https://www.rescuetime.com/anapi/data'

# https://www.rescuetime.com/anapi/setup/documentation#analytic-api-reference
resolution_times = ['day', 'hour', 'minute']
restrict_kinds = ['document', 'overview', 'efficiency']
perspectives = ['interval', 'rank']

# Function for cleaning up strings
#convert = lambda s: s.encode('ascii', 'ignore').replace('\n', '')
convert = lambda s: s.replace('\n', '')

start, end = '', ''

for perspective in perspectives:
	for restrict_kind in restrict_kinds:
		for resolution_time in resolution_times:
			print(perspective, restrict_kind, resolution_time)

			try:
				response = requests.get(url, params={
					'key': key,
					'format': 'json',
					'perspective': perspective,
					'interval': resolution_time,
					'restrict_kind': restrict_kind,
					'restrict_begin': '2017-01-01',
					'restrict_end': d.strftime('%Y-%m-%d')
				})
				print("Status Code : {}".format(response.status_code))
				data  = response.json()
			except Exception as e:
				print(response.text)
				print(e)
				exit()

			df = pd.DataFrame(data['rows'], columns=data['row_headers'])

			if restrict_kind == 'document':
				df['Document'] = df['Document'].apply(convert)

			if perspective == 'interval':
				start = data['rows'][ 0][0].split('T')[0]
				end = data['rows'][-1][0].split('T')[0]

			output = 'data/rt_{}_{}_{}_{}_{}.csv'.format(
						start, end, resolution_time, perspective, restrict_kind)
			print(output)
			df.to_csv(output, index=False)

			time.sleep(5)

merge_rescuetime_data.py

import os
import pandas as pd
import numpy as np


resolution_times = ['day', 'hour', 'minute']
perspective = 'interval'
restrict_kind = 'document'

files = [os.path.join('data', f) for f in os.listdir('data')
         if f.endswith('.csv') and (restrict_kind in f) and (perspective in f)]

for resolution_time in resolution_times:
    print(resolution_time)
    df_list = []
    for f in files:
        if resolution_time in f:
            df = pd.read_csv(f)
            print(f, df.shape)
            df_list.append(df)

    df = pd.concat(df_list)
    df['Date'] = pd.to_datetime(df['Date'])
    df = df.sort_values(by='Date')
    df.drop_duplicates(inplace=True)
    print(df.shape)

    filepath = 'rt_{}_{}_{}.csv'.format(
        resolution_time, perspective, restrict_kind)
    df.to_csv(filepath, index=False)