Migrate your data off of the Datica Backend as a Service product.

BaaS Data Export Script

#!/usr/bin/env python
import base64
import getpass
import json
import os
import os.path
import readline
import sys
import time
import urllib2

dashboard_api_key = 'browser developer.catalyze.io 32a384f5-5d11-4214-812e-b35ced9af4d7'


def signin():
    username = raw_input('Please enter your username: ')
    password = getpass.getpass('Please enter your password: ')
    signin_req = urllib2.Request('https://auth.catalyze.io/auth/signin', json.dumps({'identifier': username, 'password': password}), {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-Request-Timestamp': str(int(time.time())), 'X-Request-Nonce': base64.b64encode(os.urandom(32))})
    r = urllib2.urlopen(signin_req)
    signin_resp = r.read()
    r.close()
    del username
    del password
    signin_resp = json.loads(signin_resp)
    if 'mfaID' in signin_resp:
        otp = getpass.getpass('Please enter your {} one time passcode: '.format(signin_resp['mfaPreferredType']))
        otp_req = urllib2.Request('https://auth.catalyze.io/auth/signin/mfa/{}'.format(signin_resp['mfaID']), json.dumps({'otp': otp}), {'Content-Type': 'application/json', 'Accept': 'application/json', 'X-Request-Timestamp': str(int(time.time())), 'X-Request-Nonce': base64.b64encode(os.urandom(32))})
        r = urllib2.urlopen(otp_req)
        otp_resp = r.read()
        r.close()
        session_token = json.loads(otp_resp)['sessionToken']
    else:
        session_token = signin_resp['sessionToken']
    return session_token


def retrieve_apps(session_token):
    authall_req = urllib2.Request('https://api.catalyze.io/v2/auth/all', None, {'Authorization': 'Bearer {}'.format(session_token), 'X-API-Key': dashboard_api_key, 'Content-Type': 'application/json', 'Accept': 'application/json'})
    r = urllib2.urlopen(authall_req)
    authall_resp = r.read()
    r.close()

    app_tokens = {}
    for atd in json.loads(authall_resp)['results']:
        app_id = atd.pop('appId', None)
        app_tokens[app_id] = atd

    # fetch all applications
    apps_req = urllib2.Request('https://api.catalyze.io/v2/users/apps', None, {'Authorization': 'Bearer {}'.format(session_token), 'X-API-Key': dashboard_api_key, 'Content-Type': 'application/json', 'Accept': 'application/json'})
    r = urllib2.urlopen(apps_req)
    apps_resp = r.read()
    r.close()

    apps = {}
    for app in json.loads(apps_resp)['results']:
        apps[app['appId']] = app
    return app_tokens, apps


def list_users_for_app(app_id, session_token):
    user_req = urllib2.Request('https://api.catalyze.io/v2/users/{}/list'.format(app_id), None, {'Authorization': 'Bearer {}'.format(session_token), 'X-Api-Key': dashboard_api_key, 'Content-Type': 'application/json', 'Accept': 'application/json'})
    r = urllib2.urlopen(user_req)
    user_resp = r.read()
    r.close()
    return json.loads(user_resp)['results']


def retrieve_class(class_name, app_id, session_token):
    class_req = urllib2.Request('https://api.catalyze.io/v2/app/{}/classes/{}'.format(app_id, class_name), None, {'Authorization': 'Bearer {}'.format(session_token), 'X-Api-Key': dashboard_api_key, 'Content-Type': 'application/json', 'Accept': 'application/json'})
    r = urllib2.urlopen(class_req)
    class_resp = r.read()
    r.close()
    return json.loads(class_resp)


def list_files_for_user(user_id, api_key, session_token):
    files_req = urllib2.Request('https://api.catalyze.io/v2/users/{}/files'.format(user_id), None, {'Authorization': 'Bearer {}'.format(session_token), 'X-Api-Key': api_key, 'Content-Type': 'application/json', 'Accept': 'application/json'})
    r = urllib2.urlopen(files_req)
    files_resp = r.read()
    r.close()
    return json.loads(files_resp)


def app_export(app_id, app_token, session_token):
    # fetch the full app
    app_req = urllib2.Request('https://api.catalyze.io/v2/app/{}'.format(app_id), None, {'Authorization': 'Bearer {}'.format(session_token), 'X-Api-Key': dashboard_api_key, 'Content-Type': 'application/json', 'Accept': 'application/json'})
    r = urllib2.urlopen(app_req)
    app_resp = r.read()
    r.close()
    app = json.loads(app_resp)

    if not os.path.exists(app['name']):
        os.mkdir(app['name'], 0o700)
    with os.fdopen(os.open('{}/app.json'.format(app['name']), os.O_WRONLY | os.O_CREAT, 0o600), 'wb') as app_file:
        app_file.write(json.dumps(app, indent=4))

    # export users
    supervisors = []
    users = []
    for user in list_users_for_app(app_id, session_token):
        if user['usersId'] in app['permissions']['supervisor']:
            supervisors.append(user)
        else:
            users.append(user)
    with os.fdopen(os.open('{}/supervisors.json'.format(app['name']), os.O_WRONLY | os.O_CREAT, 0o600), 'wb') as supervisors_file:
        supervisors_file.write(json.dumps(supervisors, indent=4))
    with os.fdopen(os.open('{}/users.json'.format(app['name']), os.O_WRONLY | os.O_CREAT, 0o600), 'wb') as users_file:
        users_file.write(json.dumps(users, indent=4))

    # list files per user & export
    app_files_path = '{}/files'.format(app['name'])
    if not os.path.exists(app_files_path):
        os.mkdir(app_files_path, 0o700)
    for user_id in [u['usersId'] for u in supervisors] + [u['usersId'] for u in users]:
        user_files_path = '{}/files/{}'.format(app['name'], user_id)
        if not os.path.exists(user_files_path):
            os.mkdir(user_files_path, 0o700)
        files = list_files_for_user(user_id, app_token['apiKey'], app_token['session'])
        for stored_file in files:
            r = urllib2.urlopen('https://api.catalyze.io/v2/users/{}/files/{}'.format(user_id, stored_file['filesId']), None, {'Authorization': 'Bearer {}'.format(app_token['session']), 'X-Api-Key': app_token['apiKey'], 'Content-Type': 'application/json', 'Accept': 'application/json'})
            chunk_size = 16 * 1024
            base_filename = '{}/files/{}/{}.{}'.format(app['name'], user_id, stored_file['filesId'], stored_file['name'])
            filename = base_filename
            i = 1
            while os.path.exists(filename):
                filename = '_'.join([base_filename, str(i)])
                i += 1
            with os.fdopen(os.open(filename, os.O_WRONLY | os.O_CREAT, 0o600), 'wb') as f:
                while True:
                    chunk = r.read(chunk_size)
                    if not chunk:
                        break
                    f.write(chunk)

    # list classes
    classes_req = urllib2.Request('https://api.catalyze.io/v2/app/{}/classes'.format(app_id), None, {'Authorization': 'Bearer {}'.format(session_token), 'X-Api-Key': dashboard_api_key, 'Content-Type': 'application/json', 'Accept': 'application/json'})
    r = urllib2.urlopen(classes_req)
    classes_resp = r.read()
    r.close()
    class_names = json.loads(classes_resp)['results']

    # export class data
    classes = []
    for class_name in class_names:
        c = retrieve_class(class_name, app_id, session_token)
        classes.append({'name': class_name, 'schema': c})

        export_req = urllib2.Request('https://api.catalyze.io/v2/classes/{}/export'.format(class_name), None, {'Authorization': 'Bearer {}'.format(app_token['session']), 'X-Api-Key': app_token['apiKey'], 'Content-Type': 'application/json', 'Accept': 'application/json'})
        r = urllib2.urlopen(export_req)
        export_resp = r.read()
        r.close()
        with os.fdopen(os.open('{}/{}_export.json'.format(app['name'], class_name), os.O_WRONLY | os.O_CREAT, 0o600), 'wb') as f:
            f.write(json.dumps(json.loads(export_resp), indent=4))

    with os.fdopen(os.open('{}/classes.json'.format(app['name']), os.O_WRONLY | os.O_CREAT, 0o600), 'wb') as classes_file:
        classes_file.write(json.dumps(classes, indent=4))


def main():
    print('Before you run this script, ensure that your account has supervisor permissions for your application')
    print('')

    session_token = signin()
    if not session_token:
        print('Authentication failed')
        return 1
    print('Successfully authenticated')

    app_tokens, apps = retrieve_apps(session_token)

    if len(apps) == 0:
        print('You have no applications')
        return 0

    print('\nThe following applications have been discovered:')
    for k in apps:
        print('\t{} ({})'.format(apps[k]['name'], k))
    print('')

    for k in apps:
        print('Exporting data for application {}'.format(apps[k]['name']))
        app_export(apps[k]['appId'], app_tokens[k], session_token)
        print('Your data has been saved to the folder {}'.format(apps[k]['name']))
    return 0


if __name__ == '__main__':
    sys.exit(main())