ReOBJ

reobj.py

__author__ = '[email protected]'
########################################################################################################################
# Imports
########################################################################################################################
# Make sure when we're blindly imported, only our defined classes get imported.
__all__ = ['ReOBJ', 'ReOBJConnPool']

# We should do logging, right?
# import logging
# Remember to replace print calls with logging calls.

# Flask imports
# from flask import g

# Rethink imports
import rethinkdb as r

# Pull in all the classes and methods from rethinkdb.* except for "object"
from rethinkdb.net import connect  # , Connection, Cursor
# from rethinkdb.query import now, js, http, json, args, error, random, do, row, table, info, type_of, \
#    db, db_create, db_drop, db_list, table_create, table_drop, table_list, branch, literal, \
#    asc, desc, eq, ne, le, ge, lt, gt, any, all, add, sub, mul, div, mod, and_, or_, not_ \
#    iso8601, make_timezone, epoch_time, time, monday, tuesday, wednesday, thursday, friday, saturday, sunday, \
#    january, february, march, april, may, june, july, august, september, october, november, december \

from rethinkdb.errors import RqlRuntimeError  # , RqlError, RqlClientError, RqlCompileError, RqlDriverError
# from rethinkdb.ast import expr, RqlQuery
# import rethinkdb.docs

# Rethink configuration -- Commented out, see below in the connection pool.
# from app.config import rdb
# This Class currently uses the following database configuration:
rdb = {
    'host': 'localhost',
    'port': 28015,
    'auth_key': ''  # ,
    # 'auth_key': 'superpotatopeeler2alphaex' #,
    # 'userdb': 'kaizen_auth:users',
    # 'ticketsdb': 'kaizen:tickets',
    # 'pagedb': 'kaizen:pages'
}
# These are commented out to avoid breaking **rdb in the connection pool.
# Fixed, but still here for reference purposes.

# Collections imports
from collections import Iterable, Mapping

# Connection Pool Imports
try:  # The python3 module name first
    import queue
except ImportError:  # Fall back to renaming the python2 class
    import Queue as queue
from contextlib import contextmanager

########################################################################################################################
# Utility Classes
########################################################################################################################


class ReOBJConnPool(object):
    """ 
    ReOBJConnPool: Manage a simple pool of connections to RethinkDB
    """
    # Private attributes
    _idle_connections = queue.Queue()
    _active_connections = 0
    _maximum_connections = 5

    @contextmanager
    def get(self):
        if self._idle_connections.empty() and self._active_connections < self._maximum_connections:
            # connection = connect(**rdb)  # Broke with flask app.config due to my extra dict members.
            connection = connect(host=rdb['host'], port=rdb['port'], auth_key=rdb['auth_key'])  # Do it right.
        else:
            connection = self._idle_connections.get()
        self._active_connections += 1
        yield connection
        self._idle_connections.put(connection)
        self._active_connections -= 1

re_pool = ReOBJConnPool()


class ReDict(dict):
    """ 
    ReDict: Manage updates of dict members to RethinkDB documents.
    """
    # We'll need to fill this in somehow so we can remember where to save to.
    db_table = None
    id = None

    def __init__(self, uuid, **kwargs):
        """
        This is a ReDict representing a subdocument retrieved by it's UUID from a RethinkDB table.
        The attributes set on this object reflect the state of the subdocument when it was retrieved.
        Any attributes set on this object will be saved to the subdocument immediately.
        @param uuid: A string providing the containing document's UUID
        """
        self.id = uuid
        super(ReDict, self).__init__(self, **kwargs)  # Use parent class's init to silence warnings.

    def _save(self, item, value):
        """
        Private method. Saves contents written to this ReDict object back to the database.
        """
        _db = self.db_table.split(':')  # Split the database name and table name.
        try:
            with re_pool.get() as conn:
                r.db(_db[0]).table(_db[1]).get(self.id).update({item: value}).run(conn)
                print("ReDict:_save: Attribute set: {} to value: {}".format(item, value))
        except RqlRuntimeError:
            print("ReDict:__init_: Critical Failure: Saving Throw Failed! while looking up UUID: {}".format(self.id))

    def _consume(self, data):
        """
        Recursively converts a dict within a ReOBJ object from a dict to a ReDict
        """
        if isinstance(data, Mapping):
            return ReDict(map(self._consume, data.iteritems()))
        elif isinstance(data, Iterable):
            return type(data)(map(self._consume, data))
        else:
            return data

    def __setitem__(self, item, value):
        """
        Sets an attribute on a dict within a ReOBJ object.
        """
        print("ReDict:__setitem__: Trying to set Attribute: {} to value: {}".format(item, value))
        self._save(item, value)  # save the item and it's value(s) to the database
        super(ReDict, self).__setitem__(item, value)  # Use parent class's setitem to update the instance dictionary.
        print("ReDict:__setitem__: Attribute in dict set: {} to value: {}".format(item, value))


########################################################################################################################
# Helper Functions
########################################################################################################################


# None currently, all moved into respective classes.
# RIP Standalone DB creation functions, you're in a better place now as @classmethods

########################################################################################################################
# ReOBJ Class
########################################################################################################################


class ReOBJ(object):
    """
    ReOBJ: a mapping layer translating RethinkDB nested JSON arrays into Python objects.
    
    Given a database, table, and document UUID, ReOBJ will instantiate an object with attributes matching the document's
    This is dependant on your tables relying on RethinkDB's primary key default of 'id' being a UUID.
    Updating the ReOBJ's attributes will update meta['modified_at'] and perform a RethinkDB Upsert on the document.
    See the example _subclasses in reobj.py to discover how to look up objects by a different field or index.
    
    Since RethinkDB 1.15 queries are  'lazy' about loading, The 'meta' field is given special meaning as an array meant
     to be Plucked during list-queries describing the document's content, status, timestamps, locking information, ACLs,
     relations, ownership, etc, without actually loading the (possibly large) contents of the actual document itself.
    It's probably not a good idea to use the 'meta' array to store large amounts of data such as icons or images.
    On a side note, it's perfectly acceptable to have a document containing only the meta field.
    """

    # Once connected to the cluster, this describes the database name and table name to utilize.
    # Note that RethinkDB _only_ allows underscores and alphanumerics in database and table names.
    # The format is "database_name:table_name".
    db_table = "test:test"  # We'll use the RethinkDB defaults for now so you can have fun in REPLs.

    # This list holds any fields that should always exist for this class.
    required_fields = []  # We default to requiring no fields.

    # This attribute should always exist. It will be set during __init__ during instantiation.
    id = None  # We will use the value None for now.

    @classmethod
    def _get_conn(cls):
        """
        Private method. Get a connection to the database somehow. Go ahead and modify this method.
        @return: An open RethinkDB connection
        """
        # return g.rdb_conn  # Get the connection from flask's global g object. (kaizen-deprecated)
        # TODO: When we move the connection pool out of the main class file, we'll have to revisit this.
        return re_pool.get()  # For now, just use the global to grab one from the pool.

    @classmethod
    def get_db_table(cls):
        """
        Get the subclass's current db_table.
        @return: This class's database and table configuration.
        """
        return cls.db_table  # For now, just use the global to grab one from the pool.

    @classmethod
    def _db_setup(cls):
        """
        Private method. Create a new database matching the specified subclass's configuration
        """
        _db = cls.db_table.split(':')  # Split the database name and table name
        try:
            with cls._get_conn() as conn:
                r.db_create(_db[0]).run(conn)
            print("ReOBJ:db_setup: {} Database initialized.".format(_db[0]))
        except RqlRuntimeError:
            print("ReOBJ:db_setup: {} Database exists.".format(_db[0]))

    @classmethod
    def _table_setup(cls):
        """
        Private method. Create a new table matching the specified subclass's configuration
        """
        _db = cls.db_table.split(':')  # Split the database name and table name
        try:
            with cls._get_conn() as conn:
                r.db(_db[0]).table_create(_db[1]).run(conn)
                # r.db(_db[0]).table(_db[1]).index_create('updated_at').run(conn)
            print("ReOBJ:table_setup: {} Table initialized in {} Database.".format(_db[1], _db[0]))
        except RqlRuntimeError:
            print("ReOBJ:table_setup: {} Table exists in {} Database.".format(_db[1], _db[0]))

    @classmethod
    def create(cls, meta, **properties):
        """
        Create a new ReOBJ object, insert it into the database, then retrieve it and return it, ready for use.
        If necessary, you may retrieve it's UUID from the object directly from it's member attribute 'id'.
        @param meta: A dict containing the meta data to store, can be None.
        @param **properties: The properties to store, if any (meta data only perhaps?)
        @return: A ReOBJ instantiated from the supplied data or None.
        """
        _db = cls.db_table.split(':')  # Split the database name and table name
        if meta is None:  # Check to see if we were passed any meta data.
            meta = {}  # We were not passed any meta data, so create an empty array to hold meta data.
        meta.update({'created_at': r.now(), 'updated_at': r.now()})  # Make some timestamps.
        new_document = {'meta': meta}  # Create an array containing the meta data array.
        new_document.update(properties)  # Append the properties to the array.
        try:  # To insert the new_document into the database table.
            with cls._get_conn() as conn:
                inserted = r.db(_db[0]).table(_db[1]).insert(new_document).run(conn)
        except RqlRuntimeError:  # We failed to insert the document for some reason.
            print("ReOBJ:create: Failed to insert document with meta data: {}".format(meta))
            return None  # Just tell our caller nothing was inserted, I suppose.
        new_uuid = inserted['generated_keys'][0]  # The insert was successful
        print("ReOBJ:create: {} was created with meta data: {}".format(new_uuid, meta))
        return cls(new_uuid)  # Lookup and return a new object of our subclass.

    def populate_required(self, document):
        """
        Populates the required fields in a new ReOBJ object.
        This will be called from __init__ during object creation or instantiation.
        This function uses an iterative approach to traverse the required_fields defined for the class.
        """
        for field in self.required_fields:  # Iterate through the class-provided list of required_fields.
            path = field.split('.')  # Split the contents of 'field' by the . into the path array.
            final_section = path[-1]  # Collect the last member of the array into the final_section.
            subdocument = document  # Copy the query result into another variable to work against.
            for section in path:  # Iterate over the sections of the path
                if section != final_section:  # If this isn't 'mobile' of 'meta.phone.mobile'
                    subdocument = subdocument.setdefault(section, {})  # Descend further...
                else:  # We've found the last element, use it like a string.
                    subdocument.setdefault(section, 'Empty ' + section.capitalize() + ' Value')  # No value? make one.

    def _retrieve(self, uuid):
        """
        Private method. Retrieve a ReOBJ object by looking up it's UUID.
        @return: A ReOBJ instantiated from the supplied UUID or None.
        """
        _db = self.db_table.split(':')  # Split the database name and table name.
        try:
            with self._get_conn() as conn:
                return r.db(_db[0]).table(_db[1]).get(uuid).run(conn)
        except RqlRuntimeError:
            print("ReOBJ:__init_: Critical Failure: Saving Throw Failed! while looking up UUID: {}".format(uuid))

    def _apply(self, results):
        """
        Private method. Apply retrieved data from a query result to this ReOBJ object.
        """
        for field, value in results.iteritems():  # Iterate over the fields we got from rethink

            super(ReOBJ, self).__setattr__(field, value)  # Set our object attributes to what we received.

    def _save(self, item, value):
        """
        Private method. Saves a top level attribute on this ReOBJ object back to the database.
        """
        _db = self.db_table.split(':')  # Split the database name and table name.
        try:
            with self._get_conn() as conn:
                # Update the database with the data.
                r.db(_db[0]).table(_db[1]).get(self.id).update({item: value}).run(conn)
                print("ReOBJ:_save: Attribute set: {} to value: {}".format(item, value))
                # Update the database with a new timestamp, since we've changed data.
                r.db(_db[0]).table(_db[1]).get(self.id).update({'meta': {'updated_at': r.now()}}).run(conn)
                print("ReOBJ:_save: Timestamp updated.")
        except RqlRuntimeError:
            print("ReOBJ:__init_: Critical Failure: Saving Throw Failed! while looking up UUID: {}".format(self.id))

    def __init__(self, uuid):
        """
        This is a ReOBJ representing a document retrieved by it's UUID from a RethinkDB table.
        The attributes set on this object reflect the state of the document when it was retrieved.
        Any attributes set on this object will be saved to the document immediately.
        """
        results = self._retrieve(uuid)
        self.populate_required(results)
        self._apply(results)

    def __setattr__(self, item, value):
        """
        Sets a top level attribute on a ReOBJ object. Will not save updates to attributes containing dicts.
        """
        self._save(item, value)  # save the item and it's value(s) to the database
        super(ReOBJ, self).__setattr__(item, value)  # Use our parent class's setattr to update the instance dictionary.
        print("ReOBJ:__setattr__: Attribute set: {} to value: {}".format(item, value))


# Some example classes to demonstrate proper syntax.
# Because of their _ prefix, they'll be excluded in an import * from ReOBJ, used while debugging this as a single file.
class _User(ReOBJ):
    db_table = 'auth:users'
    required_fields = ['username', 'email']


class _Ticket(ReOBJ):
    db_table = 'support:tickets'
    required_fields = ['meta.email', 'meta.phone', 'meta.name', 'message']


class _Page(ReOBJ):
    db_table = 'web_site:pages'
    required_fields = ['meta.template', 'content']

You can also control what is imported with from ReOBJ import * by setting the __all__ attribute:

__all__ = ['ReObj', 'something_else']

Though using import * isn't recommended in general