aw

README.md

aw

aw!

aw is a Random Dataset Generator (for my data mining learning).

Give descriptions and random generators, generate dataset.

descriptions & generate dataset

>>> from aw.builder import Desc, Dataset
>>> from my_random_generator import * # random generator list
>>> from my_dataset import * # dataset
>>> movies_desc = [Desc(name='name', random_generator=movies_rg,
                 dataset=movies_set, unique=True), # movie's name cannot be same
                   Desc('rating', rating_rg)]
>>> ds_desc = [Desc(name='name', random_generator=name_rg,
                    dataset=name_set, unique=True),
               Desc('movies', count_rg,
                    children=movies_descs), # is a sub dataset
               Desc('age', age_rg),
               Desc('sexual', s_rg)]
>>> dataset = Dataset.fromdescs(ds_desc)

generated dataset

>>> dataset.show()
count: 3
+-----------+-----------+-------+-----------+
|   name    |   movies  |   age |   sexual  |
+-----------+-----------+-------+-----------+
|   Alex    |   [2]     |   19  |   male    |
+-----------+-----------+-------+-----------+
|   Bob     |   [7]     |   18  |   male    |
+-----------+-----------+-------+-----------+
|   Candy   |   [8]     |   25  |   female  |
+-----------+-----------+-------+-----------+

>>> len(dataset)
3

>>> datset.count
3

>>> dataset[0]
+-----------+-----------+-------+-----------+
|   name    |   movies  |   age |   sexual  |
+-----------+-----------+-------+-----------+
|   Alex    |   [15]    |   19  |   Male    |
+-----------+-----------+-------+-----------+

>>> '%s is a %d year's old %s.' % (dataset[0]['name'], dataset[0]['age'],\
dataset[0].['sexual'])
'Alex is a 18 year's old male.'

>>> dataset[0]['movies'].show()
count: 2
+-------------------+-----------+
|   name            |   rating  |
+-------------------+-----------+
|   Batman Beings   |   5.0     |
+-------------------+-----------+
|   Up in the air   |   4.0     |
+-------------------+-----------+

>>> dataset[0]['movies']
+-------------------+-----------+
|   name            |   rating  |
+-------------------+-----------+
|   Batman Beings   |   5.0     |
+-------------------+-----------+
|   Up in the air   |   4.0     |
+-------------------+-----------+

>>> dataset[0].movies['name']
'Batman Beings'

+-------------------+-----------+
|   name            |   rating  |
+-------------------+-----------+
|   Batman Beings   |   5.0     |
+-------------------+-----------+

random generator

Write your own random generator basic on your own rule.

When entering the generator, dataset(provided in each desc) and previous generated data will hand to the generator too.

def a_sample_random_generator(dataset=None, previous_data=None):
    random_index = do_stuff()
    if previous_data:
        random_index = modified_random_index_base_on_pre_data()
    if dataset:
        return dataset[previous_data]
    else:
        return random_index

def another_random_generator(dataset=None, previous_data=None):
    return do_stuff()

TODO

• csv loader (i.e. Dataset.fromcsv(csv_path))

• dataset formating

• data mining learning

builder.py

#coding: utf-8

from utils import expand, real_len, TAB_WIDTH


class Desc(object):
    '''Column description.

    :param name: the name of the column.
    :param random_generator: A random geneartor, which returns somethings randomly
                             basic on given dataset and previous generated data.
    :param dataset: dataset for generating random data.
    :param children: declares and gives the descriptions for the sub dataset.
    :param unique: declares whether the column is unique.
    '''

    def __init__(self, name, random_generator, dataset=None, children=None,
                 unique=None):
        self.name = name
        self.random_generator = random_generator
        self.dataset = dataset
        self.children = children
        self.unique = True if unique else False

    def __str__(self):
        if self.unique:
            return '<Desc(%s unique)>' % (self.name)
        else:
            return '<Desc(%s)>' % (self.name)

    def __repr__(self):
        return self.__str__()


class Data(dict):
    '''A data row in the dataset. Just like a dict object, but the key order is
    fixed rather than sort by hash.'''
    def __init__(self, header):
        self.header = header
        for key in header:
            self[key] = None

    def __str__(self):
        return self.table

    def __repr__(self):
        return self.__str__()

    def keys(self):
        #: Fix key's order rather than base on hash
        return self.header

    def items(self):
        return [(key, self[key]) for key in self.keys()]

    @property
    def table(self):
        '''Generate a mysql like result table'''
        # more pythonic approach for assigning value to multi vars?
        header, body, sep = [''], [''], ['']
        for key, value in self.items():
            if isinstance(value, Dataset):
                value = value.summarize()
            value = str(value)
            width = real_len(max(len(key), len(value)))
            header.append(expand(key, width))
            body.append(expand(value, width))
            sep.append('-' * (width + TAB_WIDTH))
        for i in (header, body, sep):
            i.append('')
        header = '|'.join(header)
        body = '|'.join(body)
        sep = '+'.join(sep)
        return '\n'.join((sep, header, sep, body, sep))


class Dataset(object):
    '''A dataset.'''
    def __init__(self, header, body):
        self.header = header
        self.body = body

    def __len__(self):
        return len(self.body)

    def __getitem__(self, idx):
        return self.body[idx]

    def __setitem__(self, idx, value):
        self.body[idx] = value

    def __delitem__(self, idx):
        del self.body[idx]

    def __str__(self):
        return self.table

    def __repr__(self):
        return self.__str__()

    @property
    def table(self):
        '''Generate a mysql like result table'''
        header, body, sep = [''], [], ['']

        #: find out the max length of word of each column
        max_len = {}
        for key in self.header:
            width = len(str(key))
            for line in self.body:
                if isinstance(line[key], Dataset):
                    value = line[key].summarize()
                else:
                    value = str(line[key])
                width = max(width, len(value))
            max_len[key] = real_len(width)
            #: build the header and sep in the same time
            header.append(expand(key, max_len[key]))
            sep.append('-' * (max_len[key] + TAB_WIDTH))
        for i in (header, sep):
            i.append('')
        header = '|'.join(header)
        sep = '+'.join(sep)

        #: build body
        for line in self.body:
            b = ['']
            for key, value in line.items():
                if isinstance(value, Dataset):
                    value = value.summarize()
                else:
                    value = str(value)
                width = max_len[key]
                b.append(expand(value, width))
            b.append('')
            body.append('|'.join(b))

        #: build table
        table = [sep, header, sep]
        for line in body:
            table.append(line)
            table.append(sep)
        return '\n'.join(table)
    
    def append(self, item):
        self.body.append(item)

    @property
    def count(self):
        return len(self)

    @staticmethod
    def fromheader(header):
        return Dataset(header, [])

    @staticmethod
    def fromdescs(descs, count):
        '''Build dataset from given descriptions'''
        header = [desc.name for desc in descs]
        dataset = Dataset.fromheader(header)

        for i in xrange(0, count):
            d = Data(header)
            for desc in descs:
                if desc.children:
                    #: build sub dataset
                    d[desc.name] = Dataset.fromdescs(
                                    desc.children, desc.random_generator())
                else:
                    data = desc.random_generator(desc.dataset, d)
                    if desc.unique:
                        while not dataset.is_unique(desc.name, data):
                            data = desc.random_generator(desc.dataset, d)
                    d[desc.name] = data
            dataset.append(d)

        return dataset

    def get_column(self, row_name):
        '''Get specific column'''
        return [i[row_name] for i in self]

    def is_unique(self, name, data):
        '''Check is the data in the row'''
        return not (data in self.get_column(name))

    def show(self):
        print 'count: %d\n%s' % (self.count, self.table)

    def summarize(self):
        '''Generate summary'''
        return '[%d]' % len(self)

utils.py

#coding: utf-8

from string import expandtabs


TAB_WIDTH = 4

    
def real_len(l, tw=TAB_WIDTH):
    '''Calculate the real length of a word with a tab
    :param l: length of the word
    :param tw: tab width'''
    return tw - l % tw + l


def expand(s, width, tw=TAB_WIDTH):
    '''Expand the tab'''
    #: use aleast one tab
    if width == len(s):
        tab = '\t'
    else:
        tab = '\t' * (1 + (width - real_len(len(s), tw)) / tw)
    return expandtabs('\t%s%s' % (s, tab), tw)