csv printer extension for python

cmongo.c

#ifdef PYTHON
#include <Python.h>
#endif
#include "cmongo.h"

typedef int (*debug_func)(const char *, ...);

#define BLOCK 20000000
struct BigStr {
	size_t buffer_size;
	char* buffer;
	char* buftemp;
	int curr_size;
};

int mon_connect(mongo *conn, const char * host, const int port);
char* all_query(const char* host, const int port, const char* collection);
char* filter_query(const char* host, const int port, const char* collection,
		bson* query);
char* filter_query_json(const char* host, const int port,
		const char* collection, const char* json);
void str_headings(const char *data, struct BigStr *bs);
void str_row(const char *data, struct BigStr *bs);
void debug_head(const char* str);
void adjust_size(struct BigStr *bs);

#ifdef PYTHON
static PyObject * py_all_query(PyObject *self, PyObject *args);
static PyObject * py_filter_query(PyObject *self, PyObject *args);

static PyMethodDef module_functions[] = {
	{	"all_query", py_all_query, METH_VARARGS, "CSV string of entire collection"},
	{	"filter_query", py_filter_query, METH_VARARGS, "CSV string of filtered collection"},
	{	NULL, NULL, 0, NULL}
};

static PyObject *JsonParseError;
PyMODINIT_FUNC initcmongo(void)
{
    PyObject *m;
	m = Py_InitModule("cmongo", module_functions);

    if (m == NULL)
        return;

    JsonParseError = PyErr_NewException("py_filter_query.error", NULL, NULL);
    Py_INCREF(JsonParseError);
    PyModule_AddObject(m, "error", JsonParseError);
}

static PyObject * py_all_query(PyObject *self, PyObject *args)
{
	const char *host;
	const int port;
	const char *collection;

	if (!PyArg_ParseTuple(args, "sis:all_query", &host, &port, &collection))
	return NULL;
	char* result = all_query(host, port, collection);
	return Py_BuildValue("s", result);
}

static PyObject * py_filter_query(PyObject *self, PyObject *args)
{
	const char *host;
	const int port;
	const char *collection;
	const char *json;

	if (!PyArg_ParseTuple(args, "siss:filter_query", &host, &port, &collection, &json))
		return NULL;

	bson query[1];
	if (!json_to_bson(json, query)){
		PyErr_SetString(JsonParseError, "Failed to Parse Json");
		return NULL;
	}

	char* result = filter_query(host, port, collection, query);
	bson_destroy(query);
	return Py_BuildValue("s", result);
}
#endif

#ifndef PYTHON
int main(void) {
	debug("JSON TEST:\n------------\n------------\n");
	json2bson_test();
	debug("\n------------\n------------\n");
	char collection[] = "markets.markets_trace"; //
	debug("Testing with '%s':\n", collection);
	char host[] = "127.0.0.1";
	int port = 27017;
	debug("All Results:\n");
	char* result1 = all_query(host, port, collection);
	debug_head(result1);
	if (result1 == '\0') {
		free(result1);
	}
	bson query[1];
	bson_init(query);
	bson_append_int(query, "exchange_id", 1);
	bson_finish(query);
	debug("BSON Filtered Results:\n");
	char* result2 = filter_query(host, port, collection, query);
	debug_head(result2);
	if (result2 == '\0') {
		free(result2);
	}
	debug("JSON Filtered Results:\n");
	char* result3 = filter_query_json(host, port, collection,
			"{ \"exchange_id\": 1 }");
	debug_head(result3);
	if (result3 == '\0') {
		free(result3);
	}
	return 1;
}
#endif

void debug_head(const char* str) {
	if (str == NULL) {
		debug("HEAD DEBUG: string null\n");
		return;
	}
	if (str[0] == '\0') {
		debug("HEAD DEBUG: blank string\n");
		return;
	}
	int l = 10000;
	if (l > strlen(str))
		l = strlen(str) + 1;
	char substr[l];
	memcpy(substr, str, l);
	debug("HEAD DEBUG: (total length: %d)\n", (int) strlen(str));
	debug("'%s'\n", substr);
}

char* all_query(const char* host, const int port, const char* collection) {
	bson *query = NULL;
	char* result = filter_query(host, port, collection, query);
	return result;
}

char* filter_query_json(const char* host, const int port,
		const char* collection, const char* json) {
	bson query[1];
	json_to_bson(json, query);
	char* result = filter_query(host, port, collection, query);
	bson_destroy(query);
	return result;
}

char* filter_query(const char* host, const int port, const char* collection,
		bson* query) {
	mongo conn;
	struct BigStr bs;
	bs.buffer_size = BLOCK;
	bs.buffer = (char*) malloc(bs.buffer_size);
	bs.buftemp = bs.buffer;
	bs.curr_size = 0;
	int e = mon_connect(&conn, host, port);
	if (e != 1) {
		return "ERROR";
	}
	mongo_cursor cursor[1];
	mongo_cursor_init(cursor, &conn, collection);
	if (query != NULL) {
		mongo_cursor_set_query(cursor, query);
	}
	int i = 0;
	while (mongo_cursor_next(cursor) == MONGO_OK) {
		const bson *b = &cursor->current;
		if (i == 0) {
			str_headings(b->data, &bs);
			bs.buftemp = bs.buffer + bs.curr_size;
			bs.curr_size += sprintf(bs.buftemp, "\n");
		}
		str_row(b->data, &bs);
		bs.buftemp = bs.buffer + bs.curr_size;
		bs.curr_size += sprintf(bs.buftemp, "\n");
		adjust_size(&bs);
		i++;
	}
	debug("buffer_size: %d\n", (int)bs.buffer_size);
	bs.buffer = realloc(bs.buffer, strlen(bs.buffer + 1));
	mongo_cursor_destroy(cursor);
	mongo_destroy(&conn);
	return bs.buffer;
}

void adjust_size(struct BigStr *bs) {
	if ((bs->buffer_size - bs->curr_size) < BLOCK) {
		bs->buffer_size += BLOCK;
		bs->buffer = realloc(bs->buffer, bs->buffer_size);
	}
}

int mon_connect(mongo *conn, const char * host, const int port) {
	if (mongo_client(conn, host, port) != MONGO_OK) {
		switch (conn->err) {
		case MONGO_CONN_SUCCESS:
			break;
		case MONGO_CONN_NO_SOCKET:
			debug("FAIL: Could not create a socket.\n");
			return 0;
		case MONGO_CONN_FAIL:
			debug(
					"FAIL: Could not connect to mongo. Make sure mongo is listening at: '%s' port %i\n",
					host, port);
			return 0;
		default:
			debug("MongoDB connection error number %d.\n", conn->err);
			return 0;
		}
	}
	debug("CONNECTED TO %s:%d\n", host, port);
	return 1;
}

void str_headings(const char *data, struct BigStr *bs) {
	const char *key;
	bson_iterator i;
	bson_iterator_from_buffer(&i, data);

	while (bson_iterator_next(&i)) {
		bson_type t = bson_iterator_type(&i);
		if (t == 0)
			break;
		key = bson_iterator_key(&i);
		if (key[0] == '\0')
			continue;
		bs->buftemp = bs->buffer + bs->curr_size;
		bs->curr_size += sprintf(bs->buftemp, "%s, ", key);
	}
}

void str_row(const char *data, struct BigStr *bs) {
	const char *key;
	bson_timestamp_t ts;
	char oidhex[25];
	bson_iterator i;
	bson_iterator_from_buffer(&i, data);
	while (bson_iterator_next(&i)) {
		bson_type t = bson_iterator_type(&i);
		if (t == 0)
			break;
		key = bson_iterator_key(&i);
		if (key[0] == '\0')
			continue;
		bs->buftemp = bs->buffer + bs->curr_size;
		switch (t) {
		case BSON_DOUBLE:
			bs->curr_size += sprintf(bs->buftemp, "%f",
					bson_iterator_double(&i));
			break;
		case BSON_STRING:
			bs->curr_size += sprintf(bs->buftemp, "%s",
					bson_iterator_string(&i));
			break;
		case BSON_SYMBOL:
			bs->curr_size += sprintf(bs->buftemp, "%s",
					bson_iterator_string(&i));
			break;
		case BSON_OID:
			bson_oid_to_string(bson_iterator_oid(&i), oidhex);
			bs->curr_size += sprintf(bs->buftemp, "%s", oidhex);
			break;
		case BSON_BOOL:
			bs->curr_size += sprintf(bs->buftemp, "%s",
					bson_iterator_bool(&i) ? "true" : "false");
			break;
		case BSON_DATE:
			bs->curr_size += sprintf(bs->buftemp, "%ld",
					(long int) bson_iterator_date(&i));
			break;
		case BSON_REGEX:
			bs->curr_size += sprintf(bs->buftemp, "%s",
					bson_iterator_regex(&i));
			break;
		case BSON_CODE:
			bs->curr_size += sprintf(bs->buftemp, "%s", bson_iterator_code(&i));
			break;
		case BSON_BINDATA:
			bs->curr_size += sprintf(bs->buftemp, "BSON_BINDATA");
			break;
		case BSON_UNDEFINED:
			bs->curr_size += sprintf(bs->buftemp, "BSON_UNDEFINED");
			break;
		case BSON_NULL:
			bs->curr_size += sprintf(bs->buftemp, "BSON_NULL");
			break;
		case BSON_INT:
			bs->curr_size += sprintf(bs->buftemp, "%d", bson_iterator_int(&i));
			break;
		case BSON_LONG:
			bs->curr_size += sprintf(bs->buftemp, "%ld",
					(long) bson_iterator_long(&i));
			break;
		case BSON_TIMESTAMP:
			ts = bson_iterator_timestamp(&i);
			bs->curr_size += sprintf(bs->buftemp, "i: %d, t: %d", ts.i, ts.t);
			break;
//         case BSON_CODEWSCOPE:
//             bs->curr_size += sprintf(bs->buftemp, "BSON_CODE_W_SCOPE: %s", bson_iterator_code( &i ) );
//             bson_iterator_code_scope_init( &i, &scope, 0 );
//             bs->curr_size += sprintf(bs->buftemp, "\n\t SCOPE: " );
//             bson_print( &scope );
//             bson_destroy( &scope );
//             break;
//         case BSON_OBJECT:
//         case BSON_ARRAY:
//             bs->curr_size += sprintf(bs->buftemp, "\n" );
//             bson_print_raw( bson_iterator_value( &i ) , depth + 1 );
//             break;
		default:
			bs->curr_size += sprintf(bs->buftemp, "?");
			debug("missing type: %d\n", t);
		}
		bs->buftemp = bs->buffer + bs->curr_size;
		bs->curr_size += sprintf(bs->buftemp, ", ");
		adjust_size(bs);
	}
}

cmongo.h

#include "mongo.h"
#include <string.h>
#include <stdio.h>

#ifndef PYTHON
int main();
int json2bson_test();
#endif
int json_to_bson(const char *js, bson *b);

#define DEBUG 1
#if DEBUG
	#define debug(...) printf(__VA_ARGS__);
#else
	#define debug(...)
#endif

gcc compile

gcc --std=c99 cmongo.h json2bson.c  cmongo.c -lmongoc -ljson -o mong_test

json2bson.c

#include "cmongo.h"
#include "json/json.h"

void json_to_bson_append_element( bson *b , const char *k , struct json_object *v );

void json_to_bson_append_array( bson *b , struct json_object *a ) {
    int i;
    char buf[10];
    for ( i=0; i<json_object_array_length( a ); i++ ) {
        debug( buf , "%d" , i );
        json_to_bson_append_element( b , buf , json_object_array_get_idx( a , i ) );
    }
}

void json_to_bson_append( bson *b , struct json_object *o ) {
    json_object_object_foreach( o, k, v ) {
        json_to_bson_append_element( b , k , v );
    }
}

void json_to_bson_append_element( bson *b , const char *k , struct json_object *v ) {
    if ( ! v ) {
        bson_append_null( b , k );
        return;
    }

    switch ( json_object_get_type( v ) ) {
    case json_type_int:
        bson_append_int( b , k , json_object_get_int( v ) );
        break;
    case json_type_boolean:
        bson_append_bool( b , k , json_object_get_boolean( v ) );
        break;
    case json_type_double:
        bson_append_double( b , k , json_object_get_double( v ) );
        break;
    case json_type_string:
        bson_append_string( b , k , json_object_get_string( v ) );
        break;
    case json_type_object:
        bson_append_start_object( b , k );
        json_to_bson_append( b , v );
        bson_append_finish_object( b );
        break;
    case json_type_array:
        bson_append_start_array( b , k );
        json_to_bson_append_array( b , v );
        bson_append_finish_object( b );
        break;
    default:
        debug("can't handle type for : %s\n" , json_object_to_json_string( v ) );
    }
}


int json_to_bson_insert(const char *js, bson *b ) {
    struct json_object *o = json_tokener_parse( js );

    if ( is_error( o ) ) {
        debug( "\t ERROR PARSING\n" );
        return 0;
    }
    if ( ! json_object_is_type( o , json_type_object ) ) {
        debug("json_to_bson needs a JSON object, not type\n" );
        return 0;
    }
    json_to_bson_append( b , o );
    return 1;
}

int json_to_bson(const char *js, bson *b ) {
    debug("----\nJSON: %s\n" , js );

    bson_init( b );
    int success = json_to_bson_insert( js, b);
    if(!success){
    	debug("json_to_bson_insert error\n");
    }
    bson_finish( b );
    if (DEBUG && success){
    	debug("BSON: \n");
    	bson_print( b);
    }
    return success;
}

void json_to_bson_test(char *js){
	bson b[1];
	json_to_bson(js, b);
    bson_destroy( b );
}

int json2bson_test(void) {
    json_to_bson_test("1");
    json_to_bson_test("{'array' : [1, 2, 3] }");
    json_to_bson_test("{ 'x' : true }");
    json_to_bson_test("{ 'x' : null }");
    json_to_bson_test("{ 'x' : 5.2 }");
    json_to_bson_test("{ 'x' : 'eliot' }");
    json_to_bson_test("{ 'x' : 5.2 , 'y' : 'truth' , 'z' : 1.1 }");
    json_to_bson_test("{ 'x' : 4 }");
    json_to_bson_test("{ 'x' : 5.2 , 'y' : 'truth' , 'z' : 1 }");
    json_to_bson_test("{ 'x' : 'eliot' , 'y' : true , 'z' : 1 }");
    json_to_bson_test("{ 'a' : { 'b' : 1.1 } }");
    json_to_bson_test("{ 'x' : 5.2 , 'y' : { 'a' : 'eliot' , 'b' : true } , 'z' : null }");
    json_to_bson_test("{ 'x' : 5.2 , 'y' : [ 'a' , 'eliot' , 'b' , true ] , 'z' : null }");
    return 1;
}

//int main() {
//	json2bson_test();
//}

setup.py

from distutils.core import setup, Extension
VERSION = '0.1'

cmongo = Extension('cmongo', 
                    define_macros = [('PYTHON', '1')],
                    include_dirs = ['/usr/local/include'],
                    libraries = ['mongoc', 'json'],
                    library_dirs = ['/usr/local/lib'],
                    sources = ['json2bson.c', 'cmongo.c'],
                    extra_compile_args=['--std=c99'])

setup (name = 'cmongo',
       version = VERSION,
       description = 'performs mongodb queries in c and returns csv results',
       author = 'Samuel Colvin',
       author_email = '[email protected]',
       headers=['cmongo.h'],
       long_description = '''
Package inspired by monary but complete rewritten.
''',
       ext_modules = [cmongo])