Expanded Serializer

jsonForm.js

$.fn.serializeObject = function()
{
   var o = {};
   var a = this.serializeArray();
   $.each(a, function() {
   	   var oldName = this.name;
   	   this.name = this.name.replace(/\[\]/g, '');
   	   
   	   if (this.name.indexOf('[') >= 0) {
   	   	var innerName = this.name.match(/.*\[(.*)\]/)[1];
   	   	var outterName = this.name.split("[")[0];
   	   	if (o[outterName]) {
   	   		if (o[outterName][innerName]) {
   	   			if (!o[outterName][innerName].push) {
   	   			    o[outterName][innerName] = [o[outterName][innerName]];
   	   			}
   	   			o[outterName][innerName].push(this.value || '');
   	   		}
   	   		else {
   	   			console.log(this.name);
	   	   		if ((oldName).indexOf('[]') >= 0) {
	   	   			o[outterName][innerName] = [this.value] || '';
	   	   		}
	   	   		else {
	   	   			o[outterName][innerName] = this.value || '';
	   	   		}
   	   		}
   	   	}
   	   	else {
   	   		o[outterName] = {};
   	   		if (o[outterName][innerName]) {
   	   			if (!o[outterName][innerName].push) {
   	   			    o[outterName][innerName] = [o[outterName][innerName]];
   	   			}
   	   			o[outterName][innerName].push(this.value || '');
   	   		}
   	   		else {
   	   			console.log(this.name);
   	   			if (oldName.indexOf('[]') >= 0) {
   	   				o[outterName][innerName] = [this.value] || '';
   	   			}
   	   			else {
   	   				o[outterName][innerName] = this.value || '';
   	   			}
   	   		}
   	   	}
   	   }
   	   else {
	       if (o[this.name]) {
	           if (!o[this.name].push) {
	               o[this.name] = [o[this.name]];
	           }
	           o[this.name].push(this.value || '');
	       } else {
	           o[this.name] = this.value || '';
	       }
       }
   });
   return o;
};

Some thoughts:

• The regex /.*\[(.*)\]/, upon some quick testing, looks like it will return back way more from the string than you were expecting. However, I'm not sure what specific use case for this code is, so that may not be a problem?
• outerName, not outterName.
• o[outterName][innerName] = [this.value] || ''; will always return an Array. However, that Array may have a falsy value as it's only element - I'm guessing that this is not what you want.
• The flow of this is hard to follow. It looks like it gets to the bottom of the loop first, then later on works it's way to the top. When possible, I try to make logic flow from top to bottom.

Do you have an example page where I could see this is action? It's difficult to really get a feel for this without any context.

for form:

<form id="create_puzzle" method="post" action="http://localhost:1337/playlist/create">

        <table>
            <tr>
                <td><label for="photos">Photos (up to 9)</label></td>
                <td>
                    <div id="uploader">

                    </div>

                </td>
            </tr>
            <tr>
                <td><label for="title">Title</label></td>
                <td><input type="text" name="playlist[title]" value="" /></td>
            </tr>

            <tr>
                <td><label for="tags">Tags</label></td>
                <td><input type="text" name="playlist[tags]" /></td>
            </tr>

            <tr>
                <td><label for="share">Share</label></td>
                <td><input type="text" name="invite[email]" /></td>
            </tr>

            <tr>
                <td colspan="2"><input type="submit" value="create puzzle!" /></td>
            </tr>


    </table>
    </form>

i want to serialize this to send as a json body to test the api.

var jsForm = $('#create_puzzle').serializeObject();

so that i get something like

{
  invite: { 
          email: "asdasd"
  },
  playlist: { 
           title: "asdasd", 
           tags: "xvcdsads"
  }
}

I'd say that if it works, then you're good to go. Admittedly it's not the prettiest code I've seen, but that's not really an important detail in the long run. I'd suggest writing a few simple unit tests, since the logic is fairly complex.