Javascript Multi part issues

.gitignore

node_modules
*.log

readme.md

Current State

We have 3 wrappers that are prodominately used and depended on for multipart handlers.

• Formidable
• Multiparty
• busboy

We have 2 wrappers that are prodominately used and depended on for http requests

• Request
• SuperAgent

However, the basic issue for me is here

When providing something like a terminal output, form-data cannot handle it.

However!

By simply providing the header "Transfer-Encoding: chunked" We have a working module

To run

• npm install
• npm start
• type in the terminal one of [superagent, request, superagent-bad, request-bad]

Bugs Found

• Busboy hangs when there is an issue
• Multiparty seems to throw an error regardless if everything is listened to
• Super agent and Request do not provide Transfer-Encoding:chunked when necessary

client-req-bad.js

var sa = require("request");
var cp = require("child_process");
var async = require("async");

console.log("child");

async.filter(["generic","busboy","formidable","multiparty"],function(name,next){
  var diff = cp.spawn("git",["diff","HEAD","HEAD^"],{
    env:process.env,
    uid:process.getuid(),
    gid:process.getgid()
  });

  //diff.stdout.on("data",toStr);
  //diff.stderr.on("data",toStr);
  diff.on("error",function(e){
    throw e;
  });
  diff.stdout.on("finish",function(){
    console.log("finished pushing");
  });

  var req = sa.post("http://localhost:8000/"+name)
  .on("error",function(e){
    console.error(e);
    next(false);
  })
  .on("response",function(){
    next(true);
  });

  var form = req.form();
  form.append("key1","value1");
  form.append("key2","value2");
  form.append("key3","value3");
  form.append("diff",diff.stdout,{filename:"diff.txt"});
  form.append("key4","value4");
},function(res){
  console.log("success: ",res);
});


function toStr(data){
  console.log(data.toString("utf8"));
}

client-req.js

var sa = require("request");
var cp = require("child_process");
var async = require("async");

console.log("child");

async.filter(["generic","busboy","formidable","multiparty"],function(name,next){

  var req = sa.post("http://localhost:8000/"+name,{headers: {
    'transfer-encoding': 'chunked'
  },timeout:20*1000})
  .on("error",function(e){
    console.error(e);
    next(false);
  })
  .on("response",function(){
    next(true);
  });

  var form = req.form();
  form.append("key1","value1");
  form.append("key2","value2");
  form.append("key3","value3");
  form.append("key4","value4");

  [1,2,3].forEach(function(item){
    var diff = cp.spawn("git",["diff", "HEAD", "HEAD^"],{
      env:process.env,
      uid:process.getuid(),
      gid:process.getgid()
    });
    diff.on("error",function(e){
      console.error(e,item);
      next(false);
    });
    diff.stdout.on("finish",function(){
      console.log("finished pushing ", item);
    });
    form.append("diff",diff.stdout,{filename:"diff"+item+".txt"});
  });
  [4,5,6].forEach(function(item){
    var diff = cp.spawn("git",["diff", "HEAD", "HEAD^"],{
      env:process.env,
      uid:process.getuid(),
      gid:process.getgid()
    });
    diff.on("error",function(e){
      console.error(e,item);
      next(false);
    });
    diff.stdout.on("finish",function(){
      console.log("finished pushing ", item);
    });
    form.append("diff"+item,diff.stdout,{filename:"diff.txt"});
  });
  [7,8,9].forEach(function(item){
    var diff = cp.spawn("git",["diff", "HEAD", "HEAD^"],{
      env:process.env,
      uid:process.getuid(),
      gid:process.getgid()
    });
    diff.on("error",function(e){
      console.error(e,item);
      next(false);
    });
    diff.stdout.on("finish",function(){
      console.log("finished pushing ", item);
    });
    form.append("diff"+item,diff.stdout,{filename:"diff"+item+".txt"});
  });
},function(res){
  console.log("success: ",res);
});


function toStr(data){
  console.log(data.toString("utf8"));
}

client-sa-bad.js

var sa = require("superagent");
var cp = require("child_process");
var async = require("async");

async.filter(["generic","busboy","formidable","multiparty"],function(name,next){
  var diff = cp.spawn("git",["diff", "HEAD", "HEAD^"],{
    env:process.env,
    uid:process.getuid(),
    gid:process.getgid()
  });

  //diff.stdout.on("data",toStr);
  //diff.stderr.on("data",toStr);
  diff.on("error",function(e){
    console.error(e);
    next(false);
  });
  diff.stdout.on("finish",function(){
    console.log("finished pushing");
  });

  sa.post("http://localhost:8000/"+name)
    .field("key1","value1")
    .field("key2","value2")
    .field("key3","value3")
    .attach("diff",diff.stdout,"diff.txt")
    .field("key4","value4")
    .end(function(e,res){
      if(e){
        console.error(e);
        return next(false);
      }
      next(true);
    });
},function(res){
  console.log("success: ",res);
});


function toStr(data){
  console.log(data.toString("utf8"));
}

client-sa.js

var sa = require("superagent");
var cp = require("child_process");
var async = require("async");

async.filter(["generic","busboy","formidable","multiparty"],function(name,next){
  var req = sa.post("http://localhost:8000/"+name)
    .set('Transfer-Encoding', 'chunked')
    .field("key1","value1")
    .field("key2","value2")
    .field("key3","value3")
    .field("key4","value4");
  [1,2,3].forEach(function(item){
    var diff = cp.spawn("git",["diff", "HEAD", "HEAD^"],{
      env:process.env,
      uid:process.getuid(),
      gid:process.getgid()
    });
    diff.on("error",function(e){
      console.error(e,item);
      next(false);
    });
    diff.stdout.on("finish",function(){
      console.log("finished pushing ", item);
    });
    req.attach("diff",diff.stdout,"diff"+item+".txt");
  });
  [4,5,6].forEach(function(item){
    var diff = cp.spawn("git",["diff", "HEAD", "HEAD^"],{
      env:process.env,
      uid:process.getuid(),
      gid:process.getgid()
    });
    diff.on("error",function(e){
      console.error(e,item);
      next(false);
    });
    diff.stdout.on("finish",function(){
      console.log("finished pushing ", item);
    });
    req.attach("diff"+item,diff.stdout,"diff.txt");
  });

  [7,8,9].forEach(function(item){
    var diff = cp.spawn("git",["diff", "HEAD", "HEAD^"],{
      env:process.env,
      uid:process.getuid(),
      gid:process.getgid()
    });
    diff.on("error",function(e){
      console.error(e,item);
      next(false);
    });
    diff.stdout.on("finish",function(){
      console.log("finished pushing ", item);
    });
    req.attach("diff"+item,diff.stdout,"diff"+item+".txt");
  });

  req.timeout(20*1000);
  req.end(function(e,res){
      if(e){
        console.error(e);
        return next(false);
      }
      next(true);
    });
},function(res){
  console.log("success: ",res);
});


function toStr(data){
  console.log(data.toString("utf8"));
}

package.json

{
  "name": "superagent-multiparty",
  "version": "1.0.0",
  "description": "",
  "main": "server.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1",
    "start": "node server.js"
  },
  "author": "formula1 <[email protected]> (https://github.com/formula1)",
  "license": "ISC",
  "dependencies": {
    "async": "^0.9.0",
    "busboy": "^0.2.9",
    "express": "^4.12.3",
    "formidable": "^1.0.17",
    "multiparty": "andrewrk/node-multiparty",
    "request": "^2.55.0",
    "split": "^0.3.3",
    "superagent": "^1.2.0"
  }
}

parter.busboy.js

var Busboy = require("busboy");


module.exports = function(isValid,req,res,next){
  var ret = {};
  var busboy = new Busboy({ headers: req.headers });
  busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
    if(!isValid(file,fieldname,filename)) return;
    file.resume();
    file.on("data",function(data){
      console.log("data recieved");
    })
    .on("error",next)
    .on("end",function(){
      console.log("finished file: ",fieldname,filename);
      next();
    });
  });
  busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
    console.log(fieldname, val);
  });
  busboy.on('finish', function() {
    console.log("Done parsing form");
    res.status(200).end("ok");
  });
  busboy.on('error',next);
  req.on('error',next);
  req.pipe(busboy);
};

parter.formidable.js

var formidable = require("formidable");


module.exports = function(isValid,req,res,next){
  var form = new formidable.IncomingForm();
  form.onPart = function(part) {
    if (!part.filename) return form.handlePart(part);
    if(!isValid(part,part.name,part.filename)) return;
    part.on("data",function(data){
      console.log("data recieved");
    })
    .on("error",next)
    .on("end",function(){
      console.log("part close");
    });
  };
  form.on("field", function(name, value) {
    console.log(name,value);
  });
  form.on("end", function(name, value) {
    console.log("Done parsing form");
    res.status(200).end("ok");
  });
  form.on("error",next);
  req.on("error",next);
  form.parse(req);
};

parter.generic.js

var multiparty = require("multiparty");

module.exports = function(isValid,req,res,next){
  isValid(req,"body","all");
  req.on("data",function(data){
    console.log("data recieved");
  });
  req.on("error",next);
  req.on("end",function(){
    res.status(200).end("ok");
  });
};

parter.multiparty.js

var multiparty = require("multiparty");

module.exports = function(isValid,req,res,next){
  var form = new multiparty.Form({maxFields:10, autoFiles:false});
  form.on("part", function(part){
    console.log("part: ",part.name);
    part.on("error",next);
    part.resume();
    if (!part.filename) return;
    if(!isValid(part,part.name,part.filename)) return;
    part.on("data",function(data){
      console.log("data recieved");
    })
    .on("end",function(){
      console.log("part close");
    });
  });
  form.on("field", function(name, value) {
    console.log(name,value);
  });
  form.on("close", function(){
    console.log("Done parsing form");
    res.status(200).end("ok");
  });
  form.on("error",function(){});
  form.parse(req);
};

server.js

var express = require("express");
var cp = require("child_process");

var diff, firstln;
cp.exec("git diff HEAD HEAD^",{
  env:process.env,
  uid:process.getuid(),
  gid:process.getgid()
},function(err,stdout){
  if(err) throw err;
  diff = stdout.toString("utf8");
  firstln = diff.split("\n")[0];
});

var app = express();

app.use(function(req,res,next){
  console.log(req.headers);
  next();
});

["generic","formidable","busboy","multiparty"].forEach(function(type){
  app.post("/"+type,function(req,res,next){
    console.log("hitting ",type);
    next();
  },require("./parter."+type).bind(void(0),isValid));

  app.get("/"+type,function(req,res,next){
    // show a file upload form
    res.writeHead(200, {'content-type': 'text/html'});
    res.write('<form action="/'+type+'" enctype="multipart/form-data" method="post">');
    for(var i=1;i<5;i++){
      res.write('<input type="hidden" name="key'+i+'" value="value'+i+'" /><br/>');
    }
    res.write('<input type="file" name="diff" ><br/>');
    res.write('<button type="submit">Upload</button>');
    res.end('</form>');
  });

});

app.use(function(err,req,res,next){

  if(err) console.error("Server Error:",req.path,err);
});


app.listen(8000,function(){
  var buff = "";
  process.stdin.pipe(require("split")()).on("data",function(line){
    switch(line){
      case "request": return runReqClient();
      case "superagent": return runSAClient();
      case "request-bad": return runReqClient(true);
      case "superagent-bad": return runSAClient(true);
    }
  });
});

function isValid(part,partname,filename){
  var ready = false;
  var buff = "";
  part.on("data",function(data){
    data = data.toString("utf8");
    if(data.indexOf(firstln) > -1){
      ready = true;
    }
    if(ready){
      buff += data;
    }
  });
  part.on("end",function(){
    var i = buff.indexOf(diff);
    if(i > -1){
      if(buff === diff){
        console.log(partname,"[",filename,"]", ": perfect diff");
      }else{
        var c = 0;
        while(i > -1){
          buff = buff.substring(i+diff.length);
          c++;
          i = buff.indexOf(diff);
        }
        console.log(partname,"[",filename,"]", ": has "+c+" diff(s)");
      }
    }else console.error(partname,"[",filename,"]", ": bad diff");
  });
  return true;
}

function runReqClient(bad){
  cp.fork("./client-req"+(bad?"-bad":"")+".js",{
    env:process.env,
    uid:process.getuid(),
    gid:process.getgid()
  });
}

function runSAClient(bad){
  cp.fork("./client-sa"+(bad?"-bad":"")+".js",{
    env:process.env,
    uid:process.getuid(),
    gid:process.getgid()
  });
}

process.on("uncaughtException",function(e){
  console.error("GLOBAL: ",e.stack);
});