Skip to content

Instantly share code, notes, and snippets.

@meandavejustice
Last active August 29, 2015 14:05
Show Gist options
  • Save meandavejustice/cd1813ebf214daa6acbb to your computer and use it in GitHub Desktop.
Save meandavejustice/cd1813ebf214daa6acbb to your computer and use it in GitHub Desktop.
s3 streaming upload issue UGH
01:18:45 +JSON_voorhees | I basically make two requests one is a wav file, one is an image. │
| The images uploads successfully and the wav file mostly fails. │
01:19:29 +JSON_voorhees | Every once in a while it will successfully upload like two minutes │
| later, but with the content-type of image/png │
01:19:49 +JSON_voorhees | Which leads me to believe I am basically "Crossing the streams" if │
| you will.
function upload(blob, filename) {
var xhr = new XMLHttpRequest(),
fd = new FormData();
fd.append( 'file', blob, filename);
xhr.open('POST', '/upload');
xhr.upload.onprogress = function(ev) {
progressBar.setAttribute('value', ev.loaded);
progressBar.setAttribute('max', ev.total);
};
xhr.send( fd );
}
var prefix = '4minutesand33seconds-' + new Date()
upload(globalAudioBlob, prefix + '.wav');
upload(canvas2blob(waveEl), prefix + '.png');
var Form = require("multiparty").Form;
module.exports = function (req, res, opts, cb) {
var form = new Form();
var fields = {};
var files = {};
var result = {
fields: fields,
files: files
};
var counter = 1;
form.parse(req);
form.on("field", function (key, value) {
// BUG: do not handle case where name appears multiple
// times in a form.
fields[key] = value;
});
form.on("part", function (part) {
if (part.filename === null) {
return;
}
counter++;
opts.handlePart.call(result, part, function (err, result) {
if (err) {
return cb(err);
}
files[part.name] = result;
finish();
});
});
function finish() {
if (--counter === 0) {
cb(null, {fields: fields, files: files});
}
}
form.once("error", cb);
form.once("close", finish);
};
26487 bytes written to public/bundle.js (0.45 seconds)
18 Aug 18:10:56 - [nodemon] v1.2.0
18 Aug 18:10:56 - [nodemon] to restart at any time, enter `rs`
18 Aug 18:10:56 - [nodemon] watching: *.*
18 Aug 18:10:56 - [nodemon] starting `node server.js`
multipart server listening on port 3000
part data::::::::::::::
{ _readableState:
{ highWaterMark: 16384,
buffer: [],
length: 0,
pipes: null,
pipesCount: 0,
flowing: null,
ended: false,
endEmitted: false,
reading: false,
sync: false,
needReadable: true,
emittedReadable: false,
readableListening: false,
objectMode: false,
defaultEncoding: 'utf8',
ranOut: false,
awaitDrain: 0,
readingMore: false,
decoder: null,
encoding: null },
readable: true,
domain: null,
_events:
{ end: { [Function: g] listener: [Function: onend] },
prefinish: { [Function: g] listener: [Function] },
drain: [Function] },
_maxListeners: 10,
_writableState:
{ highWaterMark: 16384,
objectMode: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function],
writecb: null,
writelen: 0,
buffer: [],
pendingcb: 0,
prefinished: false,
errorEmitted: false },
writable: true,
allowHalfOpen: true,
_transformState:
{ afterTransform: [Function],
needTransform: false,
transforming: false,
writecb: null,
writechunk: null },
headers:
{ 'content-disposition': 'form-data; name="file"; filename="4minutesand33seconds-Mon Aug 18 2014 18:11:02 GMT-0700 (PDT).wav"',
'content-type': 'audio/wav' },
name: 'file',
filename: '4minutesand33seconds-Mon Aug 18 2014 18:11:02 GMT-0700 (PDT).wav',
byteOffset: 191,
byteCount: 704556 }
4minutesand33seconds-Mon Aug 18 2014 18:11:02 GMT-0700 (PDT).png image/png
part data::::::::::::::
{ _readableState:
{ highWaterMark: 16384,
buffer: [],
length: 0,
pipes: null,
pipesCount: 0,
flowing: null,
ended: false,
endEmitted: false,
reading: false,
sync: false,
needReadable: true,
emittedReadable: false,
readableListening: false,
objectMode: false,
defaultEncoding: 'utf8',
ranOut: false,
awaitDrain: 0,
readingMore: false,
decoder: null,
encoding: null },
readable: true,
domain: null,
_events:
{ end: { [Function: g] listener: [Function: onend] },
prefinish: { [Function: g] listener: [Function] },
drain: [Function] },
_maxListeners: 10,
_writableState:
{ highWaterMark: 16384,
objectMode: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function],
writecb: null,
writelen: 0,
buffer: [],
pendingcb: 0,
prefinished: false,
errorEmitted: false },
writable: true,
allowHalfOpen: true,
_transformState:
{ afterTransform: [Function],
needTransform: false,
transforming: false,
writecb: null,
writechunk: null },
headers:
{ 'content-disposition': 'form-data; name="file"; filename="4minutesand33seconds-Mon Aug 18 2014 18:11:02 GMT-0700 (PDT).png"',
'content-type': 'image/png' },
name: 'file',
filename: '4minutesand33seconds-Mon Aug 18 2014 18:11:02 GMT-0700 (PDT).png',
byteOffset: 191,
byteCount: 8239 }
error in upload stream::: Failed to upload a part to S3: {"message":"The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.","code":"NoSuchUpload","time":"2014-08-19T01:11:04.375Z","statusCode":404,"retryable":false}
Additionally failed to abort the multipart upload on S3: NoSuchUpload: The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.
error in upload stream::: Failed to upload a part to S3: {"message":"The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.","code":"NoSuchUpload","time":"2014-08-19T01:11:06.601Z","statusCode":404,"retryable":false}
Additionally failed to abort the multipart upload on S3: NoSuchUpload: The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.
error in upload stream::: Failed to upload a part to S3: {"message":"The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.","code":"NoSuchUpload","time":"2014-08-19T01:11:08.338Z","statusCode":404,"retryable":false}
Additionally failed to abort the multipart upload on S3: NoSuchUpload: The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.
error in upload stream::: Failed to upload a part to S3: {"message":"The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.","code":"NoSuchUpload","time":"2014-08-19T01:11:10.277Z","statusCode":404,"retryable":false}
Additionally failed to abort the multipart upload on S3: NoSuchUpload: The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.
app.addRoute("/upload", function (req, res, opts, cb) {
MultipartyForm(req, res, {
handlePart: function(part) {
send2s3(part, function(err, filename) {
if (err) console.log(err);
filetracker.addFile(filename);
});
}
}, function (err, values) {
if (err) {
console.log('error in multipart form', err);
}
console.log('File uploaded succesfully');
});
});
var AWS = require('aws-sdk');
AWS.config.httpOptions = {timeout: 5000};
var Uploader = require('s3-upload-stream').Uploader;
var bucketConfig = {
"Bucket": "4minutesand33seconds",
"ACL": "public-read",
"ContentType": "audio/mpeg"
};
var s3Config = {
"accessKeyId": process.env.S3ID,
"secretAccessKey": process.env.S3SECRET
// "region": "us-west-2"
};
module.exports = function (part, cb) {
bucketConfig.Key = part.filename;
if (!~part.headers['content-type'].indexOf('wav')) {
console.log(part.filename, part.headers['content-type']);
bucketConfig.ContentType = part.headers['content-type'] ;
}
console.log('part data:::::::::::::: \n \n \n', part);
function processStream(err, uploadStream) {
if(err) {
console.log(err, uploadStream);
} else {
uploadStream.on('error', function (err) {
console.log('error in upload stream:::', err);
});
// uploadStream.on('chunk', function (data) {});
uploadStream.on('uploaded', function (data) {
cb(null, part.filename);
});
part.pipe(uploadStream);
}
}
var streamObj = new Uploader(s3Config, bucketConfig, processStream);
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment