Last active
August 29, 2015 14:05
-
-
Save meandavejustice/cd1813ebf214daa6acbb to your computer and use it in GitHub Desktop.
s3 streaming upload issue UGH
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
01:18:45 +JSON_voorhees | I basically make two requests one is a wav file, one is an image. │ | |
| The images uploads successfully and the wav file mostly fails. │ | |
01:19:29 +JSON_voorhees | Every once in a while it will successfully upload like two minutes │ | |
| later, but with the content-type of image/png │ | |
01:19:49 +JSON_voorhees | Which leads me to believe I am basically "Crossing the streams" if │ | |
| you will. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function upload(blob, filename) { | |
var xhr = new XMLHttpRequest(), | |
fd = new FormData(); | |
fd.append( 'file', blob, filename); | |
xhr.open('POST', '/upload'); | |
xhr.upload.onprogress = function(ev) { | |
progressBar.setAttribute('value', ev.loaded); | |
progressBar.setAttribute('max', ev.total); | |
}; | |
xhr.send( fd ); | |
} | |
var prefix = '4minutesand33seconds-' + new Date() | |
upload(globalAudioBlob, prefix + '.wav'); | |
upload(canvas2blob(waveEl), prefix + '.png'); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var Form = require("multiparty").Form; | |
module.exports = function (req, res, opts, cb) { | |
var form = new Form(); | |
var fields = {}; | |
var files = {}; | |
var result = { | |
fields: fields, | |
files: files | |
}; | |
var counter = 1; | |
form.parse(req); | |
form.on("field", function (key, value) { | |
// BUG: do not handle case where name appears multiple | |
// times in a form. | |
fields[key] = value; | |
}); | |
form.on("part", function (part) { | |
if (part.filename === null) { | |
return; | |
} | |
counter++; | |
opts.handlePart.call(result, part, function (err, result) { | |
if (err) { | |
return cb(err); | |
} | |
files[part.name] = result; | |
finish(); | |
}); | |
}); | |
function finish() { | |
if (--counter === 0) { | |
cb(null, {fields: fields, files: files}); | |
} | |
} | |
form.once("error", cb); | |
form.once("close", finish); | |
}; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26487 bytes written to public/bundle.js (0.45 seconds) | |
18 Aug 18:10:56 - [nodemon] v1.2.0 | |
18 Aug 18:10:56 - [nodemon] to restart at any time, enter `rs` | |
18 Aug 18:10:56 - [nodemon] watching: *.* | |
18 Aug 18:10:56 - [nodemon] starting `node server.js` | |
multipart server listening on port 3000 | |
part data:::::::::::::: | |
{ _readableState: | |
{ highWaterMark: 16384, | |
buffer: [], | |
length: 0, | |
pipes: null, | |
pipesCount: 0, | |
flowing: null, | |
ended: false, | |
endEmitted: false, | |
reading: false, | |
sync: false, | |
needReadable: true, | |
emittedReadable: false, | |
readableListening: false, | |
objectMode: false, | |
defaultEncoding: 'utf8', | |
ranOut: false, | |
awaitDrain: 0, | |
readingMore: false, | |
decoder: null, | |
encoding: null }, | |
readable: true, | |
domain: null, | |
_events: | |
{ end: { [Function: g] listener: [Function: onend] }, | |
prefinish: { [Function: g] listener: [Function] }, | |
drain: [Function] }, | |
_maxListeners: 10, | |
_writableState: | |
{ highWaterMark: 16384, | |
objectMode: false, | |
needDrain: false, | |
ending: false, | |
ended: false, | |
finished: false, | |
decodeStrings: true, | |
defaultEncoding: 'utf8', | |
length: 0, | |
writing: false, | |
corked: 0, | |
sync: true, | |
bufferProcessing: false, | |
onwrite: [Function], | |
writecb: null, | |
writelen: 0, | |
buffer: [], | |
pendingcb: 0, | |
prefinished: false, | |
errorEmitted: false }, | |
writable: true, | |
allowHalfOpen: true, | |
_transformState: | |
{ afterTransform: [Function], | |
needTransform: false, | |
transforming: false, | |
writecb: null, | |
writechunk: null }, | |
headers: | |
{ 'content-disposition': 'form-data; name="file"; filename="4minutesand33seconds-Mon Aug 18 2014 18:11:02 GMT-0700 (PDT).wav"', | |
'content-type': 'audio/wav' }, | |
name: 'file', | |
filename: '4minutesand33seconds-Mon Aug 18 2014 18:11:02 GMT-0700 (PDT).wav', | |
byteOffset: 191, | |
byteCount: 704556 } | |
4minutesand33seconds-Mon Aug 18 2014 18:11:02 GMT-0700 (PDT).png image/png | |
part data:::::::::::::: | |
{ _readableState: | |
{ highWaterMark: 16384, | |
buffer: [], | |
length: 0, | |
pipes: null, | |
pipesCount: 0, | |
flowing: null, | |
ended: false, | |
endEmitted: false, | |
reading: false, | |
sync: false, | |
needReadable: true, | |
emittedReadable: false, | |
readableListening: false, | |
objectMode: false, | |
defaultEncoding: 'utf8', | |
ranOut: false, | |
awaitDrain: 0, | |
readingMore: false, | |
decoder: null, | |
encoding: null }, | |
readable: true, | |
domain: null, | |
_events: | |
{ end: { [Function: g] listener: [Function: onend] }, | |
prefinish: { [Function: g] listener: [Function] }, | |
drain: [Function] }, | |
_maxListeners: 10, | |
_writableState: | |
{ highWaterMark: 16384, | |
objectMode: false, | |
needDrain: false, | |
ending: false, | |
ended: false, | |
finished: false, | |
decodeStrings: true, | |
defaultEncoding: 'utf8', | |
length: 0, | |
writing: false, | |
corked: 0, | |
sync: true, | |
bufferProcessing: false, | |
onwrite: [Function], | |
writecb: null, | |
writelen: 0, | |
buffer: [], | |
pendingcb: 0, | |
prefinished: false, | |
errorEmitted: false }, | |
writable: true, | |
allowHalfOpen: true, | |
_transformState: | |
{ afterTransform: [Function], | |
needTransform: false, | |
transforming: false, | |
writecb: null, | |
writechunk: null }, | |
headers: | |
{ 'content-disposition': 'form-data; name="file"; filename="4minutesand33seconds-Mon Aug 18 2014 18:11:02 GMT-0700 (PDT).png"', | |
'content-type': 'image/png' }, | |
name: 'file', | |
filename: '4minutesand33seconds-Mon Aug 18 2014 18:11:02 GMT-0700 (PDT).png', | |
byteOffset: 191, | |
byteCount: 8239 } | |
error in upload stream::: Failed to upload a part to S3: {"message":"The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.","code":"NoSuchUpload","time":"2014-08-19T01:11:04.375Z","statusCode":404,"retryable":false} | |
Additionally failed to abort the multipart upload on S3: NoSuchUpload: The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed. | |
error in upload stream::: Failed to upload a part to S3: {"message":"The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.","code":"NoSuchUpload","time":"2014-08-19T01:11:06.601Z","statusCode":404,"retryable":false} | |
Additionally failed to abort the multipart upload on S3: NoSuchUpload: The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed. | |
error in upload stream::: Failed to upload a part to S3: {"message":"The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.","code":"NoSuchUpload","time":"2014-08-19T01:11:08.338Z","statusCode":404,"retryable":false} | |
Additionally failed to abort the multipart upload on S3: NoSuchUpload: The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed. | |
error in upload stream::: Failed to upload a part to S3: {"message":"The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.","code":"NoSuchUpload","time":"2014-08-19T01:11:10.277Z","statusCode":404,"retryable":false} | |
Additionally failed to abort the multipart upload on S3: NoSuchUpload: The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
app.addRoute("/upload", function (req, res, opts, cb) { | |
MultipartyForm(req, res, { | |
handlePart: function(part) { | |
send2s3(part, function(err, filename) { | |
if (err) console.log(err); | |
filetracker.addFile(filename); | |
}); | |
} | |
}, function (err, values) { | |
if (err) { | |
console.log('error in multipart form', err); | |
} | |
console.log('File uploaded succesfully'); | |
}); | |
}); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var AWS = require('aws-sdk'); | |
AWS.config.httpOptions = {timeout: 5000}; | |
var Uploader = require('s3-upload-stream').Uploader; | |
var bucketConfig = { | |
"Bucket": "4minutesand33seconds", | |
"ACL": "public-read", | |
"ContentType": "audio/mpeg" | |
}; | |
var s3Config = { | |
"accessKeyId": process.env.S3ID, | |
"secretAccessKey": process.env.S3SECRET | |
// "region": "us-west-2" | |
}; | |
module.exports = function (part, cb) { | |
bucketConfig.Key = part.filename; | |
if (!~part.headers['content-type'].indexOf('wav')) { | |
console.log(part.filename, part.headers['content-type']); | |
bucketConfig.ContentType = part.headers['content-type'] ; | |
} | |
console.log('part data:::::::::::::: \n \n \n', part); | |
function processStream(err, uploadStream) { | |
if(err) { | |
console.log(err, uploadStream); | |
} else { | |
uploadStream.on('error', function (err) { | |
console.log('error in upload stream:::', err); | |
}); | |
// uploadStream.on('chunk', function (data) {}); | |
uploadStream.on('uploaded', function (data) { | |
cb(null, part.filename); | |
}); | |
part.pipe(uploadStream); | |
} | |
} | |
var streamObj = new Uploader(s3Config, bucketConfig, processStream); | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment