-
-
Save chapani/6072189 to your computer and use it in GitHub Desktop.
%%%------------------------------------------------------------------- | |
%%% @author egobrain <egobrain@linux-ympb> | |
%%% @copyright (C) 2012, egobrain | |
%%% @doc | |
%%% Function for uploading files and properties,which were sent as a | |
%%% multipart. Files are stored in tmp_folder with random name, | |
%%% generated by tmp_filename function. | |
%%% @end | |
%%% Created : 25 Mar 2012 by egobrain <egobrain@linux-ympb> | |
%%%------------------------------------------------------------------- | |
-module(uploader). | |
-export([files_qs/1, | |
files_qs/2]). | |
-define(MAX_FILE_SIZE, 5000000). | |
-define(MAX_FILES, unlimited). | |
-define(TMP_PATH, "/tmp"). | |
-define(TMP_FILE_NAME, fun tmp_filename/1). | |
-record(state,{ | |
headers = [], | |
max_file_size = ?MAX_FILE_SIZE :: non_neg_integer(), | |
max_files = ?MAX_FILES :: non_neg_integer() | unlimited, | |
tmp_folder = ?TMP_PATH :: string(), | |
tmp_filename = ?TMP_FILE_NAME :: function(), | |
files_cnt = 0 | |
}). | |
-record(datafile,{name = <<"">> :: binary(), | |
size = 0 :: non_neg_integer(), | |
'Content-Type' :: binary(), | |
'Original-Name' :: binary() | |
}). | |
%% =================================================================== | |
%%% Types | |
%% =================================================================== | |
-type datafile() :: #datafile{}. | |
-type headers() :: [datafile() | {cowboy_http:header(), iodata()}]. | |
-type option() :: {max_file_size, Size :: non_neg_integer()} | | |
{tmp_filename,Fun :: function()} | | |
{tmp_folder, Path :: string}. | |
-type options() :: [option()]. | |
-type error() :: {error, file_too_big | max_files_limit}. | |
-export_type([datafile/0,headers/0,options/0]). | |
%% =================================================================== | |
%%% Api | |
%% =================================================================== | |
-spec files_qs(Req :: cowboy_http:req()) -> {headers(), Req :: cowboy_http:req() } | error(). | |
files_qs(Req) -> | |
files_qs(Req, []). | |
-spec files_qs(Req :: cowboy_http:req() , Options :: options()) -> {headers(), Req :: cowboy_http:req()} | error(). | |
files_qs(Req, Options) -> | |
State = #state{headers=[], | |
files_cnt=0, | |
max_file_size= proplists:get_value(max_file_size,Options, ?MAX_FILE_SIZE), | |
max_files = proplists:get_value(max_files, Options, ?MAX_FILES), | |
tmp_folder = proplists:get_value(tmp_folder, Options, ?TMP_PATH), | |
tmp_filename = proplists:get_value(tmp_filename, Options, ?TMP_FILE_NAME) | |
}, | |
acc(Req, State). | |
%% =================================================================== | |
%%% Internal functions | |
%% =================================================================== | |
acc(Req,State) -> | |
case cowboy_req:multipart_data(Req) of | |
{headers, Headers, Req2} -> | |
acc({headers, Headers}, Req2, State); | |
{eof, Req2} -> | |
acc(eof, Req2, State) | |
end. | |
acc({headers,NewHeaders},Req,#state{headers=Headers, | |
tmp_folder=TmpFolder, | |
tmp_filename=TmpFilenameFun, | |
max_file_size=MaxFileSize, | |
max_files=MaxFiles, | |
files_cnt=FilesCnt | |
} = State) -> | |
case parse_headers(NewHeaders) of | |
{error,_Reason} = Err -> | |
Err; | |
{Name, undefined} -> | |
{Value,Req2} = acc_property(Req), | |
acc(Req2,State#state{headers=[{Name,Value}|Headers]}); | |
{Name, Filename} -> | |
case MaxFiles =:= unlimited orelse FilesCnt < MaxFiles of | |
true -> | |
TempFilename = filename:join([TmpFolder,TmpFilenameFun(Filename)]), | |
{ok, File} = file:open(TempFilename, [raw, write]), | |
Res = acc_file(Req,File,MaxFileSize), | |
file:close(File), | |
case Res of | |
{error,file_too_big} = Err -> | |
file:delete(TempFilename), | |
delete_files(Headers), | |
Err; | |
{FileSize,Req2} -> | |
FileProp = #datafile{name=TempFilename, | |
size=FileSize, | |
'Original-Name'=Filename, | |
'Content-Type'= | |
proplists:get_value('Content-Type',NewHeaders)}, | |
%% Mana shuni tugrilashimiz kerak | |
acc(Req2,State#state{headers=[FileProp,Headers],files_cnt=FilesCnt+1}) | |
end; | |
false -> | |
delete_files(Headers), | |
{error,max_files_limit} | |
end | |
end; | |
acc(eof,Req,#state{headers=Headers}) -> {Headers,Req}. | |
parse_headers(Headers) -> | |
case proplists:get_value(<<"content-disposition">>,Headers) of | |
undefined -> | |
{error,"No content disposition in header"}; | |
Value -> | |
{_FormData,Props} = cowboy_multipart:content_disposition(Value), | |
Name = proplists:get_value(<<"name">>,Props), | |
Filename = proplists:get_value(<<"filename">>,Props), | |
{Name,Filename} | |
end. | |
-spec acc_file(Req :: cowboy_http:req() , File :: file:io_device() ,MaxFileSize :: non_neg_integer()) -> | |
{FileSize :: non_neg_integer(), Req2 :: cowboy_http:req()} | | |
{error,file_too_big}. | |
acc_file(Req,File,MaxFileSize) -> acc_file(Req,File,0,MaxFileSize). | |
acc_file(Req,File,FileSize,MaxFileSize) -> | |
case cowboy_req:multipart_data(Req) of | |
{end_of_part, Req2} -> | |
{FileSize,Req2}; | |
{body,Data, Req2} -> | |
NewFileSize = FileSize + byte_size(Data), | |
case NewFileSize > MaxFileSize of | |
true -> | |
{error,file_too_big}; | |
false -> | |
ok = file:write(File,Data), | |
acc_file(Req2,File,NewFileSize,MaxFileSize) | |
end | |
end. | |
-spec acc_property(_Req) -> {Data :: binary(),_Req2}. | |
acc_property(Req) -> | |
acc_property(Req,<<>>). | |
acc_property(Req,Buff) -> | |
case cowboy_req:multipart_data(Req) of | |
{end_of_part, Req2} -> | |
{Buff, Req2}; | |
{body,Data, Req2} -> | |
acc_property(Req2,<<Buff/binary,Data/binary>>) | |
end. | |
%% =================================================================== | |
%%% Default Functions | |
%% =================================================================== | |
-spec tmp_filename(OriginalName :: binary()) -> string() | [string()]. | |
tmp_filename(_OriginalName) -> | |
atom_to_list(?MODULE) ++ integer_to_list(erlang:phash2(make_ref())). | |
delete_files(Headers) -> | |
lists:foreach(fun(#datafile{name=FName}) -> | |
file:delete(FName); | |
(_) -> ok | |
end,Headers). |
Please, tell me!
How set more timeout wait data upload?
Вecause that, i think, when i test uploading file on my Android app client i have error, please see this Request logs:
{http_req,#Port<0.2049>,ranch_tcp,keepalive,<0.301.0>,
<<"POST">>,'HTTP/1.1',
{{95,136,120,140},49970},
<<"194.8.153.60">>,undefined,8010,<<"/upload/">>,
undefined,<<>>,undefined,[],
[{<<"content-length">>,<<"1604016">>},
{<<"content-type">>,
<<"multipart/form-data; boundary=vu7qfgk_fb4fJJkk129W4fm76ACngk9Thpm">>},
{<<"host">>,<<"194.8.153.6:8010">>},
{<<"connection">>,<<"Keep-Alive">>}],
[{<<"connection">>,[<<"keep-alive">>]}],
undefined,[],waiting,undefined,
<<"--vu7qfgk_fb4fJJkk129W4fm76ACngk9Thpm\r\nContent-Disposition: form-data; name=\"\";
filename=\"1.mp4\"\r\nContent-Type: application/octet-stream\r\nContent-Transfer-Encoding: binary\r\n\r\n">>,
false,waiting,[],<<>>,undefined}
ERROR logs:
ERROR REPORT==== 31-Jul-2013::19:37:24 ===
** Cowboy handler cdn_post_file terminating in handle/2
for the reason error:{case_clause,{error,timeout}}
** Handler state was undefined
** Request was [{socket,#Port<0.2049>},
{transport,ranch_tcp},
{connection,keepalive},
{pid,<0.301.0>},
{method,<<"POST">>},
{version,'HTTP/1.1'},
{peer,{{95,136,120,140},49970}},
{host,<<"194.8.153.60">>},
{host_info,undefined},
{port,8010},
{path,<<"/upload/">>},
{path_info,undefined},
{qs,<<>>},
{qs_vals,undefined},
{bindings,[]},
{headers,[{<<"content-length">>,<<"1604016">>},
{<<"content-type">>,
<<"multipart/form-data; boundary=vu7qfgk_fb4fJJkk129W4fm76ACngk9Thpm">>},
{<<"host">>,<<"194.8.153.60:8010">>},
{<<"connection">>,<<"Keep-Alive">>}]},
{p_headers,[{<<"connection">>,[<<"keep-alive">>]}]},
{cookies,undefined},
{meta,[]},
{body_state,waiting},
{multipart,undefined},
{buffer,<<"--vu7qfgk_fb4fJJkk129W4fm76ACngk9Thpm\r\nContent-Disposition: form-data; name=\"\";
filename=\"1.mp4\"\r\nContent-Type: application/octet-stream\r\nContent-Transfer-Encoding: binary\r\n\r\n">>},
{resp_compress,false},
{resp_state,waiting},
{resp_headers,[]},
{resp_body,<<>>},
{onresponse,undefined}]
** Stacktrace: [{cowboy_req,multipart_data,3,
[{file,"src/cowboy_req.erl"},{line,825}]},
{cdn_file_uploader,acc_file,4,
[{file,"src/cdn_file_uploader.erl"},
{line,141}]},
{cdn_file_uploader,acc,3,
[{file,"src/cdn_file_uploader.erl"},
{line,99}]},
{cdn_post_file,handle_upload,2,
[{file,"src/cdn_post_file.erl"},{line,44}]},
{cowboy_handler,handler_handle,4,
[{file,"src/cowboy_handler.erl"},{line,119}]},
{cowboy_protocol,execute,4,
[{file,"src/cowboy_protocol.erl"},
{line,523}]}]
Thank You very much!!!
Thank you!
I realized!!! My Android Client send file on server very slow-slow, and need more timeout, but also for local network ( 5MB filesize = 60 s) i m set time out 100000, its badly (
Sorry, I just saw your comments. Somehow, github failed to notify me about them.
@buriwoy Thanks for your gist!
it work fine.
But tell me please, how can I get parameters from multipart form POST request, when I upload the file with the?
Web form have "User name", "Comment", and othere sending parameters
@buriwoy
Thank you for giving me time to think and understand
I myself was able to get an answer to my question:
{Opts, Req2} = file_uploader:files_qs(Req),
{_, Comment} = lists:keyfind(<<"comment">>,1, lists:flatten(Opts)),
Thanks, that's great!!!