Skip to content

Instantly share code, notes, and snippets.

@chapani
Last active December 20, 2015 04:39
Show Gist options
  • Save chapani/6072189 to your computer and use it in GitHub Desktop.
Save chapani/6072189 to your computer and use it in GitHub Desktop.
https://gist.github.com/2834137 Slightly modified to be compatible cowboy 0.6
%%%-------------------------------------------------------------------
%%% @author egobrain <egobrain@linux-ympb>
%%% @copyright (C) 2012, egobrain
%%% @doc
%%% Function for uploading files and properties,which were sent as a
%%% multipart. Files are stored in tmp_folder with random name,
%%% generated by tmp_filename function.
%%% @end
%%% Created : 25 Mar 2012 by egobrain <egobrain@linux-ympb>
%%%-------------------------------------------------------------------
-module(uploader).
-export([files_qs/1,
files_qs/2]).
-define(MAX_FILE_SIZE, 5000000).
-define(MAX_FILES, unlimited).
-define(TMP_PATH, "/tmp").
-define(TMP_FILE_NAME, fun tmp_filename/1).
-record(state,{
headers = [],
max_file_size = ?MAX_FILE_SIZE :: non_neg_integer(),
max_files = ?MAX_FILES :: non_neg_integer() | unlimited,
tmp_folder = ?TMP_PATH :: string(),
tmp_filename = ?TMP_FILE_NAME :: function(),
files_cnt = 0
}).
-record(datafile,{name = <<"">> :: binary(),
size = 0 :: non_neg_integer(),
'Content-Type' :: binary(),
'Original-Name' :: binary()
}).
%% ===================================================================
%%% Types
%% ===================================================================
-type datafile() :: #datafile{}.
-type headers() :: [datafile() | {cowboy_http:header(), iodata()}].
-type option() :: {max_file_size, Size :: non_neg_integer()} |
{tmp_filename,Fun :: function()} |
{tmp_folder, Path :: string}.
-type options() :: [option()].
-type error() :: {error, file_too_big | max_files_limit}.
-export_type([datafile/0,headers/0,options/0]).
%% ===================================================================
%%% Api
%% ===================================================================
-spec files_qs(Req :: cowboy_http:req()) -> {headers(), Req :: cowboy_http:req() } | error().
files_qs(Req) ->
files_qs(Req, []).
-spec files_qs(Req :: cowboy_http:req() , Options :: options()) -> {headers(), Req :: cowboy_http:req()} | error().
files_qs(Req, Options) ->
State = #state{headers=[],
files_cnt=0,
max_file_size= proplists:get_value(max_file_size,Options, ?MAX_FILE_SIZE),
max_files = proplists:get_value(max_files, Options, ?MAX_FILES),
tmp_folder = proplists:get_value(tmp_folder, Options, ?TMP_PATH),
tmp_filename = proplists:get_value(tmp_filename, Options, ?TMP_FILE_NAME)
},
acc(Req, State).
%% ===================================================================
%%% Internal functions
%% ===================================================================
acc(Req,State) ->
case cowboy_req:multipart_data(Req) of
{headers, Headers, Req2} ->
acc({headers, Headers}, Req2, State);
{eof, Req2} ->
acc(eof, Req2, State)
end.
acc({headers,NewHeaders},Req,#state{headers=Headers,
tmp_folder=TmpFolder,
tmp_filename=TmpFilenameFun,
max_file_size=MaxFileSize,
max_files=MaxFiles,
files_cnt=FilesCnt
} = State) ->
case parse_headers(NewHeaders) of
{error,_Reason} = Err ->
Err;
{Name, undefined} ->
{Value,Req2} = acc_property(Req),
acc(Req2,State#state{headers=[{Name,Value}|Headers]});
{Name, Filename} ->
case MaxFiles =:= unlimited orelse FilesCnt < MaxFiles of
true ->
TempFilename = filename:join([TmpFolder,TmpFilenameFun(Filename)]),
{ok, File} = file:open(TempFilename, [raw, write]),
Res = acc_file(Req,File,MaxFileSize),
file:close(File),
case Res of
{error,file_too_big} = Err ->
file:delete(TempFilename),
delete_files(Headers),
Err;
{FileSize,Req2} ->
FileProp = #datafile{name=TempFilename,
size=FileSize,
'Original-Name'=Filename,
'Content-Type'=
proplists:get_value('Content-Type',NewHeaders)},
%% Mana shuni tugrilashimiz kerak
acc(Req2,State#state{headers=[FileProp,Headers],files_cnt=FilesCnt+1})
end;
false ->
delete_files(Headers),
{error,max_files_limit}
end
end;
acc(eof,Req,#state{headers=Headers}) -> {Headers,Req}.
parse_headers(Headers) ->
case proplists:get_value(<<"content-disposition">>,Headers) of
undefined ->
{error,"No content disposition in header"};
Value ->
{_FormData,Props} = cowboy_multipart:content_disposition(Value),
Name = proplists:get_value(<<"name">>,Props),
Filename = proplists:get_value(<<"filename">>,Props),
{Name,Filename}
end.
-spec acc_file(Req :: cowboy_http:req() , File :: file:io_device() ,MaxFileSize :: non_neg_integer()) ->
{FileSize :: non_neg_integer(), Req2 :: cowboy_http:req()} |
{error,file_too_big}.
acc_file(Req,File,MaxFileSize) -> acc_file(Req,File,0,MaxFileSize).
acc_file(Req,File,FileSize,MaxFileSize) ->
case cowboy_req:multipart_data(Req) of
{end_of_part, Req2} ->
{FileSize,Req2};
{body,Data, Req2} ->
NewFileSize = FileSize + byte_size(Data),
case NewFileSize > MaxFileSize of
true ->
{error,file_too_big};
false ->
ok = file:write(File,Data),
acc_file(Req2,File,NewFileSize,MaxFileSize)
end
end.
-spec acc_property(_Req) -> {Data :: binary(),_Req2}.
acc_property(Req) ->
acc_property(Req,<<>>).
acc_property(Req,Buff) ->
case cowboy_req:multipart_data(Req) of
{end_of_part, Req2} ->
{Buff, Req2};
{body,Data, Req2} ->
acc_property(Req2,<<Buff/binary,Data/binary>>)
end.
%% ===================================================================
%%% Default Functions
%% ===================================================================
-spec tmp_filename(OriginalName :: binary()) -> string() | [string()].
tmp_filename(_OriginalName) ->
atom_to_list(?MODULE) ++ integer_to_list(erlang:phash2(make_ref())).
delete_files(Headers) ->
lists:foreach(fun(#datafile{name=FName}) ->
file:delete(FName);
(_) -> ok
end,Headers).
@Kozlov-V
Copy link

Please, tell me!
How set more timeout wait data upload?
Вecause that, i think, when i test uploading file on my Android app client i have error, please see this Request logs:

{http_req,#Port<0.2049>,ranch_tcp,keepalive,<0.301.0>,
                             <<"POST">>,'HTTP/1.1',
                             {{95,136,120,140},49970},
                             <<"194.8.153.60">>,undefined,8010,<<"/upload/">>,
                             undefined,<<>>,undefined,[],
                             [{<<"content-length">>,<<"1604016">>},
                              {<<"content-type">>,
                               <<"multipart/form-data; boundary=vu7qfgk_fb4fJJkk129W4fm76ACngk9Thpm">>},
                              {<<"host">>,<<"194.8.153.6:8010">>},
                              {<<"connection">>,<<"Keep-Alive">>}],
                             [{<<"connection">>,[<<"keep-alive">>]}],
                             undefined,[],waiting,undefined,
                             <<"--vu7qfgk_fb4fJJkk129W4fm76ACngk9Thpm\r\nContent-Disposition: form-data; name=\"\";
 filename=\"1.mp4\"\r\nContent-Type: application/octet-stream\r\nContent-Transfer-Encoding: binary\r\n\r\n">>,
                             false,waiting,[],<<>>,undefined}

ERROR logs:

ERROR REPORT==== 31-Jul-2013::19:37:24 ===
** Cowboy handler cdn_post_file terminating in handle/2
   for the reason error:{case_clause,{error,timeout}}
** Handler state was undefined
** Request was [{socket,#Port<0.2049>},
                {transport,ranch_tcp},
                {connection,keepalive},
                {pid,<0.301.0>},
                {method,<<"POST">>},
                {version,'HTTP/1.1'},
                {peer,{{95,136,120,140},49970}},
                {host,<<"194.8.153.60">>},
                {host_info,undefined},
                {port,8010},
                {path,<<"/upload/">>},
                {path_info,undefined},
                {qs,<<>>},
                {qs_vals,undefined},
                {bindings,[]},
                {headers,[{<<"content-length">>,<<"1604016">>},
                          {<<"content-type">>,
                           <<"multipart/form-data; boundary=vu7qfgk_fb4fJJkk129W4fm76ACngk9Thpm">>},
                          {<<"host">>,<<"194.8.153.60:8010">>},
                          {<<"connection">>,<<"Keep-Alive">>}]},
                {p_headers,[{<<"connection">>,[<<"keep-alive">>]}]},
                {cookies,undefined},
                {meta,[]},
                {body_state,waiting},
                {multipart,undefined},
                {buffer,<<"--vu7qfgk_fb4fJJkk129W4fm76ACngk9Thpm\r\nContent-Disposition: form-data; name=\"\";
 filename=\"1.mp4\"\r\nContent-Type: application/octet-stream\r\nContent-Transfer-Encoding: binary\r\n\r\n">>},
                {resp_compress,false},
                {resp_state,waiting},
                {resp_headers,[]},
                {resp_body,<<>>},
                {onresponse,undefined}]
** Stacktrace: [{cowboy_req,multipart_data,3,
                            [{file,"src/cowboy_req.erl"},{line,825}]},
                {cdn_file_uploader,acc_file,4,
                                   [{file,"src/cdn_file_uploader.erl"},
                                    {line,141}]},
                {cdn_file_uploader,acc,3,
                                   [{file,"src/cdn_file_uploader.erl"},
                                    {line,99}]},
                {cdn_post_file,handle_upload,2,
                               [{file,"src/cdn_post_file.erl"},{line,44}]},
                {cowboy_handler,handler_handle,4,
                                [{file,"src/cowboy_handler.erl"},{line,119}]},
                {cowboy_protocol,execute,4,
                                 [{file,"src/cowboy_protocol.erl"},
                                  {line,523}]}]

Thank You very much!!!

@Kozlov-V
Copy link

Kozlov-V commented Aug 1, 2013

Thank you!
I realized!!! My Android Client send file on server very slow-slow, and need more timeout, but also for local network ( 5MB filesize = 60 s) i m set time out 100000, its badly (

@chapani
Copy link
Author

chapani commented Aug 21, 2013

Sorry, I just saw your comments. Somehow, github failed to notify me about them.

@Kozlov-V
Copy link

@buriwoy Thanks for your gist!
it work fine.
But tell me please, how can I get parameters from multipart form POST request, when I upload the file with the?
Web form have "User name", "Comment", and othere sending parameters

@Kozlov-V
Copy link

@buriwoy
Thank you for giving me time to think and understand
I myself was able to get an answer to my question:

{Opts, Req2} = file_uploader:files_qs(Req),
{_, Comment} = lists:keyfind(<<"comment">>,1, lists:flatten(Opts)),

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment