Chapril/xmpp.chapril.org-ejabberd
25
1
Fork 0
mirror of https://github.com/processone/ejabberd.git synced 2024-11-20 16:15:59 +01:00
Code Releases Activity

Optimize HTTP requests memory usage

Browse Source 
Due to historical reasons, ejabberd loads the whole file/data
into the memory when serving an HTTP request. This is now improved:

1) For GET requests ejabberd uses sendfile(2) if the underlying
   connection is HTTP and falls back to read/write loop with 64kb
   buffer for HTTPS connections. This type of requests are handled
   by mod_http_fileserver, mod_http_upload, ejabberd_captcha, etc
2) POST requests are now limited to 20Mb and are fully downloaded
   into the memory for further processing (by ejabberd_web_admin,
   mod_bosh, etc)
3) PUT requests (e.g. for mod_http_upload) are handled by read/write
   loop with 64kb buffer
This commit is contained in:
Evgeniy Khramtsov 2018-05-14 19:30:21 +03:00
parent cb3bb710bd
commit 063737e4f5
6 changed files with 267 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/ejabberd_http.hrl
View File

@ -31,7 +31,10 @@
port = 5280 :: inet:port_number(), port = 5280 :: inet:port_number(),
opts = [] :: list(), opts = [] :: list(),
tp = http :: protocol(), tp = http :: protocol(),
headers = [] :: [{atom() | binary(), binary()}]}). headers = [] :: [{atom() | binary(), binary()}],
length = 0 :: non_neg_integer(),
sockmod :: gen_tcp | fast_tls,
socket :: inet:socket() | fast_tls:tls_socket()}).
-record(ws, -record(ws,
{socket :: inet:socket() | fast_tls:tls_socket(), {socket :: inet:socket() | fast_tls:tls_socket(),

337
src/ejabberd_http.erl
View File

@ -31,17 +31,16 @@
%% External exports %% External exports
-export([start/2, start_link/2, become_controller/1, -export([start/2, start_link/2, become_controller/1,
socket_type/0, receive_headers/1, socket_type/0, receive_headers/1, recv_file/2,
transform_listen_option/2, listen_opt_type/1]). transform_listen_option/2, listen_opt_type/1]).
-export([init/2, opt_type/1]). -export([init/2, opt_type/1]).
-include("ejabberd.hrl"). -include("ejabberd.hrl").
-include("logger.hrl"). -include("logger.hrl").
-include("xmpp.hrl"). -include("xmpp.hrl").
-include("ejabberd_http.hrl"). -include("ejabberd_http.hrl").
-include_lib("kernel/include/file.hrl").
-record(state, {sockmod, -record(state, {sockmod,
socket, socket,
@ -50,7 +49,7 @@
request_path, request_path,
request_auth, request_auth,
request_keepalive, request_keepalive,
request_content_length, request_content_length = 0,
request_lang = <<"en">>, request_lang = <<"en">>,
%% XXX bard: request handlers are configured in %% XXX bard: request handlers are configured in
%% ejabberd.cfg under the HTTP service. For example, %% ejabberd.cfg under the HTTP service. For example,
@ -85,6 +84,10 @@
"org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n" "org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n"
"">>). "">>).
-define(RECV_BUF, 65536).
-define(SEND_BUF, 65536).
-define(MAX_POST_SIZE, 20971520). %% 20Mb
start(SockData, Opts) -> start(SockData, Opts) ->
{ok, {ok,
proc_lib:spawn(ejabberd_http, init, proc_lib:spawn(ejabberd_http, init,
@ -113,7 +116,7 @@ init({SockMod, Socket}, Opts) ->
end, end,
TLSOpts = [verify_none | TLSOpts3], TLSOpts = [verify_none | TLSOpts3],
{SockMod1, Socket1} = if TLSEnabled -> {SockMod1, Socket1} = if TLSEnabled ->
inet:setopts(Socket, [{recbuf, 8192}]), inet:setopts(Socket, [{recbuf, ?RECV_BUF}]),
{ok, TLSSocket} = fast_tls:tcp_to_tls(Socket, {ok, TLSSocket} = fast_tls:tcp_to_tls(Socket,
TLSOpts), TLSOpts),
{fast_tls, TLSSocket}; {fast_tls, TLSSocket};
@ -168,18 +171,44 @@ become_controller(_Pid) ->
socket_type() -> socket_type() ->
raw. raw.
send_text(_State, none) ->
ok;
send_text(State, Text) -> send_text(State, Text) ->
case catch case (State#state.sockmod):send(State#state.socket, Text) of
(State#state.sockmod):send(State#state.socket, Text) ok -> ok;
of {error, timeout} ->
ok -> ok; ?INFO_MSG("Timeout on ~p:send", [State#state.sockmod]),
{error, timeout} -> exit(normal);
?INFO_MSG("Timeout on ~p:send", [State#state.sockmod]), Error ->
exit(normal); ?DEBUG("Error in ~p:send: ~p",
Error -> [State#state.sockmod, Error]),
?DEBUG("Error in ~p:send: ~p", exit(normal)
[State#state.sockmod, Error]), end.
exit(normal)
send_file(State, Fd, Size, FileName) ->
try
case State#state.sockmod of
gen_tcp ->
case file:sendfile(Fd, State#state.socket, 0, Size, []) of
{ok, _} -> ok
end;
_ ->
case file:read(Fd, ?SEND_BUF) of
{ok, Data} ->
send_text(State, Data),
send_file(State, Fd, Size, FileName);
eof ->
ok
end
end
catch _:{case_clause, {error, Why}} ->
if Why /= closed ->
?INFO_MSG("Failed to read ~s: ~s",
[FileName, file_format_error(Why)]),
exit(normal);
true ->
ok
end
end. end.
receive_headers(#state{trail = Trail} = State) -> receive_headers(#state{trail = Trail} = State) ->
@ -348,8 +377,8 @@ get_transfer_protocol(RE, SockMod, HostPort) ->
%% matches the requested URL path, and pass control to it. If none is %% matches the requested URL path, and pass control to it. If none is
%% found, answer with HTTP 404. %% found, answer with HTTP 404.
process([], _, _, _, _) -> ejabberd_web:error(not_found); process([], _) -> ejabberd_web:error(not_found);
process(Handlers, Request, Socket, SockMod, Trail) -> process(Handlers, Request) ->
{HandlerPathPrefix, HandlerModule, HandlerOpts, HandlersLeft} = {HandlerPathPrefix, HandlerModule, HandlerOpts, HandlersLeft} =
case Handlers of case Handlers of
[{Pfx, Mod} | Tail] -> [{Pfx, Mod} | Tail] ->
@ -369,14 +398,14 @@ process(Handlers, Request, Socket, SockMod, Trail) ->
LocalPath = lists:nthtail(length(HandlerPathPrefix), Request#request.path), LocalPath = lists:nthtail(length(HandlerPathPrefix), Request#request.path),
R = try R = try
HandlerModule:socket_handoff( HandlerModule:socket_handoff(
LocalPath, Request, Socket, SockMod, Trail, HandlerOpts) LocalPath, Request, HandlerOpts)
catch error:undef -> catch error:undef ->
HandlerModule:process(LocalPath, Request) HandlerModule:process(LocalPath, Request)
end, end,
ejabberd_hooks:run(http_request_debug, [{LocalPath, Request}]), ejabberd_hooks:run(http_request_debug, [{LocalPath, Request}]),
R; R;
false -> false ->
process(HandlersLeft, Request, Socket, SockMod, Trail) process(HandlersLeft, Request)
end. end.
extract_path_query(#state{request_method = Method, extract_path_query(#state{request_method = Method,
@ -398,24 +427,29 @@ extract_path_query(#state{request_method = Method,
extract_path_query(#state{request_method = Method, extract_path_query(#state{request_method = Method,
request_path = {abs_path, Path}, request_path = {abs_path, Path},
request_content_length = Len, request_content_length = Len,
trail = Trail,
sockmod = _SockMod, sockmod = _SockMod,
socket = _Socket} = State) socket = _Socket} = State)
when (Method =:= 'POST' orelse Method =:= 'PUT') andalso when (Method =:= 'POST' orelse Method =:= 'PUT') andalso Len>0 ->
is_integer(Len) ->
case recv_data(State, Len) of
error -> {State, false};
{NewState, Data} ->
?DEBUG("client data: ~p~n", [Data]),
case catch url_decode_q_split(Path) of case catch url_decode_q_split(Path) of
{'EXIT', _} -> {NewState, false}; {'EXIT', _} -> {State, false};
{NPath, _Query} -> {NPath, _Query} ->
LPath = normalize_path([NPE LPath = normalize_path(
|| NPE <- str:tokens(path_decode(NPath), <<"/">>)]), [NPE || NPE <- str:tokens(path_decode(NPath), <<"/">>)]),
LQuery = case catch parse_urlencoded(Data) of case Method of
{'EXIT', _Reason} -> []; 'PUT' ->
LQ -> LQ {State, {LPath, [], Trail}};
end, 'POST' ->
{NewState, {LPath, LQuery, Data}} case recv_data(State) of
{ok, Data} ->
LQuery = case catch parse_urlencoded(Data) of
{'EXIT', _Reason} -> [];
LQ -> LQ
end,
{State, {LPath, LQuery, Data}};
error ->
{State, false}
end
end end
end; end;
extract_path_query(State) -> extract_path_query(State) ->
@ -434,10 +468,10 @@ process_request(#state{request_method = Method,
request_host = Host, request_host = Host,
request_port = Port, request_port = Port,
request_tp = TP, request_tp = TP,
request_content_length = Length,
request_headers = RequestHeaders, request_headers = RequestHeaders,
request_handlers = RequestHandlers, request_handlers = RequestHandlers,
custom_headers = CustomHeaders, custom_headers = CustomHeaders} = State) ->
trail = Trail} = State) ->
case extract_path_query(State) of case extract_path_query(State) of
{State2, false} -> {State2, false} ->
{State2, make_bad_request(State)}; {State2, make_bad_request(State)};
@ -459,7 +493,10 @@ process_request(#state{request_method = Method,
path = LPath, path = LPath,
q = LQuery, q = LQuery,
auth = Auth, auth = Auth,
data = Data, length = Length,
sockmod = SockMod,
socket = Socket,
data = Data,
lang = Lang, lang = Lang,
host = Host, host = Host,
port = Port, port = Port,
@ -469,7 +506,7 @@ process_request(#state{request_method = Method,
ip = IP}, ip = IP},
RequestHandlers1 = ejabberd_hooks:run_fold( RequestHandlers1 = ejabberd_hooks:run_fold(
http_request_handlers, RequestHandlers, [Host, Request]), http_request_handlers, RequestHandlers, [Host, Request]),
Res = case process(RequestHandlers1, Request, Socket, SockMod, Trail) of Res = case process(RequestHandlers1, Request) of
El when is_record(El, xmlel) -> El when is_record(El, xmlel) ->
make_xhtml_output(State, 200, CustomHeaders, El); make_xhtml_output(State, 200, CustomHeaders, El);
{Status, Headers, El} {Status, Headers, El}
@ -482,6 +519,8 @@ process_request(#state{request_method = Method,
when is_binary(Output) or is_list(Output) -> when is_binary(Output) or is_list(Output) ->
make_text_output(State, Status, make_text_output(State, Status,
Headers ++ CustomHeaders, Output); Headers ++ CustomHeaders, Output);
{Status, Headers, {file, FileName}} ->
make_file_output(State, Status, Headers, FileName);
{Status, Reason, Headers, Output} {Status, Reason, Headers, Output}
when is_binary(Output) or is_list(Output) -> when is_binary(Output) or is_list(Output) ->
make_text_output(State, Status, Reason, make_text_output(State, Status, Reason,
@ -535,114 +574,80 @@ is_ipchain_trusted(UserIPs, Masks) ->
end end
end, UserIPs). end, UserIPs).
recv_data(State, Len) -> recv_data(State, Len, <<>>). recv_data(#state{request_content_length = Len}) when Len >= ?MAX_POST_SIZE ->
error;
recv_data(State, 0, Acc) -> {State, Acc}; recv_data(#state{request_content_length = Len, trail = Trail,
recv_data(#state{trail = Trail} = State, Len, <<>>) when byte_size(Trail) > Len -> sockmod = SockMod, socket = Socket}) ->
<<Data:Len/binary, Rest/binary>> = Trail, NewLen = Len - byte_size(Trail),
{State#state{trail = Rest}, Data}; if NewLen > 0 ->
recv_data(State, Len, Acc) -> case SockMod:recv(Socket, NewLen, 60000) of
case State#state.trail of {ok, Data} -> {ok, <<Trail/binary, Data/binary>>};
<<>> -> {error, _} -> error
case (State#state.sockmod):recv(State#state.socket,
min(Len, 16#4000000), 300000)
of
{ok, Data} ->
recv_data(State, Len - byte_size(Data), <<Acc/binary, Data/binary>>);
Err ->
?DEBUG("Cannot receive HTTP data: ~p", [Err]),
error
end; end;
_ -> true ->
Trail = (State#state.trail), {ok, Trail}
recv_data(State#state{trail = <<>>},
Len - byte_size(Trail), <<Acc/binary, Trail/binary>>)
end. end.
make_xhtml_output(State, Status, Headers, XHTML) -> recv_file(#request{length = Len, data = Trail,
Data = case lists:member(html, Headers) of sockmod = SockMod, socket = Socket}, Path) ->
true -> case file:open(Path, [write, exclusive, raw]) of
iolist_to_binary([?HTML_DOCTYPE, {ok, Fd} ->
fxml:element_to_binary(XHTML)]); case file:write(Fd, Trail) of
_ -> ok ->
iolist_to_binary([?XHTML_DOCTYPE, NewLen = max(0, Len - byte_size(Trail)),
fxml:element_to_binary(XHTML)]) case do_recv_file(NewLen, SockMod, Socket, Fd) of
end, ok ->
Headers1 = case lists:keysearch(<<"Content-Type">>, 1, ok;
Headers) {error, _} = Err ->
of file:delete(Path),
{value, _} -> Err
[{<<"Content-Length">>, end;
integer_to_binary(byte_size(Data))} {error, _} = Err ->
| Headers]; file:delete(Path),
_ -> Err
[{<<"Content-Type">>, <<"text/html; charset=utf-8">>}, end;
{<<"Content-Length">>, {error, _} = Err ->
integer_to_binary(byte_size(Data))} Err
| Headers] end.
do_recv_file(0, _SockMod, _Socket, Fd) ->
file:close(Fd);
do_recv_file(Len, SockMod, Socket, Fd) ->
ChunkLen = min(Len, ?RECV_BUF),
try
{ok, Data} = SockMod:recv(Socket, ChunkLen, timer:seconds(30)),
ok = file:write(Fd, Data),
do_recv_file(Len-ChunkLen, SockMod, Socket, Fd)
catch _:{badmatch, {error, _} = Err} ->
file:close(Fd),
Err
end.
make_headers(State, Status, Reason, Headers, Data) ->
Len = if is_integer(Data) -> Data;
true -> iolist_size(Data)
end,
Headers1 = [{<<"Content-Length">>, integer_to_binary(Len)} | Headers],
Headers2 = case lists:keyfind(<<"Content-Type">>, 1, Headers) of
{_, _} ->
Headers1;
false ->
[{<<"Content-Type">>, <<"text/html; charset=utf-8">>}
| Headers1]
end, end,
HeadersOut = case {State#state.request_version, HeadersOut = case {State#state.request_version,
State#state.request_keepalive} State#state.request_keepalive} of
of {{1, 1}, true} -> Headers2;
{{1, 1}, true} -> Headers1; {_, true} ->
{_, true} -> [{<<"Connection">>, <<"keep-alive">>} | Headers2];
[{<<"Connection">>, <<"keep-alive">>} | Headers1]; {_, false} ->
{_, false} -> [{<<"Connection">>, <<"close">>} | Headers2]
[{<<"Connection">>, <<"close">>} | Headers1]
end, end,
Version = case State#state.request_version of Version = case State#state.request_version of
{1, 1} -> <<"HTTP/1.1 ">>; {1, 1} -> <<"HTTP/1.1 ">>;
_ -> <<"HTTP/1.0 ">> _ -> <<"HTTP/1.0 ">>
end, end,
H = lists:map(fun ({Attr, Val}) -> H = [[Attr, <<": ">>, Val, <<"\r\n">>] || {Attr, Val} <- HeadersOut],
[Attr, <<": ">>, Val, <<"\r\n">>];
(_) -> []
end,
HeadersOut),
SL = [Version,
integer_to_binary(Status), <<" ">>,
code_to_phrase(Status), <<"\r\n">>],
Data2 = case State#state.request_method of
'HEAD' -> <<"">>;
_ -> Data
end,
[SL, H, <<"\r\n">>, Data2].
make_text_output(State, Status, Headers, Text) ->
make_text_output(State, Status, <<"">>, Headers, Text).
make_text_output(State, Status, Reason, Headers, Text) ->
Data = iolist_to_binary(Text),
Headers1 = case lists:keysearch(<<"Content-Type">>, 1,
Headers)
of
{value, _} ->
[{<<"Content-Length">>,
integer_to_binary(byte_size(Data))}
| Headers];
_ ->
[{<<"Content-Type">>, <<"text/html; charset=utf-8">>},
{<<"Content-Length">>,
integer_to_binary(byte_size(Data))}
| Headers]
end,
HeadersOut = case {State#state.request_version,
State#state.request_keepalive}
of
{{1, 1}, true} -> Headers1;
{_, true} ->
[{<<"Connection">>, <<"keep-alive">>} | Headers1];
{_, false} ->
[{<<"Connection">>, <<"close">>} | Headers1]
end,
Version = case State#state.request_version of
{1, 1} -> <<"HTTP/1.1 ">>;
_ -> <<"HTTP/1.0 ">>
end,
H = lists:map(fun ({Attr, Val}) ->
[Attr, <<": ">>, Val, <<"\r\n">>]
end,
HeadersOut),
NewReason = case Reason of NewReason = case Reason of
<<"">> -> code_to_phrase(Status); <<"">> -> code_to_phrase(Status);
_ -> Reason _ -> Reason
@ -650,11 +655,55 @@ make_text_output(State, Status, Reason, Headers, Text) ->
SL = [Version, SL = [Version,
integer_to_binary(Status), <<" ">>, integer_to_binary(Status), <<" ">>,
NewReason, <<"\r\n">>], NewReason, <<"\r\n">>],
[SL, H, <<"\r\n">>].
make_xhtml_output(State, Status, Headers, XHTML) ->
Data = case State#state.request_method of
'HEAD' -> <<"">>;
_ ->
DocType = case lists:member(html, Headers) of
true -> ?HTML_DOCTYPE;
false -> ?XHTML_DOCTYPE
end,
iolist_to_binary([DocType, fxml:element_to_binary(XHTML)])
end,
EncodedHdrs = make_headers(State, Status, <<"">>, Headers, Data),
[EncodedHdrs, Data].
make_text_output(State, Status, Headers, Text) ->
make_text_output(State, Status, <<"">>, Headers, Text).
make_text_output(State, Status, Reason, Headers, Text) ->
Data = iolist_to_binary(Text),
Data2 = case State#state.request_method of Data2 = case State#state.request_method of
'HEAD' -> <<"">>; 'HEAD' -> <<"">>;
_ -> Data _ -> Data
end, end,
[SL, H, <<"\r\n">>, Data2]. EncodedHdrs = make_headers(State, Status, Reason, Headers, Data2),
[EncodedHdrs, Data2].
make_file_output(State, Status, Headers, FileName) ->
case file:read_file_info(FileName) of
{ok, #file_info{size = Size}} when State#state.request_method == 'HEAD' ->
make_headers(State, Status, <<"">>, Headers, Size);
{ok, #file_info{size = Size}} ->
case file:open(FileName, [raw, read]) of
{ok, Fd} ->
EncodedHdrs = make_headers(State, Status, <<"">>, Headers, Size),
send_text(State, EncodedHdrs),
send_file(State, Fd, Size, FileName),
file:close(Fd),
none;
{error, Why} ->
Reason = file_format_error(Why),
?ERROR_MSG("Failed to open ~s: ~s", [FileName, Reason]),
make_text_output(State, 404, Reason, [], <<>>)
end;
{error, Why} ->
Reason = file_format_error(Why),
?ERROR_MSG("Failed to read info of ~s: ~s", [FileName, Reason]),
make_text_output(State, 404, Reason, [], <<>>)
end.
parse_lang(Langs) -> parse_lang(Langs) ->
case str:tokens(Langs, <<",; ">>) of case str:tokens(Langs, <<",; ">>) of
@ -662,6 +711,12 @@ parse_lang(Langs) ->
[] -> <<"en">> [] -> <<"en">>
end. end.
file_format_error(Reason) ->
case file:format_error(Reason) of
"unknown POSIX error" -> atom_to_list(Reason);
Text -> Text
end.
% Code below is taken (with some modifications) from the yaws webserver, which % Code below is taken (with some modifications) from the yaws webserver, which
% is distributed under the following license: % is distributed under the following license:
% %

7
src/ejabberd_http_ws.erl
View File

@ -35,7 +35,7 @@
terminate/3, send_xml/2, setopts/2, sockname/1, terminate/3, send_xml/2, setopts/2, sockname/1,
peername/1, controlling_process/2, become_controller/2, peername/1, controlling_process/2, become_controller/2,
monitor/1, reset_stream/1, close/1, change_shaper/2, monitor/1, reset_stream/1, close/1, change_shaper/2,
socket_handoff/6, opt_type/1]). socket_handoff/3, opt_type/1]).
-include("ejabberd.hrl"). -include("ejabberd.hrl").
-include("logger.hrl"). -include("logger.hrl").
@ -121,9 +121,8 @@ change_shaper({http_ws, _FsmRef, _IP}, _Shaper) ->
%% TODO??? %% TODO???
ok. ok.
socket_handoff(LocalPath, Request, Socket, SockMod, Buf, Opts) -> socket_handoff(LocalPath, Request, Opts) ->
ejabberd_websocket:socket_handoff(LocalPath, Request, Socket, SockMod, ejabberd_websocket:socket_handoff(LocalPath, Request, Opts, ?MODULE, fun get_human_html_xmlel/0).
Buf, Opts, ?MODULE, fun get_human_html_xmlel/0).
%%% Internal %%% Internal

13
src/ejabberd_websocket.erl
View File

@ -42,7 +42,7 @@
-author('ecestari@process-one.net'). -author('ecestari@process-one.net').
-export([check/2, socket_handoff/8]). -export([check/2, socket_handoff/5]).
-include("ejabberd.hrl"). -include("ejabberd.hrl").
-include("logger.hrl"). -include("logger.hrl").
@ -89,8 +89,9 @@ check(_Path, Headers) ->
socket_handoff(LocalPath, #request{method = 'GET', ip = IP, q = Q, path = Path, socket_handoff(LocalPath, #request{method = 'GET', ip = IP, q = Q, path = Path,
headers = Headers, host = Host, port = Port, headers = Headers, host = Host, port = Port,
opts = HOpts}, socket = Socket, sockmod = SockMod,
Socket, SockMod, Buf, _Opts, HandlerModule, InfoMsgFun) -> data = Buf, opts = HOpts},
_Opts, HandlerModule, InfoMsgFun) ->
case check(LocalPath, Headers) of case check(LocalPath, Headers) of
true -> true ->
WS = #ws{socket = Socket, WS = #ws{socket = Socket,
@ -109,11 +110,11 @@ socket_handoff(LocalPath, #request{method = 'GET', ip = IP, q = Q, path = Path,
_ -> _ ->
{200, ?HEADER, InfoMsgFun()} {200, ?HEADER, InfoMsgFun()}
end; end;
socket_handoff(_, #request{method = 'OPTIONS'}, _, _, _, _, _, _) -> socket_handoff(_, #request{method = 'OPTIONS'}, _, _, _) ->
{200, ?OPTIONS_HEADER, []}; {200, ?OPTIONS_HEADER, []};
socket_handoff(_, #request{method = 'HEAD'}, _, _, _, _, _, _) -> socket_handoff(_, #request{method = 'HEAD'}, _, _, _) ->
{200, ?HEADER, []}; {200, ?HEADER, []};
socket_handoff(_, _, _, _, _, _, _, _) -> socket_handoff(_, _, _, _, _) ->
{400, ?HEADER, #xmlel{name = <<"h1">>, {400, ?HEADER, #xmlel{name = <<"h1">>,
children = [{xmlcdata, <<"400 Bad Request">>}]}}. children = [{xmlcdata, <<"400 Bad Request">>}]}}.

3
src/mod_http_fileserver.erl
View File

@ -351,13 +351,12 @@ serve_file(FileInfo, FileName, CustomHeaders, DefaultContentType, ContentTypes)
?DEBUG("Delivering: ~s", [FileName]), ?DEBUG("Delivering: ~s", [FileName]),
ContentType = content_type(FileName, DefaultContentType, ContentType = content_type(FileName, DefaultContentType,
ContentTypes), ContentTypes),
{ok, FileContents} = file:read_file(FileName),
{FileInfo#file_info.size, 200, {FileInfo#file_info.size, 200,
[{<<"Server">>, <<"ejabberd">>}, [{<<"Server">>, <<"ejabberd">>},
{<<"Last-Modified">>, last_modified(FileInfo)}, {<<"Last-Modified">>, last_modified(FileInfo)},
{<<"Content-Type">>, ContentType} {<<"Content-Type">>, ContentType}
| CustomHeaders], | CustomHeaders],
FileContents}. {file, FileName}}.
%%---------------------------------------------------------------------- %%----------------------------------------------------------------------
%% Log file %% Log file

98
src/mod_http_upload.erl
View File

@ -377,13 +377,13 @@ process(LocalPath, #request{method = Method, host = Host, ip = IP})
[Method, ?ADDR_TO_STR(IP), Host]), [Method, ?ADDR_TO_STR(IP), Host]),
http_response(404); http_response(404);
process(_LocalPath, #request{method = 'PUT', host = Host, ip = IP, process(_LocalPath, #request{method = 'PUT', host = Host, ip = IP,
data = Data} = Request) -> length = Length} = Request) ->
{Proc, Slot} = parse_http_request(Request), {Proc, Slot} = parse_http_request(Request),
case catch gen_server:call(Proc, {use_slot, Slot, byte_size(Data)}) of case catch gen_server:call(Proc, {use_slot, Slot, Length}) of
{ok, Path, FileMode, DirMode, GetPrefix, Thumbnail, CustomHeaders} -> {ok, Path, FileMode, DirMode, GetPrefix, Thumbnail, CustomHeaders} ->
?DEBUG("Storing file from ~s for ~s: ~s", ?DEBUG("Storing file from ~s for ~s: ~s",
[?ADDR_TO_STR(IP), Host, Path]), [?ADDR_TO_STR(IP), Host, Path]),
case store_file(Path, Data, FileMode, DirMode, case store_file(Path, Request, FileMode, DirMode,
GetPrefix, Slot, Thumbnail) of GetPrefix, Slot, Thumbnail) of
ok -> ok ->
http_response(201, CustomHeaders); http_response(201, CustomHeaders);
@ -396,7 +396,7 @@ process(_LocalPath, #request{method = 'PUT', host = Host, ip = IP,
end; end;
{error, size_mismatch} -> {error, size_mismatch} ->
?INFO_MSG("Rejecting file from ~s for ~s: Unexpected size (~B)", ?INFO_MSG("Rejecting file from ~s for ~s: Unexpected size (~B)",
[?ADDR_TO_STR(IP), Host, byte_size(Data)]), [?ADDR_TO_STR(IP), Host, Length]),
http_response(413); http_response(413);
{error, invalid_slot} -> {error, invalid_slot} ->
?INFO_MSG("Rejecting file from ~s for ~s: Invalid slot", ?INFO_MSG("Rejecting file from ~s for ~s: Invalid slot",
@ -414,8 +414,9 @@ process(_LocalPath, #request{method = Method, host = Host, ip = IP} = Request)
case catch gen_server:call(Proc, get_conf) of case catch gen_server:call(Proc, get_conf) of
{ok, DocRoot, CustomHeaders} -> {ok, DocRoot, CustomHeaders} ->
Path = str:join([DocRoot | Slot], <<$/>>), Path = str:join([DocRoot | Slot], <<$/>>),
case file:read_file(Path) of case file:read(Path, [read]) of
{ok, Data} -> {ok, Fd} ->
file:close(Fd),
?INFO_MSG("Serving ~s to ~s", [Path, ?ADDR_TO_STR(IP)]), ?INFO_MSG("Serving ~s to ~s", [Path, ?ADDR_TO_STR(IP)]),
ContentType = guess_content_type(FileName), ContentType = guess_content_type(FileName),
Headers1 = case ContentType of Headers1 = case ContentType of
@ -428,7 +429,7 @@ process(_LocalPath, #request{method = Method, host = Host, ip = IP} = Request)
end, end,
Headers2 = [{<<"Content-Type">>, ContentType} | Headers1], Headers2 = [{<<"Content-Type">>, ContentType} | Headers1],
Headers3 = Headers2 ++ CustomHeaders, Headers3 = Headers2 ++ CustomHeaders,
http_response(200, Headers3, Data); http_response(200, Headers3, {file, Path});
{error, eacces} -> {error, eacces} ->
?INFO_MSG("Cannot serve ~s to ~s: Permission denied", ?INFO_MSG("Cannot serve ~s to ~s: Permission denied",
[Path, ?ADDR_TO_STR(IP)]), [Path, ?ADDR_TO_STR(IP)]),
@ -720,17 +721,17 @@ parse_http_request(#request{host = Host, path = Path}) ->
end, end,
{gen_mod:get_module_proc(ProcURL, ?MODULE), Slot}. {gen_mod:get_module_proc(ProcURL, ?MODULE), Slot}.
-spec store_file(binary(), binary(), -spec store_file(binary(), http_request(),
integer() | undefined, integer() | undefined,
integer() | undefined, integer() | undefined,
binary(), slot(), boolean()) binary(), slot(), boolean())
-> ok | {ok, [{binary(), binary()}], binary()} | {error, term()}. -> ok | {ok, [{binary(), binary()}], binary()} | {error, term()}.
store_file(Path, Data, FileMode, DirMode, GetPrefix, Slot, Thumbnail) -> store_file(Path, Request, FileMode, DirMode, GetPrefix, Slot, Thumbnail) ->
case do_store_file(Path, Data, FileMode, DirMode) of case do_store_file(Path, Request, FileMode, DirMode) of
ok when Thumbnail -> ok when Thumbnail ->
case identify(Path, Data) of case identify(Path) of
{ok, MediaInfo} -> {ok, MediaInfo} ->
case convert(Path, Data, MediaInfo) of case convert(Path, MediaInfo) of
{ok, OutPath, OutMediaInfo} -> {ok, OutPath, OutMediaInfo} ->
[UserDir, RandDir | _] = Slot, [UserDir, RandDir | _] = Slot,
FileName = filename:basename(OutPath), FileName = filename:basename(OutPath),
@ -753,16 +754,14 @@ store_file(Path, Data, FileMode, DirMode, GetPrefix, Slot, Thumbnail) ->
Err Err
end. end.
-spec do_store_file(file:filename_all(), binary(), -spec do_store_file(file:filename_all(), http_request(),
integer() | undefined, integer() | undefined,
integer() | undefined) integer() | undefined)
-> ok | {error, term()}. -> ok | {error, term()}.
do_store_file(Path, Data, FileMode, DirMode) -> do_store_file(Path, Request, FileMode, DirMode) ->
try try
ok = filelib:ensure_dir(Path), ok = filelib:ensure_dir(Path),
{ok, Io} = file:open(Path, [write, exclusive, raw]), ok = ejabberd_http:recv_file(Request, Path),
Ok = file:write(Io, Data),
ok = file:close(Io),
if is_integer(FileMode) -> if is_integer(FileMode) ->
ok = file:change_mode(Path, FileMode); ok = file:change_mode(Path, FileMode);
FileMode == undefined -> FileMode == undefined ->
@ -775,8 +774,7 @@ do_store_file(Path, Data, FileMode, DirMode) ->
ok = file:change_mode(UserDir, DirMode); ok = file:change_mode(UserDir, DirMode);
DirMode == undefined -> DirMode == undefined ->
ok ok
end, end
ok = Ok % Raise an exception if file:write/2 failed.
catch catch
_:{badmatch, {error, Error}} -> _:{badmatch, {error, Error}} ->
{error, Error}; {error, Error};
@ -801,7 +799,8 @@ http_response(Code, ExtraHeaders) ->
Message = <<(code_to_message(Code))/binary, $\n>>, Message = <<(code_to_message(Code))/binary, $\n>>,
http_response(Code, ExtraHeaders, Message). http_response(Code, ExtraHeaders, Message).
-spec http_response(100..599, [{binary(), binary()}], binary()) -type http_body() :: binary() | {file, file:filename()}.
-spec http_response(100..599, [{binary(), binary()}], http_body())
-> {pos_integer(), [{binary(), binary()}], binary()}. -> {pos_integer(), [{binary(), binary()}], binary()}.
http_response(Code, ExtraHeaders, Body) -> http_response(Code, ExtraHeaders, Body) ->
Headers = case proplists:is_defined(<<"Content-Type">>, ExtraHeaders) of Headers = case proplists:is_defined(<<"Content-Type">>, ExtraHeaders) of
@ -824,22 +823,30 @@ code_to_message(_Code) -> <<"">>.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
%% Image manipulation stuff. %% Image manipulation stuff.
%%-------------------------------------------------------------------- %%--------------------------------------------------------------------
-spec identify(binary(), binary()) -> {ok, media_info()} | pass. -spec identify(binary()) -> {ok, media_info()} | pass.
identify(Path, Data) -> identify(Path) ->
case eimp:identify(Data) of try
{ok, Info} -> {ok, Fd} = file:open(Path, [read, raw]),
{ok, #media_info{ {ok, Data} = file:read(Fd, 1024),
type = proplists:get_value(type, Info), case eimp:identify(Data) of
width = proplists:get_value(width, Info), {ok, Info} ->
height = proplists:get_value(height, Info)}}; {ok, #media_info{
{error, Why} -> type = proplists:get_value(type, Info),
?DEBUG("Cannot identify type of ~s: ~s", width = proplists:get_value(width, Info),
[Path, eimp:format_error(Why)]), height = proplists:get_value(height, Info)}};
{error, Why} ->
?DEBUG("Cannot identify type of ~s: ~s",
[Path, eimp:format_error(Why)]),
pass
end
catch _:{badmatch, {error, Reason}} ->
?DEBUG("Failed to read file ~s: ~s",
[Path, file:format_error(Reason)]),
pass pass
end. end.
-spec convert(binary(), binary(), media_info()) -> {ok, binary(), media_info()} | pass. -spec convert(binary(), media_info()) -> {ok, binary(), media_info()} | pass.
convert(Path, Data, #media_info{type = T, width = W, height = H} = Info) -> convert(Path, #media_info{type = T, width = W, height = H} = Info) ->
if W * H >= 25000000 -> if W * H >= 25000000 ->
?DEBUG("The image ~s is more than 25 Mpix", [Path]), ?DEBUG("The image ~s is more than 25 Mpix", [Path]),
pass; pass;
@ -855,19 +862,26 @@ convert(Path, Data, #media_info{type = T, width = W, height = H} = Info) ->
true -> {300, 300} true -> {300, 300}
end, end,
OutInfo = #media_info{type = T, width = W1, height = H1}, OutInfo = #media_info{type = T, width = W1, height = H1},
case eimp:convert(Data, T, [{scale, {W1, H1}}]) of case file:read_file(Path) of
{ok, OutData} -> {ok, Data} ->
case file:write_file(OutPath, OutData) of case eimp:convert(Data, T, [{scale, {W1, H1}}]) of
ok -> {ok, OutData} ->
{ok, OutPath, OutInfo}; case file:write_file(OutPath, OutData) of
ok ->
{ok, OutPath, OutInfo};
{error, Why} ->
?ERROR_MSG("Failed to write to ~s: ~s",
[OutPath, file:format_error(Why)]),
pass
end;
{error, Why} -> {error, Why} ->
?ERROR_MSG("Failed to write to ~s: ~s", ?ERROR_MSG("Failed to convert ~s to ~s: ~s",
[OutPath, file:format_error(Why)]), [Path, OutPath, eimp:format_error(Why)]),
pass pass
end; end;
{error, Why} -> {error, Why} ->
?ERROR_MSG("Failed to convert ~s to ~s: ~s", ?ERROR_MSG("Failed to read file ~s: ~s",
[Path, OutPath, eimp:format_error(Why)]), [Path, file:format_error(Why)]),
pass pass
end end
end. end.