From 5dfd95e5a7d9194dd882a1dbccd3d42f5fed005f Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Mon, 26 Oct 2015 13:10:10 +0100 Subject: [PATCH 01/13] Support user quotas for HTTP File Upload mod_http_upload_quota implements two features: - When a "hard quota" is exceeded during a file upload, old files are removed until the disk usage equals or falls below the "soft quota". - Once a day, all uploaded files (and directories) older than a configurable number of days are deleted. --- src/mod_http_upload_quota.erl | 345 ++++++++++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 src/mod_http_upload_quota.erl diff --git a/src/mod_http_upload_quota.erl b/src/mod_http_upload_quota.erl new file mode 100644 index 000000000..1b7828a54 --- /dev/null +++ b/src/mod_http_upload_quota.erl @@ -0,0 +1,345 @@ +%%%------------------------------------------------------------------- +%%% File : mod_http_upload_quota.erl +%%% Author : Holger Weiss +%%% Purpose : Quota management for HTTP File Upload (XEP-0363) +%%% Created : 15 Oct 2015 by Holger Weiss +%%%------------------------------------------------------------------- + +-module(mod_http_upload_quota). +-author('holger@zedat.fu-berlin.de'). + +-define(GEN_SERVER, gen_server). +-define(PROCNAME, ?MODULE). +-define(TIMEOUT, timer:hours(24)). +-define(INITIAL_TIMEOUT, timer:minutes(10)). +-define(FORMAT(Error), file:format_error(Error)). + +-behaviour(?GEN_SERVER). +-behaviour(gen_mod). + +%% gen_mod/supervisor callbacks. +-export([start_link/3, + start/2, + stop/1, + mod_opt_type/1]). + +%% gen_server callbacks. +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +%% ejabberd_hooks callback. +-export([handle_slot_request/5]). + +-include("jlib.hrl"). +-include("logger.hrl"). +-include_lib("kernel/include/file.hrl"). + +-record(state, + {server_host :: binary(), + access_soft_quota :: atom(), + access_hard_quota :: atom(), + max_days :: pos_integer() | infinity, + docroot :: binary(), + disk_usage = dict:new() :: term(), + timers :: [timer:tref()]}). + +-type state() :: #state{}. + +%%-------------------------------------------------------------------- +%% gen_mod/supervisor callbacks. +%%-------------------------------------------------------------------- + +-spec start_link(binary(), atom(), gen_mod:opts()) + -> {ok, pid()} | ignore | {error, _}. + +start_link(ServerHost, Proc, Opts) -> + ?GEN_SERVER:start_link({local, Proc}, ?MODULE, {ServerHost, Opts}, []). + +-spec start(binary(), gen_mod:opts()) -> {ok, _} | {ok, _, _} | {error, _}. + +start(ServerHost, Opts) -> + Proc = mod_http_upload:get_proc_name(ServerHost, ?PROCNAME), + Spec = {Proc, + {?MODULE, start_link, [ServerHost, Proc, Opts]}, + permanent, + 3000, + worker, + [?MODULE]}, + supervisor:start_child(ejabberd_sup, Spec). + +-spec stop(binary()) -> ok. + +stop(ServerHost) -> + Proc = mod_http_upload:get_proc_name(ServerHost, ?PROCNAME), + ok = supervisor:terminate_child(ejabberd_sup, Proc), + ok = supervisor:delete_child(ejabberd_sup, Proc). + +-spec mod_opt_type(atom()) -> fun((term()) -> term()) | [atom()]. + +mod_opt_type(access_soft_quota) -> + fun(A) when is_atom(A) -> A end; +mod_opt_type(access_hard_quota) -> + fun(A) when is_atom(A) -> A end; +mod_opt_type(max_days) -> + fun(I) when is_integer(I), I > 0 -> I; + (infinity) -> infinity + end; +mod_opt_type(_) -> + [access_soft_quota, access_hard_quota, max_days]. + +%%-------------------------------------------------------------------- +%% gen_server callbacks. +%%-------------------------------------------------------------------- + +-spec init({binary(), gen_mod:opts()}) -> {ok, state()}. + +init({ServerHost, Opts}) -> + process_flag(trap_exit, true), + AccessSoftQuota = gen_mod:get_opt(access_soft_quota, Opts, + fun(A) when is_atom(A) -> A end, + soft_upload_quota), + AccessHardQuota = gen_mod:get_opt(access_hard_quota, Opts, + fun(A) when is_atom(A) -> A end, + hard_upload_quota), + MaxDays = gen_mod:get_opt(max_days, Opts, + fun(I) when is_integer(I), I > 0 -> I; + (infinity) -> infinity + end, + infinity), + DocRoot1 = gen_mod:get_module_opt(ServerHost, mod_http_upload, docroot, + fun iolist_to_binary/1, + <<"@HOME@/upload">>), + DocRoot2 = mod_http_upload:expand_home(str:strip(DocRoot1, right, $/)), + Timers = if MaxDays == infinity -> []; + true -> + {ok, T1} = timer:send_after(?INITIAL_TIMEOUT, sweep), + {ok, T2} = timer:send_interval(?TIMEOUT, sweep), + [T1, T2] + end, + ejabberd_hooks:add(http_upload_slot_request, ServerHost, ?MODULE, + handle_slot_request, 50), + {ok, #state{server_host = ServerHost, + access_soft_quota = AccessSoftQuota, + access_hard_quota = AccessHardQuota, + max_days = MaxDays, + docroot = DocRoot2, + timers = Timers}}. + +-spec handle_call(_, {pid(), _}, state()) -> {noreply, state()}. + +handle_call(Request, From, State) -> + ?ERROR_MSG("Got unexpected request from ~p: ~p", [From, Request]), + {noreply, State}. + +-spec handle_cast(_, state()) -> {noreply, state()}. + +handle_cast({handle_slot_request, #jid{user = U, server = S} = JID, Path, Size}, + #state{server_host = ServerHost, + access_soft_quota = AccessSoftQuota, + access_hard_quota = AccessHardQuota, + disk_usage = DiskUsage} = State) -> + HardQuota = case acl:match_rule(ServerHost, AccessHardQuota, JID) of + Hard when is_integer(Hard), Hard > 0 -> + Hard * 1024 * 1024; + _ -> + 0 + end, + SoftQuota = case acl:match_rule(ServerHost, AccessSoftQuota, JID) of + Soft when is_integer(Soft), Soft > 0 -> + Soft * 1024 * 1024; + _ -> + 0 + end, + OldSize = case dict:find({U, S}, DiskUsage) of + {ok, Value} -> + Value; + error -> + undefined + end, + NewSize = case {HardQuota, SoftQuota} of + {0, 0} -> + ?DEBUG("No quota specified for ~s", + [jlib:jid_to_string(JID)]), + Size; + {0, _} -> + ?WARNING_MSG("No hard quota specified for ~s", + [jlib:jid_to_string(JID)]), + enforce_quota(Path, Size, OldSize, SoftQuota, SoftQuota); + {_, 0} -> + ?WARNING_MSG("No soft quota specified for ~s", + [jlib:jid_to_string(JID)]), + enforce_quota(Path, Size, OldSize, HardQuota, HardQuota); + _ when SoftQuota > HardQuota -> + ?WARNING_MSG("Bad quota for ~s (soft: ~p, hard: ~p)", + [jlib:jid_to_string(JID), + SoftQuota, HardQuota]), + enforce_quota(Path, Size, OldSize, SoftQuota, SoftQuota); + _ -> + ?DEBUG("Enforcing quota for ~s", + [jlib:jid_to_string(JID)]), + enforce_quota(Path, Size, OldSize, SoftQuota, HardQuota) + end, + {noreply, State#state{disk_usage = dict:store({U, S}, NewSize, DiskUsage)}}; +handle_cast(Request, State) -> + ?ERROR_MSG("Got unexpected request: ~p", [Request]), + {noreply, State}. + +-spec handle_info(_, state()) -> {noreply, state()}. + +handle_info(sweep, #state{server_host = ServerHost, + docroot = DocRoot, + max_days = MaxDays} = State) + when is_integer(MaxDays), MaxDays > 0 -> + ?DEBUG("Got 'sweep' message for ~s", [ServerHost]), + case file:list_dir(DocRoot) of + {ok, Entries} -> + BackThen = secs_since_epoch() - (MaxDays * 86400), + DocRootS = binary_to_list(DocRoot), + PathNames = lists:map(fun(Entry) -> DocRootS ++ "/" ++ Entry end, + Entries), + UserDirs = lists:filter(fun filelib:is_dir/1, PathNames), + lists:foreach(fun(UserDir) -> delete_old_files(UserDir, BackThen) end, + UserDirs); + {error, Error} -> + ?ERROR_MSG("Cannot open document root ~s: ~s", + [DocRoot, ?FORMAT(Error)]) + end, + {noreply, State}; +handle_info(Info, State) -> + ?ERROR_MSG("Got unexpected info: ~p", [Info]), + {noreply, State}. + +-spec terminate(normal | shutdown | {shutdown, _} | _, _) -> ok. + +terminate(Reason, #state{server_host = ServerHost, timers = Timers}) -> + ?DEBUG("Stopping upload quota process for ~s: ~p", [ServerHost, Reason]), + ejabberd_hooks:delete(http_upload_slot_request, ServerHost, ?MODULE, + handle_slot_request, 50), + lists:foreach(fun(Timer) -> timer:cancel(Timer) end, Timers). + +-spec code_change({down, _} | _, state(), _) -> {ok, state()}. + +code_change(_OldVsn, #state{server_host = ServerHost} = State, _Extra) -> + ?DEBUG("Updating upload quota process for ~s", [ServerHost]), + {ok, State}. + +%%-------------------------------------------------------------------- +%% ejabberd_hooks callback. +%%-------------------------------------------------------------------- + +-spec handle_slot_request(term(), jid(), binary(), non_neg_integer(), binary()) + -> term(). + +handle_slot_request(allow, #jid{lserver = ServerHost} = JID, Path, Size, + _Lang) -> + Proc = mod_http_upload:get_proc_name(ServerHost, ?PROCNAME), + ?GEN_SERVER:cast(Proc, {handle_slot_request, JID, Path, Size}), + allow; +handle_slot_request(Acc, _JID, _Path, _Size, _Lang) -> Acc. + +%%-------------------------------------------------------------------- +%% Internal functions. +%%-------------------------------------------------------------------- + +-spec enforce_quota(file:filename_all(), non_neg_integer(), + non_neg_integer() | undefined, non_neg_integer(), + non_neg_integer()) + -> non_neg_integer(). + +enforce_quota(_UserDir, SlotSize, OldSize, _MinSize, MaxSize) + when is_integer(OldSize), OldSize + SlotSize =< MaxSize -> + OldSize + SlotSize; +enforce_quota(UserDir, SlotSize, _OldSize, MinSize, MaxSize) -> + Files = lists:sort(fun({_PathA, _SizeA, TimeA}, {_PathB, _SizeB, TimeB}) -> + TimeA > TimeB + end, gather_file_info(UserDir)), + {DelFiles, OldSize, NewSize} = + lists:foldl(fun({_Path, Size, _Time}, {[], AccSize, AccSize}) + when AccSize + Size + SlotSize =< MinSize -> + {[], AccSize + Size, AccSize + Size}; + ({Path, Size, _Time}, {[], AccSize, AccSize}) -> + {[Path], AccSize + Size, AccSize}; + ({Path, Size, _Time}, {AccFiles, AccSize, NewSize}) -> + {[Path | AccFiles], AccSize + Size, NewSize} + end, {[], 0, 0}, Files), + if OldSize + SlotSize > MaxSize -> + lists:foreach(fun(File) -> del_file_and_dir(File) end, DelFiles), + file:del_dir(UserDir), % In case it's empty, now. + NewSize + SlotSize; + true -> + OldSize + SlotSize + end. + +-spec delete_old_files(file:filename_all(), integer()) -> ok. + +delete_old_files(UserDir, Timestamp) -> + FileInfo = gather_file_info(UserDir), + case [Path || {Path, _Size, Time} <- FileInfo, Time < Timestamp] of + [] -> + ok; + OldFiles -> + lists:foreach(fun(File) -> del_file_and_dir(File) end, OldFiles), + file:del_dir(UserDir) % In case it's empty, now. + end. + +-spec gather_file_info(file:filename_all()) + -> [{binary(), non_neg_integer(), non_neg_integer()}]. + +gather_file_info(Dir) when is_binary(Dir) -> + gather_file_info(binary_to_list(Dir)); +gather_file_info(Dir) -> + case file:list_dir(Dir) of + {ok, Entries} -> + lists:foldl(fun(Entry, Acc) -> + Path = Dir ++ "/" ++ Entry, + case file:read_file_info(Path, [{time, posix}]) of + {ok, #file_info{type = directory}} -> + gather_file_info(Path) ++ Acc; + {ok, #file_info{type = regular, + mtime = Time, + size = Size}} -> + [{Path, Size, Time} | Acc]; + {ok, _Info} -> + ?DEBUG("Won't stat(2) non-regular file ~s", + [Path]), + Acc; + {error, Error} -> + ?ERROR_MSG("Cannot stat(2) ~s: ~s", + [Path, ?FORMAT(Error)]), + Acc + end + end, [], Entries); + {error, enoent} -> + ?DEBUG("Directory ~s doesn't exist", [Dir]), + []; + {error, Error} -> + ?ERROR_MSG("Cannot open directory ~s: ~s", [Dir, ?FORMAT(Error)]), + [] + end. + +-spec del_file_and_dir(file:name_all()) -> ok. + +del_file_and_dir(File) -> + case file:delete(File) of + ok -> + ?INFO_MSG("Removed ~s", [File]), + Dir = filename:dirname(File), + case file:del_dir(Dir) of + ok -> + ?DEBUG("Removed ~s", [Dir]); + {error, Error} -> + ?INFO_MSG("Cannot remove ~s: ~s", [Dir, ?FORMAT(Error)]) + end; + {error, Error} -> + ?WARNING_MSG("Cannot remove ~s: ~s", [File, ?FORMAT(Error)]) + end. + +-spec secs_since_epoch() -> non_neg_integer(). + +secs_since_epoch() -> + {MegaSecs, Secs, _MicroSecs} = os:timestamp(), + MegaSecs * 1000000 + Secs. From b5a09f8b1594f5775c48870032cfc702e0ca6017 Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Mon, 26 Oct 2015 22:10:32 +0100 Subject: [PATCH 02/13] mod_http_upload: Strip newline from command output The list_to_integer/1 function doesn't cope with trailing newline characters. --- src/mod_http_upload.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mod_http_upload.erl b/src/mod_http_upload.erl index 40f194be2..88ca53f7b 100644 --- a/src/mod_http_upload.erl +++ b/src/mod_http_upload.erl @@ -838,7 +838,7 @@ code_to_message(_Code) -> <<"">>. identify(Path) -> Cmd = lists:flatten(io_lib:fwrite("identify -format \"ok %m %h %w\" ~s", [Path])), - Res = os:cmd(Cmd), + Res = string:strip(os:cmd(Cmd), right, $\n), case string:tokens(Res, " ") of ["ok", T, H, W] -> {ok, #media_info{ @@ -869,7 +869,7 @@ convert(Path, #media_info{type = T, width = W, height = H}) -> {ok, OutPath}; Err -> ?ERROR_MSG("failed to convert ~s to ~s: ~s", - [Path, OutPath, Err]), + [Path, OutPath, string:strip(Err, right, $\n)]), pass end; true -> From 3e7ee6af6d21d70e1d03ad84ba4f3f6f4ed8df95 Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Mon, 26 Oct 2015 22:30:58 +0100 Subject: [PATCH 03/13] mod_http_upload: Add/fix function specifications --- src/mod_http_upload.erl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/mod_http_upload.erl b/src/mod_http_upload.erl index 88ca53f7b..817f93b6f 100644 --- a/src/mod_http_upload.erl +++ b/src/mod_http_upload.erl @@ -719,6 +719,12 @@ iq_disco_info(Lang, Name) -> %% HTTP request handling. +-spec store_file(file:filename_all(), binary(), + integer() | undefined, + integer() | undefined, + binary(), binary(), boolean()) + -> ok | {ok, [{binary(), binary()}], binary()} | {error, term()}. + store_file(Path, Data, FileMode, DirMode, GetPrefix, LocalPath, Thumbnail) -> case do_store_file(Path, Data, FileMode, DirMode) of ok when Thumbnail -> @@ -747,7 +753,9 @@ store_file(Path, Data, FileMode, DirMode, GetPrefix, LocalPath, Thumbnail) -> Err end. --spec do_store_file(file:filename_all(), binary(), integer(), integer()) +-spec do_store_file(file:filename_all(), binary(), + integer() | undefined, + integer() | undefined) -> ok | {error, term()}. do_store_file(Path, Data, FileMode, DirMode) -> From 62ea763089850125eb17024a70aeec80db6a48ed Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Mon, 26 Oct 2015 22:32:12 +0100 Subject: [PATCH 04/13] mod_http_upload: Fix string()/binary() type issue --- src/mod_http_upload.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mod_http_upload.erl b/src/mod_http_upload.erl index 817f93b6f..35fd6ca34 100644 --- a/src/mod_http_upload.erl +++ b/src/mod_http_upload.erl @@ -888,7 +888,7 @@ convert(Path, #media_info{type = T, width = W, height = H}) -> -spec thumb_el(string(), binary()) -> xmlel(). thumb_el(Path, URI) -> - ContentType = guess_content_type(Path), + ContentType = guess_content_type(list_to_binary(Path)), case identify(Path) of {ok, #media_info{height = H, width = W}} -> #xmlel{name = <<"thumbnail">>, From 788049be7b649ad3c8ad0adb804a0496af5b6191 Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Mon, 26 Oct 2015 22:42:07 +0100 Subject: [PATCH 05/13] ejabberdctl: export CONTRIB_MODULES_CONF_DIR --- ejabberdctl.template | 1 + 1 file changed, 1 insertion(+) diff --git a/ejabberdctl.template b/ejabberdctl.template index f90f71e53..a4327ac9f 100755 --- a/ejabberdctl.template +++ b/ejabberdctl.template @@ -156,6 +156,7 @@ export ERL_INETRC export ERL_MAX_PORTS export ERL_MAX_ETS_TABLES export CONTRIB_MODULES_PATH +export CONTRIB_MODULES_CONF_DIR export ERL_LIBS # start server From 47b17acbafd37ac423685f52c26839b33b856390 Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Mon, 26 Oct 2015 22:46:58 +0100 Subject: [PATCH 06/13] ejabberdctl.cfg: Document CONTRIB_MODULES_CONF_DIR --- ejabberdctl.cfg.example | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ejabberdctl.cfg.example b/ejabberdctl.cfg.example index 63c917358..d121aa824 100644 --- a/ejabberdctl.cfg.example +++ b/ejabberdctl.cfg.example @@ -169,6 +169,18 @@ # #CONTRIB_MODULES_PATH=/opt/ejabberd-modules +#. +#' CONTRIB_MODULES_CONF_DIR: configuration directory for contributed modules +# +# Specify the full path to the configuration directory for contributed ejabberd +# modules. In order to configure a module named mod_foo, a mod_foo.yml file can +# be created in this directory. This file will then be used instead of the +# default configuration file provided with the module. +# +# Default: $CONTRIB_MODULES_PATH/conf +# +#CONTRIB_MODULES_CONF_DIR=/etc/ejabberd/modules + #. #' # vim: foldmarker=#',#. foldmethod=marker: From a497cdff5ee0bf41749d076a445ba7371ddbd69e Mon Sep 17 00:00:00 2001 From: Christophe Romain Date: Wed, 28 Oct 2015 15:53:12 +0100 Subject: [PATCH 07/13] set dependencies versions for 15.10 --- rebar.config.script | 46 ++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/rebar.config.script b/rebar.config.script index e0a8923cf..833595d48 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -54,17 +54,17 @@ SrcDirs = lists:foldl( Acc end, [], Cfg), -Deps = [{p1_cache_tab, ".*", {git, "https://github.com/processone/cache_tab"}}, - {p1_tls, ".*", {git, "https://github.com/processone/tls"}}, - {p1_stringprep, ".*", {git, "https://github.com/processone/stringprep"}}, - {p1_xml, ".*", {git, "https://github.com/processone/xml"}}, - {esip, ".*", {git, "https://github.com/processone/p1_sip"}}, - {p1_stun, ".*", {git, "https://github.com/processone/stun"}}, - {p1_yaml, ".*", {git, "https://github.com/processone/p1_yaml"}}, - {p1_utils, ".*", {git, "https://github.com/processone/p1_utils"}}, - {jiffy, ".*", {git, "https://github.com/davisp/jiffy"}}, - {oauth2, ".*", {git, "https://github.com/prefiks/oauth2.git"}}, - {xmlrpc, ".*", {git, "https://github.com/rds13/xmlrpc.git"}}], +Deps = [{p1_cache_tab, ".*", {git, "https://github.com/processone/cache_tab", "f7ea12b0ba962a3d2f9a406d2954cf7de4e27230"}}, + {p1_tls, ".*", {git, "https://github.com/processone/tls", "e56321afd974e9da33da913cd31beebc8e73e75f"}}, + {p1_stringprep, ".*", {git, "https://github.com/processone/stringprep", "3c640237a3a7831dc39de6a6d329d3a9af25c579"}}, + {p1_xml, ".*", {git, "https://github.com/processone/xml", "1c8b016b0ac7986efb823baf1682a43565449e65"}}, + {esip, ".*", {git, "https://github.com/processone/p1_sip", "d662d3fe7f6288b444ea321d854de0bd6d40e022"}}, + {p1_stun, ".*", {git, "https://github.com/processone/stun", "061bdae484268cbf0457ad4797e74b8516df3ad1"}}, + {p1_yaml, ".*", {git, "https://github.com/processone/p1_yaml", "79f756ba73a235c4d3836ec07b5f7f2b55f49638"}}, + {p1_utils, ".*", {git, "https://github.com/processone/p1_utils", "d7800881e6702723ce58b7646b60c9e4cd25d563"}}, + {jiffy, ".*", {git, "https://github.com/davisp/jiffy", "cfc61a2e952dc3182e0f9b1473467563699992e2"}}, + {oauth2, ".*", {git, "https://github.com/prefiks/oauth2.git", "e6da9912e5d8f658e7e868f41a102d085bdbef59"}}, + {xmlrpc, ".*", {git, "https://github.com/rds13/xmlrpc.git", "42e6e96a0fe7106830274feed915125feb1056f3"}}], CFLags = proplists:get_value(cflags, Cfg, ""), CPPFLags = proplists:get_value(cppflags, Cfg, ""), @@ -94,30 +94,30 @@ PostHooks = [ConfigureCmd("p1_tls", ""), CfgDeps = lists:flatmap( fun({mysql, true}) -> - [{p1_mysql, ".*", {git, "https://github.com/processone/mysql"}}]; + [{p1_mysql, ".*", {git, "https://github.com/processone/mysql", "dfa87da95f8fdb92e270741c2a53f796b682f918"}}]; ({pgsql, true}) -> - [{p1_pgsql, ".*", {git, "https://github.com/processone/pgsql"}}]; + [{p1_pgsql, ".*", {git, "https://github.com/processone/pgsql", "e72c03c60bfcb56bbb5d259342021d9cb3581dac"}}]; ({sqlite, true}) -> - [{sqlite3, ".*", {git, "https://github.com/alexeyr/erlang-sqlite3"}}]; + [{sqlite3, ".*", {git, "https://github.com/alexeyr/erlang-sqlite3", "8350dc603804c503f99c92bfd2eab1dd6885758e"}}]; ({pam, true}) -> - [{p1_pam, ".*", {git, "https://github.com/processone/epam"}}]; + [{p1_pam, ".*", {git, "https://github.com/processone/epam", "d3ce290b7da75d780a03e86e7a8198a80e9826a6"}}]; ({zlib, true}) -> - [{p1_zlib, ".*", {git, "https://github.com/processone/zlib"}}]; + [{p1_zlib, ".*", {git, "https://github.com/processone/zlib", "e3d4222b7aae616d7ef2e7e2fa0bbf451516c602"}}]; ({riak, true}) -> [{riakc, ".*", {git, "https://github.com/basho/riak-erlang-client", {tag, "1.4.2"}}}]; ({elixir, true}) -> - [{rebar_elixir_plugin, ".*", {git, "https://github.com/yrashk/rebar_elixir_plugin"}}, - {elixir, ".*", {git, "https://github.com/elixir-lang/elixir", {branch, "v1.0"}}}]; + [{rebar_elixir_plugin, ".*", {git, "https://github.com/yrashk/rebar_elixir_plugin", "7058379b7c7e017555647f6b9cecfd87cd50f884"}}, + {elixir, ".*", {git, "https://github.com/elixir-lang/elixir", "1d9548fd285d243721b7eba71912bde2ffd1f6c3"}}]; ({iconv, true}) -> - [{p1_iconv, ".*", {git, "https://github.com/processone/eiconv"}}]; + [{p1_iconv, ".*", {git, "https://github.com/processone/eiconv", "8b7542b1aaf0a851f335e464956956985af6d9a2"}}]; ({lager, true}) -> - [{lager, ".*", {git, "https://github.com/basho/lager"}}]; + [{lager, ".*", {git, "https://github.com/basho/lager", "4d2ec8c701e1fa2d386f92f2b83b23faf8608ac3"}}]; ({lager, false}) -> - [{p1_logger, ".*", {git, "https://github.com/processone/p1_logger"}}]; + [{p1_logger, ".*", {git, "https://github.com/processone/p1_logger", "3e19507fd5606a73694917158767ecb3f5704e3f"}}]; ({tools, true}) -> - [{meck, "0.*", {git, "https://github.com/eproxus/meck"}}]; + [{meck, "0.*", {git, "https://github.com/eproxus/meck", "0845277398b8326f9dddddd9fc3cf73467ba6877"}}]; ({redis, true}) -> - [{eredis, ".*", {git, "https://github.com/wooga/eredis"}}]; + [{eredis, ".*", {git, "https://github.com/wooga/eredis", "bf12ecb30253c84a2331f4f0d93fd68856fcb9f4"}}]; (_) -> [] end, Cfg), From 1d5a441ce87ede4fc74e43fe4b76c8a2278c4350 Mon Sep 17 00:00:00 2001 From: Christophe Romain Date: Wed, 28 Oct 2015 16:07:00 +0100 Subject: [PATCH 08/13] Revert "set dependencies versions for 15.10" This reverts commit a497cdff5ee0bf41749d076a445ba7371ddbd69e. --- rebar.config.script | 46 ++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/rebar.config.script b/rebar.config.script index 833595d48..e0a8923cf 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -54,17 +54,17 @@ SrcDirs = lists:foldl( Acc end, [], Cfg), -Deps = [{p1_cache_tab, ".*", {git, "https://github.com/processone/cache_tab", "f7ea12b0ba962a3d2f9a406d2954cf7de4e27230"}}, - {p1_tls, ".*", {git, "https://github.com/processone/tls", "e56321afd974e9da33da913cd31beebc8e73e75f"}}, - {p1_stringprep, ".*", {git, "https://github.com/processone/stringprep", "3c640237a3a7831dc39de6a6d329d3a9af25c579"}}, - {p1_xml, ".*", {git, "https://github.com/processone/xml", "1c8b016b0ac7986efb823baf1682a43565449e65"}}, - {esip, ".*", {git, "https://github.com/processone/p1_sip", "d662d3fe7f6288b444ea321d854de0bd6d40e022"}}, - {p1_stun, ".*", {git, "https://github.com/processone/stun", "061bdae484268cbf0457ad4797e74b8516df3ad1"}}, - {p1_yaml, ".*", {git, "https://github.com/processone/p1_yaml", "79f756ba73a235c4d3836ec07b5f7f2b55f49638"}}, - {p1_utils, ".*", {git, "https://github.com/processone/p1_utils", "d7800881e6702723ce58b7646b60c9e4cd25d563"}}, - {jiffy, ".*", {git, "https://github.com/davisp/jiffy", "cfc61a2e952dc3182e0f9b1473467563699992e2"}}, - {oauth2, ".*", {git, "https://github.com/prefiks/oauth2.git", "e6da9912e5d8f658e7e868f41a102d085bdbef59"}}, - {xmlrpc, ".*", {git, "https://github.com/rds13/xmlrpc.git", "42e6e96a0fe7106830274feed915125feb1056f3"}}], +Deps = [{p1_cache_tab, ".*", {git, "https://github.com/processone/cache_tab"}}, + {p1_tls, ".*", {git, "https://github.com/processone/tls"}}, + {p1_stringprep, ".*", {git, "https://github.com/processone/stringprep"}}, + {p1_xml, ".*", {git, "https://github.com/processone/xml"}}, + {esip, ".*", {git, "https://github.com/processone/p1_sip"}}, + {p1_stun, ".*", {git, "https://github.com/processone/stun"}}, + {p1_yaml, ".*", {git, "https://github.com/processone/p1_yaml"}}, + {p1_utils, ".*", {git, "https://github.com/processone/p1_utils"}}, + {jiffy, ".*", {git, "https://github.com/davisp/jiffy"}}, + {oauth2, ".*", {git, "https://github.com/prefiks/oauth2.git"}}, + {xmlrpc, ".*", {git, "https://github.com/rds13/xmlrpc.git"}}], CFLags = proplists:get_value(cflags, Cfg, ""), CPPFLags = proplists:get_value(cppflags, Cfg, ""), @@ -94,30 +94,30 @@ PostHooks = [ConfigureCmd("p1_tls", ""), CfgDeps = lists:flatmap( fun({mysql, true}) -> - [{p1_mysql, ".*", {git, "https://github.com/processone/mysql", "dfa87da95f8fdb92e270741c2a53f796b682f918"}}]; + [{p1_mysql, ".*", {git, "https://github.com/processone/mysql"}}]; ({pgsql, true}) -> - [{p1_pgsql, ".*", {git, "https://github.com/processone/pgsql", "e72c03c60bfcb56bbb5d259342021d9cb3581dac"}}]; + [{p1_pgsql, ".*", {git, "https://github.com/processone/pgsql"}}]; ({sqlite, true}) -> - [{sqlite3, ".*", {git, "https://github.com/alexeyr/erlang-sqlite3", "8350dc603804c503f99c92bfd2eab1dd6885758e"}}]; + [{sqlite3, ".*", {git, "https://github.com/alexeyr/erlang-sqlite3"}}]; ({pam, true}) -> - [{p1_pam, ".*", {git, "https://github.com/processone/epam", "d3ce290b7da75d780a03e86e7a8198a80e9826a6"}}]; + [{p1_pam, ".*", {git, "https://github.com/processone/epam"}}]; ({zlib, true}) -> - [{p1_zlib, ".*", {git, "https://github.com/processone/zlib", "e3d4222b7aae616d7ef2e7e2fa0bbf451516c602"}}]; + [{p1_zlib, ".*", {git, "https://github.com/processone/zlib"}}]; ({riak, true}) -> [{riakc, ".*", {git, "https://github.com/basho/riak-erlang-client", {tag, "1.4.2"}}}]; ({elixir, true}) -> - [{rebar_elixir_plugin, ".*", {git, "https://github.com/yrashk/rebar_elixir_plugin", "7058379b7c7e017555647f6b9cecfd87cd50f884"}}, - {elixir, ".*", {git, "https://github.com/elixir-lang/elixir", "1d9548fd285d243721b7eba71912bde2ffd1f6c3"}}]; + [{rebar_elixir_plugin, ".*", {git, "https://github.com/yrashk/rebar_elixir_plugin"}}, + {elixir, ".*", {git, "https://github.com/elixir-lang/elixir", {branch, "v1.0"}}}]; ({iconv, true}) -> - [{p1_iconv, ".*", {git, "https://github.com/processone/eiconv", "8b7542b1aaf0a851f335e464956956985af6d9a2"}}]; + [{p1_iconv, ".*", {git, "https://github.com/processone/eiconv"}}]; ({lager, true}) -> - [{lager, ".*", {git, "https://github.com/basho/lager", "4d2ec8c701e1fa2d386f92f2b83b23faf8608ac3"}}]; + [{lager, ".*", {git, "https://github.com/basho/lager"}}]; ({lager, false}) -> - [{p1_logger, ".*", {git, "https://github.com/processone/p1_logger", "3e19507fd5606a73694917158767ecb3f5704e3f"}}]; + [{p1_logger, ".*", {git, "https://github.com/processone/p1_logger"}}]; ({tools, true}) -> - [{meck, "0.*", {git, "https://github.com/eproxus/meck", "0845277398b8326f9dddddd9fc3cf73467ba6877"}}]; + [{meck, "0.*", {git, "https://github.com/eproxus/meck"}}]; ({redis, true}) -> - [{eredis, ".*", {git, "https://github.com/wooga/eredis", "bf12ecb30253c84a2331f4f0d93fd68856fcb9f4"}}]; + [{eredis, ".*", {git, "https://github.com/wooga/eredis"}}]; (_) -> [] end, Cfg), From f112a913218ffedc86c643fcefe5c3c44bf0aa5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Chmielowski?= Date: Wed, 28 Oct 2015 18:12:39 +0100 Subject: [PATCH 09/13] Copy elixir files in copy-files makefile target --- Makefile.in | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Makefile.in b/Makefile.in index 391d81612..f12ccf397 100644 --- a/Makefile.in +++ b/Makefile.in @@ -112,12 +112,19 @@ spec: $(ERL) -noinput +B -pa ebin -pa deps/*/ebin -eval \ 'case xml_gen:compile("tools/xmpp_codec.spec") of ok -> halt(0); _ -> halt(1) end.' -TO_DEST=$(foreach path,$(1),$(if $(filter deps/%,$(path)),$(patsubst deps/%,$(LIBDIR)/%,$(path)),$(patsubst %,$(LIBDIR)/ejabberd/%,$(path)))) +JOIN_PATHS=$(if $(wordlist 2,1000,$(1)),$(firstword $(1))/$(call JOIN_PATHS,$(wordlist 2,1000,$(1))),$(1)) + +ELIXIR_TO_DEST=$(LIBDIR) $(wordlist 2,2,$(1)) $(wordlist 5,1000,$(1)) +DEPS_TO_DEST=$(LIBDIR) $(wordlist 2,1000,$(1)) +MAIN_TO_DEST=$(LIBDIR) ejabberd $(1) +TO_DEST_SINGLE=$(if $(subst XdepsX,,X$(word 1,$(1))X),$(call MAIN_TO_DEST,$(1)),$(if $(subst XlibX,,X$(word 3,$(1))X),$(call DEPS_TO_DEST,$(1)),$(call ELIXIR_TO_DEST,$(1)))) +TO_DEST=$(foreach path,$(1),$(call JOIN_PATHS,$(call TO_DEST_SINGLE,$(subst /, ,$(path))))) + FILTER_DIRS=$(foreach path,$(1),$(if $(wildcard $(path)/*),,$(path))) FILES_WILDCARD=$(call FILTER_DIRS,$(foreach w,$(1),$(wildcard $(w)))) -DEPS_FILES:=$(call FILES_WILDCARD,deps/*/ebin/*.beam deps/*/ebin/*.app deps/*/priv/* deps/*/priv/lib/* deps/*/priv/bin/* deps/*/include/*.hrl) -DEPS_FILES_FILTERED:=$(filter-out %/epam,$(DEPS_FILES)) +DEPS_FILES:=$(call FILES_WILDCARD,deps/*/ebin/*.beam deps/*/ebin/*.app deps/*/priv/* deps/*/priv/lib/* deps/*/priv/bin/* deps/*/include/*.hrl deps/*/lib/*/ebin/*.beam deps/*/lib/*/ebin/*.app) +DEPS_FILES_FILTERED:=$(filter-out %/epam deps/elixir/ebin/elixir.app,$(DEPS_FILES)) DEPS_DIRS:=$(sort deps/ $(wildcard deps/*) $(dir $(DEPS_FILES))) MAIN_FILES:=$(filter-out %/configure.beam,$(call FILES_WILDCARD,ebin/*.beam ebin/*.app priv/msgs/*.msg priv/lib/* include/*.hrl)) @@ -129,7 +136,7 @@ endef $(foreach file,$(DEPS_FILES_FILTERED) $(MAIN_FILES),$(eval $(call COPY_template,$(file)))) -$(call TO_DEST,$(MAIN_DIRS) $(DEPS_DIRS)): +$(sort $(call TO_DEST,$(MAIN_DIRS) $(DEPS_DIRS))): $(INSTALL) -d $@ $(call TO_DEST,deps/p1_pam/priv/bin/epam): $(LIBDIR)/%: deps/% $(call TO_DEST,deps/p1_pam/priv/bin/) @@ -172,10 +179,6 @@ install: all copy-files -e "s*@installuser@*$(INIT_USER)*" ejabberd.init.template \ > ejabberd.init chmod 755 ejabberd.init - # Install Elixir and Elixir dependancies - -$(INSTALL) -m 644 deps/*/lib/*/ebin/*.app $(BEAMDIR) - -$(INSTALL) -m 644 deps/*/lib/*/ebin/*.beam $(BEAMDIR) - rm -f $(BEAMDIR)/configure.beam # # Binary C programs $(INSTALL) -d $(PBINDIR) From 32fe74c923f151334dab5656dcc2ac5ea328b09d Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Wed, 28 Oct 2015 23:52:33 +0100 Subject: [PATCH 10/13] mod_http_upload: Document protocol support --- src/mod_http_upload.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mod_http_upload.erl b/src/mod_http_upload.erl index 35fd6ca34..68a7818fc 100644 --- a/src/mod_http_upload.erl +++ b/src/mod_http_upload.erl @@ -8,6 +8,8 @@ -module(mod_http_upload). -author('holger@zedat.fu-berlin.de'). +-protocol({xep, 363, '0.1'}) + -define(GEN_SERVER, gen_server). -define(SERVICE_REQUEST_TIMEOUT, 5000). % 5 seconds. -define(SLOT_TIMEOUT, 18000000). % 5 hours. From 43e78147143fdf2a31add628fe589134abde506c Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Wed, 28 Oct 2015 23:53:53 +0100 Subject: [PATCH 11/13] mod_mam: Update supported XEP version --- src/mod_mam.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mod_mam.erl b/src/mod_mam.erl index 7781cff9a..c23e695d9 100644 --- a/src/mod_mam.erl +++ b/src/mod_mam.erl @@ -25,7 +25,7 @@ %%%------------------------------------------------------------------- -module(mod_mam). --protocol({xep, 313, '0.3'}). +-protocol({xep, 313, '0.4'}). -behaviour(gen_mod). From 915383e150a4d25144f363310ba9a0882493f752 Mon Sep 17 00:00:00 2001 From: Holger Weiss Date: Thu, 29 Oct 2015 10:09:55 +0100 Subject: [PATCH 12/13] mod_http_upload: Add missing trailing dot (Thanks to Matthias Rieber.) --- src/mod_http_upload.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mod_http_upload.erl b/src/mod_http_upload.erl index 68a7818fc..6356e6368 100644 --- a/src/mod_http_upload.erl +++ b/src/mod_http_upload.erl @@ -8,7 +8,7 @@ -module(mod_http_upload). -author('holger@zedat.fu-berlin.de'). --protocol({xep, 363, '0.1'}) +-protocol({xep, 363, '0.1'}). -define(GEN_SERVER, gen_server). -define(SERVICE_REQUEST_TIMEOUT, 5000). % 5 seconds. From bee767158da6240f13cf9d56e857d72ec3b7dbf1 Mon Sep 17 00:00:00 2001 From: Arno B Date: Thu, 29 Oct 2015 19:39:21 +0100 Subject: [PATCH 13/13] feat(ejabberdctl): Foreground option for docker Docker needs services to run in foreground. --- ejabberdctl.template | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/ejabberdctl.template b/ejabberdctl.template index a4327ac9f..147bff97e 100755 --- a/ejabberdctl.template +++ b/ejabberdctl.template @@ -228,6 +228,21 @@ iexlive() --erl \"$ERLANG_OPTS\" --erl $ARGS --erl \"$@\"" } +# start server in the foreground +foreground() +{ + check_start + $EXEC_CMD "$ERL \ + $NAME $ERLANG_NODE \ + -noinput \ + -pa $EJABBERD_EBIN_PATH \ + $MNESIA_OPTS \ + $KERNEL_OPTS \ + $EJABBERD_OPTS \ + -s ejabberd \ + $ERLANG_OPTS $ARGS \"$@\"" +} + debugwarning() { if [ "$EJABBERD_BYPASS_WARNINGS" != "true" ] ; then @@ -300,11 +315,12 @@ help() { echo "" echo "Commands to start an ejabberd node:" - echo " start Start an ejabberd node in server mode" - echo " debug Attach an interactive Erlang shell to a running ejabberd node" - echo " iexdebug Attach an interactive Elixir shell to a running ejabberd node" - echo " live Start an ejabberd node in live (interactive) mode" - echo " iexlive Start an ejabberd node in live (interactive) mode, within an Elixir shell" + echo " start Start an ejabberd node in server mode" + echo " debug Attach an interactive Erlang shell to a running ejabberd node" + echo " iexdebug Attach an interactive Elixir shell to a running ejabberd node" + echo " live Start an ejabberd node in live (interactive) mode" + echo " iexlive Start an ejabberd node in live (interactive) mode, within an Elixir shell" + echo " foreground Start an ejabberd node in server mode (attached)" echo "" echo "Optional parameters when starting an ejabberd node:" echo " --config-dir dir Config ejabberd: $ETC_DIR" @@ -468,6 +484,7 @@ case $ARGS in ' iexdebug') iexdebug;; ' live') live;; ' iexlive') iexlive;; + ' foreground') foreground;; ' ping'*) ping ${ARGS# ping};; ' etop') etop;; ' started') wait_for_status 0 30 2;; # wait 30x2s before timeout