Kill and restart lager when it's overloaded

src/ejabberd_logger.erl

@@ -27,7 +27,8 @@
 -behaviour(ejabberd_config).
 
 %% API
--export([start/0, reopen_log/0, rotate_log/0, get/0, set/1, get_log_path/0, opt_type/1]).
+-export([start/0, restart/0, reopen_log/0, rotate_log/0, get/0, set/1,
+	 get_log_path/0, opt_type/1]).
 
 -include("ejabberd.hrl").
 
@@ -102,6 +103,11 @@ get_string_env(Name, Default) ->
 
 %% @spec () -> ok
 start() ->
+    start(4).
+
+-spec start(loglevel()) -> ok.
+start(Level) ->
+    LLevel = get_lager_loglevel(Level),
     StartedApps = application:which_applications(5000),
     case lists:keyfind(logger, 1, StartedApps) of
         %% Elixir logger is started. We assume everything is in place
@@ -109,24 +115,24 @@ start() ->
         {logger, _, _} ->
             error_logger:info_msg("Ignoring ejabberd logger options, using Elixir Logger.", []),
             %% Do not start lager, we rely on Elixir Logger
-            do_start_for_logger();
+            do_start_for_logger(LLevel);
         _ ->
-            do_start()
+            do_start(LLevel)
     end.
 
-do_start_for_logger() ->
+do_start_for_logger(Level) ->
     application:load(sasl),
     application:set_env(sasl, sasl_error_logger, false),
     application:load(lager),
     application:set_env(lager, error_logger_redirect, false),
     application:set_env(lager, error_logger_whitelist, ['Elixir.Logger.ErrorHandler']),
     application:set_env(lager, crash_log, false),
-    application:set_env(lager, handlers, [{elixir_logger_backend, [{level, info}]}]),
+    application:set_env(lager, handlers, [{elixir_logger_backend, [{level, Level}]}]),
     ejabberd:start_app(lager),
     ok.
 
 %% Start lager
-do_start() ->
+do_start(Level) ->
     application:load(sasl),
     application:set_env(sasl, sasl_error_logger, false),
     application:load(lager),
@@ -141,8 +147,8 @@ do_start() ->
     application:set_env(lager, error_logger_hwm, LogRateLimit),
     application:set_env(
       lager, handlers,
-      [{lager_console_backend, info},
-       {lager_file_backend, [{file, ConsoleLog}, {level, info}, {date, LogRotateDate},
+      [{lager_console_backend, Level},
+       {lager_file_backend, [{file, ConsoleLog}, {level, Level}, {date, LogRotateDate},
                              {count, LogRotateCount}, {size, LogRotateSize}]},
        {lager_file_backend, [{file, ErrorLog}, {level, error}, {date, LogRotateDate},
                              {count, LogRotateCount}, {size, LogRotateSize}]}]),
@@ -156,6 +162,11 @@ do_start() ->
 		  end, gen_event:which_handlers(lager_event)),
     ok.
 
+restart() ->
+    Level = ejabberd_config:get_option(loglevel, 4),
+    application:stop(lager),
+    start(Level).
+
 %% @spec () -> ok
 reopen_log() ->
     %% Lager detects external log rotation automatically.
@@ -187,15 +198,7 @@ get() ->
 
 %% @spec (loglevel() | {loglevel(), list()}) -> {module, module()}
 set(LogLevel) when is_integer(LogLevel) ->
-    LagerLogLevel = case LogLevel of
-                        0 -> none;
-                        1 -> critical;
-                        2 -> error;
-                        3 -> warning;
-                        4 -> info;
-                        5 -> debug;
-			E ->  throw({wrong_loglevel, E})
-                    end,
+    LagerLogLevel = get_lager_loglevel(LogLevel),
     case get_lager_loglevel() of
         LagerLogLevel ->
             ok;
@@ -228,6 +231,17 @@ get_lager_loglevel() ->
                 end,
                 none, Handlers).
 
+get_lager_loglevel(LogLevel) ->
+    case LogLevel of
+	0 -> none;
+	1 -> critical;
+	2 -> error;
+	3 -> warning;
+	4 -> info;
+	5 -> debug;
+	E -> erlang:error({wrong_loglevel, E})
+    end.
+
 get_lager_handlers() ->
     case catch gen_event:which_handlers(lager_event) of
         {'EXIT',noproc} ->

src/ejabberd_system_monitor.erl

@@ -68,11 +68,7 @@ start() ->
     ejabberd:start_app(os_mon).
 
 excluded_apps() ->
-    %% FIXME: lager gets overloaded very often, but
-    %% it fails to get recovered after brutal kill,
-    %% so it's better to make it tolerate crashes
-    %% rather than ignoring the overload
-    [os_mon, mnesia, sasl, stdlib, kernel, lager].
+    [os_mon, mnesia, sasl, stdlib, kernel].
 
 %%%===================================================================
 %%% gen_event callbacks
@@ -270,8 +266,11 @@ do_kill(Stats, Threshold) ->
 			       false;
 			   false ->
 			       case kill_proc(Name) of
-				   false -> false;
-				   Pid -> {true, Pid}
+				   false ->
+				       false;
+				   Pid ->
+				       maybe_restart_app(App),
+				       {true, Pid}
 			       end
 		       end;
 		  (_) ->
@@ -295,6 +294,12 @@ kill_proc(Pid) ->
     exit(Pid, kill),
     Pid.
 
+-spec maybe_restart_app(atom()) -> any().
+maybe_restart_app(lager) ->
+    ejabberd_logger:restart();
+maybe_restart_app(_) ->
+    ok.
+
 -spec opt_type(oom_killer) -> fun((boolean()) -> boolean());
 	      (atom()) -> [atom()].
 opt_type(oom_killer) ->