From 55f8aa1b22802c8e38feee6671814d9e717eac6e Mon Sep 17 00:00:00 2001
From: Evgeniy Khramtsov <ekhramtsov@process-one.net>
Date: Thu, 21 Jun 2018 14:35:19 +0300
Subject: [PATCH] Add new options for OOM watchdog

* oom_watermark: 1..100
  Start OOM watchdog only when system memory usage exceeds
  this value in percents. When the usage drops below the value,
  OOM watchdog is stopped. The default is 80 (percents).
  Note that once OOM watchdog is started, it performs full garbage
  collection periodically: this can be seen as spikes in CPU
  utilization and drops in RAM usage. If your system is permanently
  above the watermark, it may cause significant CPU overhead.

* oom_queue: positive integer
  Only trigger OOM killer when total amount of messages in all queues
  of all Erlang processes is above this value. The default is 10000.
  Note that this value only takes effect when `oom_killer` is set
  to `true` (this is the default). Otherwise, only a warning will
  be logged.
---
 src/ejabberd_app.erl            |  2 +-
 src/ejabberd_system_monitor.erl | 28 +++++++++++++++++++++-------
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/src/ejabberd_app.erl b/src/ejabberd_app.erl
index 7efb9f54b..284be384b 100644
--- a/src/ejabberd_app.erl
+++ b/src/ejabberd_app.erl
@@ -50,9 +50,9 @@ start(normal, _Args) ->
 	    ejabberd_mnesia:start(),
 	    file_queue_init(),
 	    maybe_add_nameservers(),
-	    ejabberd_system_monitor:start(),
 	    case ejabberd_sup:start_link() of
 		{ok, SupPid} ->
+		    ejabberd_system_monitor:start(),
 		    register_elixir_config_hooks(),
 		    ejabberd_cluster:wait_for_sync(infinity),
 		    {T2, _} = statistics(wall_clock),
diff --git a/src/ejabberd_system_monitor.erl b/src/ejabberd_system_monitor.erl
index aa22317d1..d84d6b807 100644
--- a/src/ejabberd_system_monitor.erl
+++ b/src/ejabberd_system_monitor.erl
@@ -31,7 +31,7 @@
 -author('ekhramtsov@process-one.net').
 
 %% API
--export([start/0, opt_type/1]).
+-export([start/0, opt_type/1, config_reloaded/0]).
 
 %% gen_event callbacks
 -export([init/1, handle_event/2, handle_call/2,
@@ -67,15 +67,21 @@ start() ->
     application:set_env(os_mon, start_os_sup, false),
     application:set_env(os_mon, start_memsup, true),
     application:set_env(os_mon, start_disksup, false),
-    ejabberd:start_app(os_mon).
+    ejabberd:start_app(os_mon),
+    set_oom_watermark().
 
 excluded_apps() ->
     [os_mon, mnesia, sasl, stdlib, kernel].
 
+-spec config_reloaded() -> ok.
+config_reloaded() ->
+    set_oom_watermark().
+
 %%%===================================================================
 %%% gen_event callbacks
 %%%===================================================================
 init([]) ->
+    ejabberd_hooks:add(config_reloaded, ?MODULE, config_reloaded, 50),
     {ok, #state{}}.
 
 handle_event({set_alarm, {system_memory_high_watermark, _}}, State) ->
@@ -112,7 +118,7 @@ handle_info(Info, State) ->
     {ok, State}.
 
 terminate(_Reason, _State) ->
-    ok.
+    ejabberd_hooks:delete(config_reloaded, ?MODULE, config_reloaded, 50).
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
@@ -128,7 +134,8 @@ handle_overload(State) ->
 handle_overload(_State, Procs) ->
     AppPids = get_app_pids(),
     {TotalMsgs, ProcsNum, Apps, Stats} = overloaded_procs(AppPids, Procs),
-    if TotalMsgs >= 10000 ->
+    MaxMsgs = ejabberd_config:get_option(oom_queue, 10000),
+    if TotalMsgs >= MaxMsgs ->
 	    SortedStats = lists:reverse(lists:keysort(#proc_stat.qlen, Stats)),
 	    error_logger:warning_msg(
 	      "The system is overloaded with ~b messages "
@@ -312,14 +319,21 @@ kill_proc(Pid) ->
     exit(Pid, kill),
     Pid.
 
+-spec set_oom_watermark() -> ok.
+set_oom_watermark() ->
+    WaterMark = ejabberd_config:get_option(oom_watermark, 80),
+    memsup:set_sysmem_high_watermark(WaterMark/100).
+
 -spec maybe_restart_app(atom()) -> any().
 maybe_restart_app(lager) ->
     ejabberd_logger:restart();
 maybe_restart_app(_) ->
     ok.
 
--spec opt_type(oom_killer) -> fun((boolean()) -> boolean());
-	      (atom()) -> [atom()].
 opt_type(oom_killer) ->
     fun(B) when is_boolean(B) -> B end;
-opt_type(_) -> [oom_killer].
+opt_type(oom_watermark) ->
+    fun(I) when is_integer(I), I>0, I<100 -> I end;
+opt_type(oom_queue) ->
+    fun(I) when is_integer(I)>0 -> I end;
+opt_type(_) -> [oom_killer, oom_watermark, oom_queue].