Add log_burst_limit_* options (#3865)

* Add log_burst_limit_* options

On our ejabberd deployment we were sometimes seeing more than 500
msgs/sec of legitimate traffic, however this was getting silently
dropped. Provide config options to enable this limit to be configured
from the config file.

* Pass new logging vars in via ejabberdctl

.github/container/ejabberdctl.template

@@ -91,7 +91,9 @@ ERL_INETRC="$CONFIG_DIR"/inetrc
 # define ejabberd parameters
 EJABBERD_OPTS="$EJABBERD_OPTS\
 $(sed '/^log_rotate_size/!d;s/:[ \t]*\([0-9]\{1,\}\).*/ \1/;s/:[ \t]*\(infinity\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\
-$(sed '/^log_rotate_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")"
+$(sed '/^log_rotate_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\
+$(sed '/^log_burst_limit_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\
+$(sed '/^log_burst_limit_window_time/!d;s/:[ \t]*\([0-9]*[a-z]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")"
 [ -n "$EJABBERD_OPTS" ] && EJABBERD_OPTS="-ejabberd $EJABBERD_OPTS"
 EJABBERD_OPTS="-mnesia dir \"$SPOOL_DIR\" $MNESIA_OPTIONS $EJABBERD_OPTS -s ejabberd"
 

ejabberdctl.template

@@ -90,7 +90,9 @@ ERL_INETRC="$CONFIG_DIR"/inetrc
 # define ejabberd parameters
 EJABBERD_OPTS="$EJABBERD_OPTS\
 $(sed '/^log_rotate_size/!d;s/:[ \t]*\([0-9]\{1,\}\).*/ \1/;s/:[ \t]*\(infinity\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\
-$(sed '/^log_rotate_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")"
+$(sed '/^log_rotate_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\
+$(sed '/^log_burst_limit_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\
+$(sed '/^log_burst_limit_window_time/!d;s/:[ \t]*\([0-9]*[a-z]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")"
 [ -n "$EJABBERD_OPTS" ] && EJABBERD_OPTS="-ejabberd $EJABBERD_OPTS"
 EJABBERD_OPTS="-mnesia dir \"$SPOOL_DIR\" $MNESIA_OPTIONS $EJABBERD_OPTS -s ejabberd"
 

src/ejabberd_logger.erl

@@ -266,13 +266,17 @@ start(Level) ->
     ErrorLog = filename:join([Dir, "error.log"]),
     LogRotateSize = get_integer_env(log_rotate_size, 10*1024*1024),
     LogRotateCount = get_integer_env(log_rotate_count, 1),
+    LogBurstLimitWindowTime = get_integer_env(log_burst_limit_window_time, 1000),
+    LogBurstLimitCount = get_integer_env(log_burst_limit_count, 500),
     Config = #{max_no_bytes => LogRotateSize,
 	       max_no_files => LogRotateCount,
 	       filesync_repeat_interval => no_repeat,
 	       file_check => 1000,
 	       sync_mode_qlen => 1000,
 	       drop_mode_qlen => 1000,
-	       flush_qlen => 5000},
+	       flush_qlen => 5000,
+	       burst_limit_window_time => LogBurstLimitWindowTime,
+	       burst_limit_max_count => LogBurstLimitCount},
     FmtConfig = #{legacy_header => false,
 		  time_designator => $ ,
 		  max_size => 100*1024,

src/ejabberd_option.erl

@@ -72,6 +72,8 @@
 -export([ldap_tls_verify/0, ldap_tls_verify/1]).
 -export([ldap_uids/0, ldap_uids/1]).
 -export([listen/0]).
+-export([log_burst_limit_count/0]).
+-export([log_burst_limit_window_time/0]).
 -export([log_rotate_count/0]).
 -export([log_rotate_size/0]).
 -export([loglevel/0]).
@@ -583,6 +585,14 @@ ldap_uids(Host) ->
 listen() ->
     ejabberd_config:get_option({listen, global}).
 
+-spec log_burst_limit_count() -> pos_integer().
+log_burst_limit_count() ->
+    ejabberd_config:get_option({log_burst_limit_count, global}).
+
+-spec log_burst_limit_window_time() -> pos_integer().
+log_burst_limit_window_time() ->
+    ejabberd_config:get_option({log_burst_limit_window_time, global}).
+
 -spec log_rotate_count() -> non_neg_integer().
 log_rotate_count() ->
     ejabberd_config:get_option({log_rotate_count, global}).

src/ejabberd_options.erl

@@ -222,6 +222,10 @@ opt_type(log_rotate_count) ->
     econf:non_neg_int();
 opt_type(log_rotate_size) ->
     econf:pos_int(infinity);
+opt_type(log_burst_limit_window_time) ->
+    econf:timeout(second);
+opt_type(log_burst_limit_count) ->
+    econf:pos_int();
 opt_type(loglevel) ->
     fun(N) when is_integer(N) ->
 	    (econf:and_then(
@@ -576,6 +580,8 @@ options() ->
      {listen, []},
      {log_rotate_count, 1},
      {log_rotate_size, 10*1024*1024},
+     {log_burst_limit_window_time, timer:seconds(1)},
+     {log_burst_limit_count, 500},
      {max_fsm_queue, undefined},
      {modules, []},
      {negotiation_timeout, timer:seconds(30)},
@@ -722,6 +728,8 @@ globals() ->
      loglevel,
      log_rotate_count,
      log_rotate_size,
+     log_burst_limit_count,
+     log_burst_limit_window_time,
      negotiation_timeout,
      net_ticktime,
      new_sql_schema,

src/ejabberd_options_doc.erl

@@ -815,6 +815,17 @@ doc() ->
             ?T("The size (in bytes) of a log file to trigger rotation. "
                "If set to 'infinity', log rotation is disabled. "
                "The default value is '10485760' (that is, 10 Mb).")}},
+     {log_burst_limit_count,
+      #{value => ?T("Number"),
+        desc =>
+            ?T("The number of messages to accept in "
+               "`log_burst_limit_window_time` period before starting to "
+               "drop them. Default 500")}},
+     {log_burst_limit_window_time,
+      #{value => ?T("Number"),
+        desc =>
+            ?T("The time period to rate-limit log messages "
+               "by. Defaults to 1 second.")}},
      {max_fsm_queue,
       #{value => ?T("Size"),
         desc =>