From 555ff2db4c051ce5263f8e1dfc958aad0c0d186a Mon Sep 17 00:00:00 2001 From: Mark Zealey <6083471+mzealey@users.noreply.github.com> Date: Mon, 1 Aug 2022 11:07:54 +0100 Subject: [PATCH] Add log_burst_limit_* options (#3865) * Add log_burst_limit_* options On our ejabberd deployment we were sometimes seeing more than 500 msgs/sec of legitimate traffic, however this was getting silently dropped. Provide config options to enable this limit to be configured from the config file. * Pass new logging vars in via ejabberdctl --- .github/container/ejabberdctl.template | 4 +++- ejabberdctl.template | 4 +++- src/ejabberd_logger.erl | 6 +++++- src/ejabberd_option.erl | 10 ++++++++++ src/ejabberd_options.erl | 8 ++++++++ src/ejabberd_options_doc.erl | 11 +++++++++++ 6 files changed, 40 insertions(+), 3 deletions(-) diff --git a/.github/container/ejabberdctl.template b/.github/container/ejabberdctl.template index 02284cc79..7ba20d68e 100755 --- a/.github/container/ejabberdctl.template +++ b/.github/container/ejabberdctl.template @@ -91,7 +91,9 @@ ERL_INETRC="$CONFIG_DIR"/inetrc # define ejabberd parameters EJABBERD_OPTS="$EJABBERD_OPTS\ $(sed '/^log_rotate_size/!d;s/:[ \t]*\([0-9]\{1,\}\).*/ \1/;s/:[ \t]*\(infinity\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\ -$(sed '/^log_rotate_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")" +$(sed '/^log_rotate_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\ +$(sed '/^log_burst_limit_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\ +$(sed '/^log_burst_limit_window_time/!d;s/:[ \t]*\([0-9]*[a-z]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")" [ -n "$EJABBERD_OPTS" ] && EJABBERD_OPTS="-ejabberd $EJABBERD_OPTS" EJABBERD_OPTS="-mnesia dir \"$SPOOL_DIR\" $MNESIA_OPTIONS $EJABBERD_OPTS -s ejabberd" diff --git a/ejabberdctl.template b/ejabberdctl.template index 5b9ef36cb..60b295c27 100755 --- a/ejabberdctl.template +++ b/ejabberdctl.template @@ -90,7 +90,9 @@ ERL_INETRC="$CONFIG_DIR"/inetrc # define ejabberd parameters EJABBERD_OPTS="$EJABBERD_OPTS\ $(sed '/^log_rotate_size/!d;s/:[ \t]*\([0-9]\{1,\}\).*/ \1/;s/:[ \t]*\(infinity\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\ -$(sed '/^log_rotate_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")" +$(sed '/^log_rotate_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\ +$(sed '/^log_burst_limit_count/!d;s/:[ \t]*\([0-9]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")\ +$(sed '/^log_burst_limit_window_time/!d;s/:[ \t]*\([0-9]*[a-z]*\).*/ \1/;s/^/ /' "$EJABBERD_CONFIG_PATH")" [ -n "$EJABBERD_OPTS" ] && EJABBERD_OPTS="-ejabberd $EJABBERD_OPTS" EJABBERD_OPTS="-mnesia dir \"$SPOOL_DIR\" $MNESIA_OPTIONS $EJABBERD_OPTS -s ejabberd" diff --git a/src/ejabberd_logger.erl b/src/ejabberd_logger.erl index e4fe34e0e..3d6c08650 100644 --- a/src/ejabberd_logger.erl +++ b/src/ejabberd_logger.erl @@ -266,13 +266,17 @@ start(Level) -> ErrorLog = filename:join([Dir, "error.log"]), LogRotateSize = get_integer_env(log_rotate_size, 10*1024*1024), LogRotateCount = get_integer_env(log_rotate_count, 1), + LogBurstLimitWindowTime = get_integer_env(log_burst_limit_window_time, 1000), + LogBurstLimitCount = get_integer_env(log_burst_limit_count, 500), Config = #{max_no_bytes => LogRotateSize, max_no_files => LogRotateCount, filesync_repeat_interval => no_repeat, file_check => 1000, sync_mode_qlen => 1000, drop_mode_qlen => 1000, - flush_qlen => 5000}, + flush_qlen => 5000, + burst_limit_window_time => LogBurstLimitWindowTime, + burst_limit_max_count => LogBurstLimitCount}, FmtConfig = #{legacy_header => false, time_designator => $ , max_size => 100*1024, diff --git a/src/ejabberd_option.erl b/src/ejabberd_option.erl index 29b5b40e8..063502f80 100644 --- a/src/ejabberd_option.erl +++ b/src/ejabberd_option.erl @@ -72,6 +72,8 @@ -export([ldap_tls_verify/0, ldap_tls_verify/1]). -export([ldap_uids/0, ldap_uids/1]). -export([listen/0]). +-export([log_burst_limit_count/0]). +-export([log_burst_limit_window_time/0]). -export([log_rotate_count/0]). -export([log_rotate_size/0]). -export([loglevel/0]). @@ -583,6 +585,14 @@ ldap_uids(Host) -> listen() -> ejabberd_config:get_option({listen, global}). +-spec log_burst_limit_count() -> pos_integer(). +log_burst_limit_count() -> + ejabberd_config:get_option({log_burst_limit_count, global}). + +-spec log_burst_limit_window_time() -> pos_integer(). +log_burst_limit_window_time() -> + ejabberd_config:get_option({log_burst_limit_window_time, global}). + -spec log_rotate_count() -> non_neg_integer(). log_rotate_count() -> ejabberd_config:get_option({log_rotate_count, global}). diff --git a/src/ejabberd_options.erl b/src/ejabberd_options.erl index 8837ef475..bbf218532 100644 --- a/src/ejabberd_options.erl +++ b/src/ejabberd_options.erl @@ -222,6 +222,10 @@ opt_type(log_rotate_count) -> econf:non_neg_int(); opt_type(log_rotate_size) -> econf:pos_int(infinity); +opt_type(log_burst_limit_window_time) -> + econf:timeout(second); +opt_type(log_burst_limit_count) -> + econf:pos_int(); opt_type(loglevel) -> fun(N) when is_integer(N) -> (econf:and_then( @@ -576,6 +580,8 @@ options() -> {listen, []}, {log_rotate_count, 1}, {log_rotate_size, 10*1024*1024}, + {log_burst_limit_window_time, timer:seconds(1)}, + {log_burst_limit_count, 500}, {max_fsm_queue, undefined}, {modules, []}, {negotiation_timeout, timer:seconds(30)}, @@ -722,6 +728,8 @@ globals() -> loglevel, log_rotate_count, log_rotate_size, + log_burst_limit_count, + log_burst_limit_window_time, negotiation_timeout, net_ticktime, new_sql_schema, diff --git a/src/ejabberd_options_doc.erl b/src/ejabberd_options_doc.erl index 9d4cea829..1d1dd8913 100644 --- a/src/ejabberd_options_doc.erl +++ b/src/ejabberd_options_doc.erl @@ -815,6 +815,17 @@ doc() -> ?T("The size (in bytes) of a log file to trigger rotation. " "If set to 'infinity', log rotation is disabled. " "The default value is '10485760' (that is, 10 Mb).")}}, + {log_burst_limit_count, + #{value => ?T("Number"), + desc => + ?T("The number of messages to accept in " + "`log_burst_limit_window_time` period before starting to " + "drop them. Default 500")}}, + {log_burst_limit_window_time, + #{value => ?T("Number"), + desc => + ?T("The time period to rate-limit log messages " + "by. Defaults to 1 second.")}}, {max_fsm_queue, #{value => ?T("Size"), desc =>