Chapril/xmpp.chapril.org-ejabberd
25
1
Fork 0
mirror of https://github.com/processone/ejabberd.git synced 2024-12-16 17:15:55 +01:00
Code Releases Activity

Improve database and caching in mod_shared_roster

Browse Source 
This makes us keep cache of groups that use wildcards no matter
of cache settings, and tries to not same fetch data multiple times
in roster get operations.
This commit is contained in:
Paweł Chmielowski 2021-04-16 10:34:00 +02:00
parent 54916caf65
commit 5b0f0d8352
1 changed files with 169 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

333
src/mod_shared_roster.erl
View File

@ -150,18 +150,17 @@ depends(_Host, _Opts) ->
-spec init_cache(module(), binary(), gen_mod:opts()) -> ok. -spec init_cache(module(), binary(), gen_mod:opts()) -> ok.
init_cache(Mod, Host, Opts) -> init_cache(Mod, Host, Opts) ->
ets_cache:new(?SPECIAL_GROUPS_CACHE, [{max_size, 4}]),
case use_cache(Mod, Host) of case use_cache(Mod, Host) of
true -> true ->
CacheOpts = cache_opts(Opts), CacheOpts = cache_opts(Opts),
ets_cache:new(?GROUP_OPTS_CACHE, CacheOpts), ets_cache:new(?GROUP_OPTS_CACHE, CacheOpts),
ets_cache:new(?USER_GROUPS_CACHE, CacheOpts), ets_cache:new(?USER_GROUPS_CACHE, CacheOpts),
ets_cache:new(?GROUP_EXPLICIT_USERS_CACHE, CacheOpts), ets_cache:new(?GROUP_EXPLICIT_USERS_CACHE, CacheOpts);
ets_cache:new(?SPECIAL_GROUPS_CACHE, CacheOpts);
false -> false ->
ets_cache:delete(?GROUP_OPTS_CACHE), ets_cache:delete(?GROUP_OPTS_CACHE),
ets_cache:delete(?USER_GROUPS_CACHE), ets_cache:delete(?USER_GROUPS_CACHE),
ets_cache:delete(?GROUP_EXPLICIT_USERS_CACHE), ets_cache:delete(?GROUP_EXPLICIT_USERS_CACHE)
ets_cache:delete(?SPECIAL_GROUPS_CACHE)
end. end.
-spec cache_opts(gen_mod:opts()) -> [proplists:property()]. -spec cache_opts(gen_mod:opts()) -> [proplists:property()].
@ -186,45 +185,36 @@ cache_nodes(Mod, Host) ->
end. end.
-spec get_user_roster([#roster{}], {binary(), binary()}) -> [#roster{}]. -spec get_user_roster([#roster{}], {binary(), binary()}) -> [#roster{}].
get_user_roster(Items, US) -> get_user_roster(Items, {U, S} = US) ->
{U, S} = US, {DisplayedGroups, Cache} = get_user_displayed_groups(US),
DisplayedGroups = get_user_displayed_groups(US), SRUsers = lists:foldl(
SRUsers = lists:foldl(fun (Group, Acc1) -> fun(Group, Acc1) ->
GroupLabel = get_group_label(S, Group), %++ GroupLabel = get_group_label_cached(S, Group, Cache),
lists:foldl(fun (User, Acc2) -> lists:foldl(
if User == US -> Acc2; fun(User, Acc2) ->
true -> if User == US -> Acc2;
dict:append(User, true ->
GroupLabel, dict:append(User, GroupLabel, Acc2)
Acc2) end
end end,
end, Acc1, get_group_users_cached(S, Group, Cache))
Acc1, get_group_users(S, Group)) end,
end, dict:new(), DisplayedGroups),
dict:new(), DisplayedGroups), {NewItems1, SRUsersRest} = lists:mapfoldl(
{NewItems1, SRUsersRest} = lists:mapfoldl(fun (Item, fun(Item, SRUsers1) ->
SRUsers1) -> {_, _, {U1, S1, _}} = Item#roster.usj,
{_, _, {U1, S1, _}} = US1 = {U1, S1},
Item#roster.usj, case dict:find(US1, SRUsers1) of
US1 = {U1, S1}, {ok, GroupLabels} ->
case dict:find(US1, {Item#roster{subscription = both,
SRUsers1) groups = Item#roster.groups ++ GroupLabels,
of ask = none},
{ok, GroupLabels} -> dict:erase(US1, SRUsers1)};
{Item#roster{subscription error ->
= {Item, SRUsers1}
both, end
groups = end,
Item#roster.groups ++ GroupLabels, SRUsers, Items),
ask =
none},
dict:erase(US1,
SRUsers1)};
error ->
{Item, SRUsers1}
end
end,
SRUsers, Items),
SRItems = [#roster{usj = {U, S, {U1, S1, <<"">>}}, SRItems = [#roster{usj = {U, S, {U1, S1, <<"">>}},
us = US, jid = {U1, S1, <<"">>}, us = US, jid = {U1, S1, <<"">>},
name = get_rosteritem_name(U1, S1), name = get_rosteritem_name(U1, S1),
@ -261,7 +251,7 @@ process_item(RosterItem, Host) ->
{UserTo, ServerTo, ResourceTo} = RosterItem#roster.jid, {UserTo, ServerTo, ResourceTo} = RosterItem#roster.jid,
NameTo = RosterItem#roster.name, NameTo = RosterItem#roster.name,
USTo = {UserTo, ServerTo}, USTo = {UserTo, ServerTo},
DisplayedGroups = get_user_displayed_groups(USFrom), {DisplayedGroups, Cache} = get_user_displayed_groups(USFrom),
CommonGroups = lists:filter(fun (Group) -> CommonGroups = lists:filter(fun (Group) ->
is_user_in_group(USTo, Group, Host) is_user_in_group(USTo, Group, Host)
end, end,
@ -271,7 +261,7 @@ process_item(RosterItem, Host) ->
%% Roster item cannot be removed: We simply reset the original groups: %% Roster item cannot be removed: We simply reset the original groups:
_ when RosterItem#roster.subscription == remove -> _ when RosterItem#roster.subscription == remove ->
GroupLabels = lists:map(fun (Group) -> GroupLabels = lists:map(fun (Group) ->
get_group_label(Host, Group) get_group_label_cached(Host, Group, Cache)
end, end,
CommonGroups), CommonGroups),
RosterItem#roster{subscription = both, ask = none, RosterItem#roster{subscription = both, ask = none,
@ -352,18 +342,16 @@ get_jid_info({Subscription, Ask, Groups}, User, Server,
US = {LUser, LServer}, US = {LUser, LServer},
{U1, S1, _} = jid:tolower(JID), {U1, S1, _} = jid:tolower(JID),
US1 = {U1, S1}, US1 = {U1, S1},
DisplayedGroups = get_user_displayed_groups(US), {DisplayedGroups, Cache} = get_user_displayed_groups(US),
SRUsers = lists:foldl(fun (Group, Acc1) -> SRUsers = lists:foldl(
GroupLabel = get_group_label(LServer, Group), %++ fun(Group, Acc1) ->
lists:foldl(fun (User1, Acc2) -> GroupLabel = get_group_label_cached(LServer, Group, Cache), %++
dict:append(User1, lists:foldl(
GroupLabel, fun(User1, Acc2) ->
Acc2) dict:append(User1, GroupLabel, Acc2)
end, end, Acc1, get_group_users_cached(LServer, Group, Cache))
Acc1, end,
get_group_users(LServer, Group)) dict:new(), DisplayedGroups),
end,
dict:new(), DisplayedGroups),
case dict:find(US1, SRUsers) of case dict:find(US1, SRUsers) of
{ok, GroupLabels} -> {ok, GroupLabels} ->
NewGroups = if Groups == [] -> GroupLabels; NewGroups = if Groups == [] -> GroupLabels;
@ -398,7 +386,7 @@ process_subscription(Direction, User, Server, JID,
{U1, S1, _} = {U1, S1, _} =
jid:tolower(jid:remove_resource(JID)), jid:tolower(jid:remove_resource(JID)),
US1 = {U1, S1}, US1 = {U1, S1},
DisplayedGroups = get_user_displayed_groups(US), {DisplayedGroups, _} = get_user_displayed_groups(US),
SRUsers = lists:usort(lists:flatmap(fun (Group) -> SRUsers = lists:usort(lists:flatmap(fun (Group) ->
get_group_users(LServer, Group) get_group_users(LServer, Group)
end, end,
@ -425,10 +413,16 @@ create_group(Host, Group) ->
create_group(Host, Group, Opts) -> create_group(Host, Group, Opts) ->
Mod = gen_mod:db_mod(Host, ?MODULE), Mod = gen_mod:db_mod(Host, ?MODULE),
case proplists:get_value(all_users, Opts, false) orelse
proplists:get_value(online_users, Opts, false) of
true ->
update_wildcard_cache(Host, Group, Opts);
_ ->
ok
end,
case use_cache(Mod, Host) of case use_cache(Mod, Host) of
true -> true ->
ets_cache:insert(?GROUP_OPTS_CACHE, {Host, Group}, Opts, cache_nodes(Mod, Host)), ets_cache:insert(?GROUP_OPTS_CACHE, {Host, Group}, Opts, cache_nodes(Mod, Host));
ets_cache:clear(?SPECIAL_GROUPS_CACHE, cache_nodes(Mod, Host));
_ -> _ ->
ok ok
end, end,
@ -436,18 +430,32 @@ create_group(Host, Group, Opts) ->
delete_group(Host, Group) -> delete_group(Host, Group) ->
Mod = gen_mod:db_mod(Host, ?MODULE), Mod = gen_mod:db_mod(Host, ?MODULE),
update_wildcard_cache(Host, Group, []),
case use_cache(Mod, Host) of case use_cache(Mod, Host) of
true -> true ->
ets_cache:delete(?GROUP_OPTS_CACHE, {Host, Group}, cache_nodes(Mod, Host)), ets_cache:delete(?GROUP_OPTS_CACHE, {Host, Group}, cache_nodes(Mod, Host)),
ets_cache:clear(?USER_GROUPS_CACHE, cache_nodes(Mod, Host)), ets_cache:clear(?USER_GROUPS_CACHE, cache_nodes(Mod, Host)),
ets_cache:clear(?GROUP_EXPLICIT_USERS_CACHE, cache_nodes(Mod, Host)), ets_cache:delete(?GROUP_EXPLICIT_USERS_CACHE, {Host, Group}, cache_nodes(Mod, Host));
ets_cache:clear(?SPECIAL_GROUPS_CACHE, cache_nodes(Mod, Host));
_ -> _ ->
ok ok
end, end,
Mod:delete_group(Host, Group). Mod:delete_group(Host, Group).
get_groups_opts_cached(Host1, Group1, Cache) ->
{Host, Group} = split_grouphost(Host1, Group1),
case Cache of
#{{Group, Host} := Opts} ->
{Opts, Cache};
_ ->
Opts = get_group_opts_int(Host, Group),
{Opts, Cache#{{Group, Host} => Opts}}
end.
get_group_opts(Host1, Group1) -> get_group_opts(Host1, Group1) ->
{Host, Group} = split_grouphost(Host1, Group1),
get_group_opts_int(Host, Group).
get_group_opts_int(Host1, Group1) ->
{Host, Group} = split_grouphost(Host1, Group1), {Host, Group} = split_grouphost(Host1, Group1),
Mod = gen_mod:db_mod(Host, ?MODULE), Mod = gen_mod:db_mod(Host, ?MODULE),
Res = case use_cache(Mod, Host) of Res = case use_cache(Mod, Host) of
@ -470,11 +478,11 @@ get_group_opts(Host1, Group1) ->
set_group_opts(Host, Group, Opts) -> set_group_opts(Host, Group, Opts) ->
Mod = gen_mod:db_mod(Host, ?MODULE), Mod = gen_mod:db_mod(Host, ?MODULE),
update_wildcard_cache(Host, Group, Opts),
case use_cache(Mod, Host) of case use_cache(Mod, Host) of
true -> true ->
ets_cache:delete(?GROUP_OPTS_CACHE, {Host, Group}, cache_nodes(Mod, Host)), ets_cache:delete(?GROUP_OPTS_CACHE, {Host, Group}, cache_nodes(Mod, Host)),
ets_cache:insert(?GROUP_OPTS_CACHE, {Host, Group}, Opts, cache_nodes(Mod, Host)), ets_cache:insert(?GROUP_OPTS_CACHE, {Host, Group}, Opts, cache_nodes(Mod, Host));
ets_cache:clear(?SPECIAL_GROUPS_CACHE, cache_nodes(Mod, Host));
_ -> _ ->
ok ok
end, end,
@ -493,13 +501,13 @@ get_user_groups(US) ->
false -> false ->
Mod:get_user_groups(US, Host) Mod:get_user_groups(US, Host)
end, end,
UG ++ get_special_users_groups(Host). UG ++ get_groups_with_wildcards(Host, both).
is_group_enabled(Host1, Group1) -> get_group_opt_cached(Host, Group, Opt, Default, Cache) ->
{Host, Group} = split_grouphost(Host1, Group1), case get_groups_opts_cached(Host, Group, Cache) of
case get_group_opts(Host, Group) of {error, _} -> Default;
error -> false; {Opts, _} ->
Opts -> not lists:member(disabled, Opts) proplists:get_value(Opt, Opts, Default)
end. end.
%% @spec (Host::string(), Group::string(), Opt::atom(), Default) -> OptValue | Default %% @spec (Host::string(), Group::string(), Opt::atom(), Default) -> OptValue | Default
@ -507,16 +515,17 @@ get_group_opt(Host, Group, Opt, Default) ->
case get_group_opts(Host, Group) of case get_group_opts(Host, Group) of
error -> Default; error -> Default;
Opts -> Opts ->
case lists:keysearch(Opt, 1, Opts) of proplists:get_value(Opt, Opts, Default)
{value, {_, Val}} -> Val;
false -> Default
end
end. end.
get_online_users(Host) -> get_online_users(Host) ->
lists:usort([{U, S} lists:usort([{U, S}
|| {U, S, _} <- ejabberd_sm:get_vh_session_list(Host)]). || {U, S, _} <- ejabberd_sm:get_vh_session_list(Host)]).
get_group_users_cached(Host, Group, Cache) ->
{Opts, _} = get_groups_opts_cached(Host, Group, Cache),
get_group_users(Host, Group, Opts).
get_group_users(Host1, Group1) -> get_group_users(Host1, Group1) ->
{Host, Group} = split_grouphost(Host1, Group1), {Host, Group} = split_grouphost(Host1, Group1),
get_group_users(Host, Group, get_group_opts(Host, Group)). get_group_users(Host, Group, get_group_opts(Host, Group)).
@ -547,82 +556,69 @@ get_group_explicit_users(Host, Group) ->
Mod:get_group_explicit_users(Host, Group) Mod:get_group_explicit_users(Host, Group)
end. end.
get_group_label(Host1, Group1) -> get_group_label_cached(Host, Group, Cache) ->
{Host, Group} = split_grouphost(Host1, Group1), get_group_opt_cached(Host, Group, label, Group, Cache).
get_group_opt(Host, Group, label, Group).
%% Get list of names of groups that have @all@/@online@/etc in the memberlist -spec update_wildcard_cache(binary(), binary(), list()) -> ok.
get_special_users_groups(Host) -> update_wildcard_cache(Host, Group, NewOpts) ->
Extract =
fun() ->
lists:filtermap(
fun({Group, Opts}) ->
case proplists:get_value(all_users, Opts, false) orelse
proplists:get_value(online_users, Opts, false) of
true -> {true, Group};
false -> false
end
end,
groups_with_opts(Host))
end,
Mod = gen_mod:db_mod(Host, ?MODULE), Mod = gen_mod:db_mod(Host, ?MODULE),
case use_cache(Mod, Host) of Online = get_groups_with_wildcards(Host, online),
true -> Both = get_groups_with_wildcards(Host, both),
ets_cache:lookup( IsOnline = proplists:get_value(online_users, NewOpts, false),
?SPECIAL_GROUPS_CACHE, {Host, false}, IsAll = proplists:get_value(all_users, NewOpts, false),
fun() ->
{cache, Extract()}
end);
false ->
Extract()
end.
OnlineUpdated = lists:member(Group, Online) /= IsOnline,
BothUpdated = lists:member(Group, Both) /= (IsOnline orelse IsAll),
%% Get list of names of groups that have @online@ in the memberlist if
get_special_users_groups_online(Host) -> OnlineUpdated ->
Extract = NewOnline = case IsOnline of
fun() -> true -> [Group | Online];
lists:filtermap( _ -> Online -- [Group]
fun({Group, Opts}) -> end,
case proplists:get_value(online_users, Opts, false) of ets_cache:update(?SPECIAL_GROUPS_CACHE, {Host, online},
true -> {true, Group}; NewOnline, fun() -> ok end, cache_nodes(Mod, Host));
false -> false true -> ok
end
end,
groups_with_opts(Host))
end, end,
Mod = gen_mod:db_mod(Host, ?MODULE), if
case use_cache(Mod, Host) of BothUpdated ->
true -> NewBoth = case IsOnline orelse IsAll of
ets_cache:lookup( true -> [Group | Both];
?SPECIAL_GROUPS_CACHE, {Host, true}, _ -> Both -- [Group]
fun() -> end,
{cache, Extract()} ets_cache:update(?SPECIAL_GROUPS_CACHE, {Host, both},
end); NewBoth, fun() -> ok end, cache_nodes(Mod, Host));
false -> true -> ok
Extract() end,
end. ok.
-spec get_groups_with_wildcards(binary(), online | both) -> list(binary()).
get_groups_with_wildcards(Host, Type) ->
ets_cache:lookup(
?SPECIAL_GROUPS_CACHE, {Host, Type},
fun() ->
Res = lists:filtermap(
fun({Group, Opts}) ->
case proplists:get_value(online_users, Opts, false) orelse
(Type == both andalso proplists:get_value(all_users, Opts, false)) of
true -> {true, Group};
false -> false
end
end,
groups_with_opts(Host)),
{cache, Res}
end).
%% Given two lists of groupnames and their options, %% Given two lists of groupnames and their options,
%% return the list of displayed groups to the second list %% return the list of displayed groups to the second list
displayed_groups(GroupsOpts, SelectedGroupsOpts) -> displayed_groups(GroupsOpts, SelectedGroupsOpts) ->
DisplayedGroups = lists:usort(lists:flatmap(fun DisplayedGroups = lists:usort(lists:flatmap(
({_Group, Opts}) -> fun
[G ({_Group, Opts}) ->
|| G [G || G <- proplists:get_value(displayed_groups, Opts, []),
<- proplists:get_value(displayed_groups, not lists:member(disabled, Opts)]
Opts, end, SelectedGroupsOpts)),
[]), [G || G <- DisplayedGroups, not lists:member(disabled, proplists:get_value(G, GroupsOpts, []))].
not
lists:member(disabled,
Opts)]
end,
SelectedGroupsOpts)),
[G
|| G <- DisplayedGroups,
not
lists:member(disabled,
proplists:get_value(G, GroupsOpts, []))].
%% Given a list of group names with options, %% Given a list of group names with options,
%% for those that have @all@ in memberlist, %% for those that have @all@ in memberlist,
@ -645,24 +641,35 @@ get_user_displayed_groups(LUser, LServer, GroupsOpts) ->
%% @doc Get the list of groups that are displayed to this user %% @doc Get the list of groups that are displayed to this user
get_user_displayed_groups(US) -> get_user_displayed_groups(US) ->
Host = element(2, US), Host = element(2, US),
DisplayedGroups1 = lists:usort(lists:flatmap(fun {Groups, Cache} =
(Group) -> lists:foldl(
case fun(Group, {Groups, Cache}) ->
is_group_enabled(Host, case get_groups_opts_cached(Host, Group, Cache) of
Group) {error, Cache2} ->
of {Groups, Cache2};
true -> {Opts, Cache3} ->
get_group_opt(Host, case lists:member(disabled, Opts) of
Group, false ->
displayed_groups, {proplists:get_value(displayed_groups, Opts, []) ++ Groups, Cache3};
[]); _ ->
false -> [] {Groups, Cache3}
end end
end, end
get_user_groups(US))), end, {[], #{}}, get_user_groups(US)),
[Group lists:foldl(
|| Group <- DisplayedGroups1, fun(Group, {Groups0, Cache0}) ->
is_group_enabled(Host, Group)]. case get_groups_opts_cached(Host, Group, Cache0) of
{error, Cache1} ->
{Groups0, Cache1};
{Opts, Cache2} ->
case lists:member(disabled, Opts) of
false ->
{[Group|Groups0], Cache2};
_ ->
{Groups0, Cache2}
end
end
end, {[], Cache}, lists:usort(Groups)).
is_user_in_group(US, Group, Host) -> is_user_in_group(US, Group, Host) ->
Mod = gen_mod:db_mod(Host, ?MODULE), Mod = gen_mod:db_mod(Host, ?MODULE),
@ -865,7 +872,6 @@ unset_presence(LUser, LServer, Resource, Status) ->
[LUser, LServer, Resource, Status, length(Resources)]), [LUser, LServer, Resource, Status, length(Resources)]),
case length(Resources) of case length(Resources) of
0 -> 0 ->
OnlineGroups = get_special_users_groups_online(LServer),
lists:foreach( lists:foreach(
fun(OG) -> fun(OG) ->
DisplayedToGroups = displayed_to_groups(OG, LServer), DisplayedToGroups = displayed_to_groups(OG, LServer),
@ -873,8 +879,7 @@ unset_presence(LUser, LServer, Resource, Status) ->
LServer, remove, DisplayedToGroups), LServer, remove, DisplayedToGroups),
push_displayed_to_user(LUser, LServer, push_displayed_to_user(LUser, LServer,
LServer, remove, DisplayedToGroups) LServer, remove, DisplayedToGroups)
end, end, get_groups_with_wildcards(LServer, online));
OnlineGroups);
_ -> ok _ -> ok
end. end.