Chapril/xmpp.chapril.org-ejabberd
25
1
Fork 0
mirror of https://github.com/processone/ejabberd.git synced 2024-11-30 16:36:29 +01:00
Code Releases Activity

Support for roster versioning (EJAB-964)

Browse Source 
Introduces two options for mod_roster and mod_roster_odbc:
- {versioning, true | false}   Enable or disable roster versioning on ejabberd.
- {store_current_id, true | false}   If true, the current roster version is stored on DB (internal or odbc). Otherwise it is calculated on the fly each time.

Performance:
Setting store_current_id to true should help in reducing the load for both ejabberd and the DB.

Details: 
If store_current_id is false,  the roster version is a hash of the entire roster. If store_current_id is true, the roster version is a hash, but of the current time
(this has to do with transactional semantics; we need to perform both the roster update and the version update on the same transaction, but we don't   
have the entire roster when we are changing a single item on DB. Loading it there requires significant changes to be introduced, so I opted for this simpler approach).

In either case, there is no difference for the clients, the roster version ID is opaque.

IMPORTANT:
mod_shared_roster is not compatible with the option 'store_current_id'.  Shared roster and roster versioning can be both enabled, but store_current_id MUST be set to false.

SVN Revision: 2428
This commit is contained in:
Pablo Polvorin 2009-08-06 15:45:13 +00:00
parent 6aa3706bec
commit 1b85310f1a
9 changed files with 336 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/ejabberd_c2s.erl
View File

@ -310,12 +310,17 @@ wait_for_stream({xmlstreamstart, #xmlel{ns = NS} = Opening}, StateData) ->
_ ->
case StateData#state.resource of
undefined ->
RosterVersioningFeature =
case roster_versioning:is_enabled(ServerB) of
true -> [roster_versioning:stream_feature()];
false -> []
end,
send_element(
StateData,
exmpp_stream:features([
exmpp_server_binding:feature(),
exmpp_server_session:feature()
])),
| RosterVersioningFeature])),
fsm_next_state(wait_for_bind,
StateData#state{
server = ServerB,

125
src/mod_roster.erl
View File

@ -42,7 +42,8 @@
get_jid_info/4,
item_to_xml/1,
webadmin_page/3,
webadmin_user/4]).
webadmin_user/4,
roster_versioning_enabled/1]).
-include_lib("exmpp/include/exmpp.hrl").
@ -74,8 +75,11 @@ start(Host, Opts) when is_list(Host) ->
IQDisc = gen_mod:get_opt(iqdisc, Opts, one_queue),
mnesia:create_table(roster,[{disc_copies, [node()]},
{attributes, record_info(fields, roster)}]),
mnesia:create_table(roster_version, [{disc_copies, [node()]},
{attributes, record_info(fields, roster_version)}]),
update_table(),
mnesia:add_table_index(roster, us),
mnesia:add_table_index(roster_version, us),
ejabberd_hooks:add(roster_get, HostB,
?MODULE, get_user_roster, 50),
ejabberd_hooks:add(roster_in_subscription, HostB,
@ -127,6 +131,16 @@ stop(Host) when is_list(Host) ->
gen_iq_handler:remove_iq_handler(ejabberd_sm, HostB,
?NS_ROSTER).
%% @spec (Host) -> true | false
%% @type Host = binary()
roster_versioning_enabled(Host) ->
gen_mod:get_module_opt(binary_to_list(Host), ?MODULE, versioning, false).
%% @spec (Host) -> true | false
%% @type Host = binary()
roster_version_on_db(Host) ->
gen_mod:get_module_opt(binary_to_list(Host), ?MODULE, store_current_id, false).
%% @spec (From, To, IQ_Rec) -> IQ_Result
%% From = exmpp_jid:jid()
%% To = exmpp_jid:jid()
@ -156,23 +170,90 @@ process_local_iq(From, To, #iq{type = set} = IQ_Rec)
when ?IS_JID(From), ?IS_JID(To), ?IS_IQ_RECORD(IQ_Rec) ->
process_iq_set(From, To, IQ_Rec).
roster_hash(Items) ->
sha:sha(term_to_binary(
lists:sort(
[R#roster{groups = lists:sort(Grs)} ||
R = #roster{groups = Grs} <- Items]))).
roster_version(LServer ,LUser) ->
US = {LUser, LServer},
case roster_version_on_db(LServer) of
true ->
case mnesia:dirty_read(roster_version, US) of
[#roster_version{version =V}] -> V;
[] -> not_found
end;
false ->
roster_hash(ejabberd_hooks:run_fold(roster_get, LServer, [], [US]))
end.
%% @spec (From, To, IQ_Rec) -> IQ_Result
%% From = exmpp_jid:jid()
%% To = exmpp_jid:jid()
%% IQ_Rec = exmpp_iq:iq()
%% IQ_Result = exmpp_iq:iq()
%% Load roster from DB only if neccesary
%% It is neccesary if
%% - roster versioning is disabled in server OR
%% - roster versioning is not used by the client OR
%% - roster versioning is used by server and client BUT the server isn't storing version IDs on db OR
%% - the roster version from client don't match current version
process_iq_get(From, To, IQ_Rec) ->
US = {exmpp_jid:prep_node(From), exmpp_jid:prep_domain(From)},
case catch ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [], [US]) of
Items when is_list(Items) ->
XItems = lists:map(fun item_to_xml/1, Items),
Result = #xmlel{ns = ?NS_ROSTER, name = 'query',
children = XItems},
exmpp_iq:result(IQ_Rec, Result);
_ ->
exmpp_iq:error(IQ_Rec, 'internal-server-error')
end.
US = {_, LServer} = {exmpp_jid:prep_node(From), exmpp_jid:prep_domain(From)},
try
{ItemsToSend, VersionToSend} =
case {exmpp_xml:get_attribute_as_list(exmpp_iq:get_request(IQ_Rec), ver, not_found),
roster_versioning_enabled(LServer),
roster_version_on_db(LServer)} of
{not_found, _ , _} ->
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [], [US])), false};
{_, false, _} ->
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [], [US])), false};
{RequestedVersion, true, true} ->
%% Retrieve version from DB. Only load entire roster
%% when neccesary.
case mnesia:dirty_read(roster_version, US) of
[#roster_version{version = RequestedVersion}] ->
{false, false};
[#roster_version{version = NewVersion}] ->
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [], [US])), NewVersion};
[] ->
RosterVersion = sha:sha(term_to_binary(now())),
mnesia:dirty_write(#roster_version{us = US, version = RosterVersion}),
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [], [US])), RosterVersion}
end;
{RequestedVersion, true, false} ->
RosterItems = ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [] , [US]),
case roster_hash(RosterItems) of
RequestedVersion ->
{false, false};
New ->
{lists:map(fun item_to_xml/1, RosterItems), New}
end
end,
case {ItemsToSend, VersionToSend} of
{false, false} ->
exmpp_iq:result(IQ_Rec);
{Items, false} ->
exmpp_iq:result(IQ_Rec, exmpp_xml:element(?NS_ROSTER, 'query', [] , Items));
{Items, Version} ->
exmpp_iq:result(IQ_Rec, exmpp_xml:element(?NS_ROSTER, 'query', [?XMLATTR('ver', Version)], Items))
end
catch
_:_ ->
exmpp_iq:error(IQ_Rec, 'internal-server-error')
end.
%% @spec (Acc, US) -> New_Acc
%% Acc = [rosteritem()]
@ -279,6 +360,10 @@ process_item_set(From, To, #xmlel{} = El) ->
%% subscription information from there:
Item3 = ejabberd_hooks:run_fold(roster_process_item,
exmpp_jid:prep_domain(From), Item2, [exmpp_jid:prep_domain(From)]),
case roster_version_on_db(LServer) of
true -> mnesia:write(#roster_version{us = {LUser, LServer}, version = sha:sha(term_to_binary(now()))});
false -> ok
end,
{Item, Item3}
end,
case mnesia:transaction(F) of
@ -389,9 +474,15 @@ push_item(User, Server, From, Item)
[{item,
Item#roster.jid,
Item#roster.subscription}]}),
lists:foreach(fun(Resource) ->
push_item(User, Server, Resource, From, Item)
end, ejabberd_sm:get_user_resources(User, Server)).
case roster_versioning_enabled(Server) of
true ->
roster_versioning:push_item(Server, User, From, Item, roster_version(Server, User));
false ->
lists:foreach(fun(Resource) ->
push_item(User, Server, Resource, From, Item)
end, ejabberd_sm:get_user_resources(User, Server))
end.
%% @spec (User, Server, Resource, From, Item) -> term()
%% User = binary()
@ -548,6 +639,10 @@ process_subscription(Direction, User, Server, JID1, Type, Reason)
ask = Pending,
askmessage = AskBinary},
mnesia:write(NewItem),
case roster_version_on_db(Server) of
true -> mnesia:write(#roster_version{us = {User, Server}, version = sha:sha(term_to_binary(now()))});
false -> ok
end,
{{push, NewItem}, AutoReply}
end
end,

2
src/mod_roster.hrl
View File

@ -29,3 +29,5 @@
askmessage = <<>>,
xs = []}).
-record(roster_version, {us,
version}).

117
src/mod_roster_odbc.erl
View File

@ -41,7 +41,8 @@
remove_user/2,
get_jid_info/4,
webadmin_page/3,
webadmin_user/4]).
webadmin_user/4,
roster_versioning_enabled/1]).
-include_lib("exmpp/include/exmpp.hrl").
@ -102,6 +103,12 @@ stop(Host) ->
gen_iq_handler:remove_iq_handler(ejabberd_sm, HostB, ?NS_ROSTER).
roster_versioning_enabled(Host) ->
gen_mod:get_module_opt(binary_to_list(Host), ?MODULE, versioning, false).
roster_version_on_db(Host) ->
gen_mod:get_module_opt(binary_to_list(Host), ?MODULE, store_current_id, false).
process_iq(From, To, IQ_Rec) ->
LServer = exmpp_jid:prep_domain_as_list(From),
case lists:member(LServer, ?MYHOSTS) of
@ -118,17 +125,84 @@ process_local_iq(From, To, #iq{type = set} = IQ_Rec) ->
roster_hash(Items) ->
sha:sha(term_to_binary(
lists:sort(
[R#roster{groups = lists:sort(Grs)} ||
R = #roster{groups = Grs} <- Items]))).
roster_version(LServer ,LUser) ->
US = {LUser, LServer},
case roster_version_on_db(LServer) of
true ->
case odbc_queries:get_roster_version(ejabberd_odbc:escape(LServer), ejabberd_odbc:escape(LUser)) of
{selected, ["version"], [{Version}]} -> Version;
{selected, ["version"], []} -> not_found
end;
false ->
roster_hash(ejabberd_hooks:run_fold(roster_get, LServer, [], [US]))
end.
%% Load roster from DB only if neccesary.
%% It is neccesary if
%% - roster versioning is disabled in server OR
%% - roster versioning is not used by the client OR
%% - roster versioning is used by server and client, BUT the server isn't storing versions on db OR
%% - the roster version from client don't match current version.
process_iq_get(From, To, IQ_Rec) ->
US = {exmpp_jid:prep_node(From), exmpp_jid:prep_domain(From)},
case catch ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [], [US]) of
Items when is_list(Items) ->
XItems = lists:map(fun item_to_xml/1, Items),
Result = #xmlel{ns = ?NS_ROSTER, name = 'query',
children = XItems},
exmpp_iq:result(IQ_Rec, Result);
_ ->
exmpp_iq:error(IQ_Rec, 'internal-server-error')
end.
US = {LUser, LServer} = {exmpp_jid:prep_node(From), exmpp_jid:prep_domain(From)},
try
{ItemsToSend, VersionToSend} =
case {exmpp_xml:get_attribute_as_list(exmpp_iq:get_request(IQ_Rec), ver, not_found),
roster_versioning_enabled(LServer),
roster_version_on_db(LServer)} of
{not_found, _ , _} ->
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [], [US])), false};
{_, false, _} ->
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [], [US])), false};
{RequestedVersion, true, true} ->
%% Retrieve version from DB. Only load entire roster
%% when neccesary.
case odbc_queries:get_roster_version(ejabberd_odbc:escape(LServer), ejabberd_odbc:escape(LUser)) of
{selected, ["version"], [{RequestedVersion}]} ->
{false, false};
{selected, ["version"], [{NewVersion}]} ->
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [], [US])), NewVersion};
{selected, ["version"], []} ->
RosterVersion = sha:sha(term_to_binary(now())),
{atomic, {updated,1}} = odbc_queries:sql_transaction(binary_to_list(LServer), fun() ->
odbc_queries:set_roster_version(ejabberd_odbc:escape(LUser), RosterVersion)
end),
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [], [US])), RosterVersion}
end;
{RequestedVersion, true, false} ->
RosterItems = ejabberd_hooks:run_fold(roster_get, exmpp_jid:prep_domain(To), [] , [US]),
case roster_hash(RosterItems) of
RequestedVersion ->
{false, false};
New ->
{lists:map(fun item_to_xml/1, RosterItems), New}
end
end,
case {ItemsToSend, VersionToSend} of
{false, false} ->
exmpp_iq:result(IQ_Rec);
{Items, false} ->
exmpp_iq:result(IQ_Rec, exmpp_xml:element(?NS_ROSTER, 'query', [] , Items));
{Items, Version} ->
exmpp_iq:result(IQ_Rec, exmpp_xml:element(?NS_ROSTER, 'query', [?XMLATTR('ver', Version)], Items))
end
catch
_:_ ->
exmpp_iq:error(IQ_Rec, 'internal-server-error')
end.
get_user_roster(Acc, {LUser, LServer}) ->
Items = get_roster(LUser, LServer),
@ -263,6 +337,11 @@ process_item_set(From, To, #xmlel{} = El) ->
%% subscription information from there:
Item3 = ejabberd_hooks:run_fold(roster_process_item,
exmpp_jid:prep_domain(From), Item2, [exmpp_jid:prep_domain(From)]),
case roster_version_on_db(Server) of
true -> odbc_queries:set_roster_version(Username, sha:sha(term_to_binary(now())));
false -> ok
end,
{Item, Item3}
end,
case odbc_queries:sql_transaction(LServer, F) of
@ -352,9 +431,15 @@ push_item(User, Server, From, Item) when is_binary(User), is_binary(Server) ->
[{item,
Item#roster.jid,
Item#roster.subscription}]}),
lists:foreach(fun(Resource) ->
push_item(User, Server, Resource, From, Item)
end, ejabberd_sm:get_user_resources(User, Server)).
case roster_versioning_enabled(Server) of
true ->
roster_versioning:push_item(Server, User, From, Item, roster_version(Server, User));
false ->
lists:foreach(fun(Resource) ->
push_item(User, Server, Resource, From, Item)
end, ejabberd_sm:get_user_resources(User, Server))
end.
% TODO: don't push to those who not load roster
push_item(User, Server, Resource, From, Item) ->
@ -492,6 +577,10 @@ process_subscription(Direction, User, Server, JID1, Type, Reason)
askmessage = AskBinary},
ItemVals = record_to_string(NewItem),
odbc_queries:roster_subscribe(LServer, Username, SJID, ItemVals),
case roster_version_on_db(Server) of
true -> odbc_queries:set_roster_version(Username, sha:sha(term_to_binary(now())));
false -> ok
end,
{{push, NewItem}, AutoReply}
end
end,

15
src/odbc/mssql2000.sql
View File

@ -209,6 +209,14 @@ CREATE TABLE [dbo].[privacy_list_data] (
) ON [PRIMARY]
GO
/* Not tested on mssql */
CREATE TABLE [dbo].[roster_version] (
[username] [varchar] (250) NOT NULL ,
[version] [varchar] (64) NOT NULL
) ON [PRIMARY]
GO
/* Constraints to add:
- id in privacy_list is a SERIAL autogenerated number
- id in privacy_list_data must exist in the table privacy_list */
@ -244,6 +252,13 @@ ALTER TABLE [dbo].[users] WITH NOCHECK ADD
) WITH FILLFACTOR = 90 ON [PRIMARY]
GO
ALTER TABLE [dbo].[roster_version] WITH NOCHECK ADD
CONSTRAINT [PK_roster_version] PRIMARY KEY CLUSTERED
(
[username]
) WITH FILLFACTOR = 90 ON [PRIMARY]
GO
ALTER TABLE [dbo].[vcard] WITH NOCHECK ADD
CONSTRAINT [PK_vcard] PRIMARY KEY CLUSTERED
(

6
src/odbc/mysql.sql
View File

@ -148,6 +148,12 @@ CREATE TABLE private_storage (
CREATE INDEX i_private_storage_username USING BTREE ON private_storage(username);
CREATE UNIQUE INDEX i_private_storage_username_namespace USING BTREE ON private_storage(username(75), namespace(75));
-- Not tested in mysql
CREATE TABLE roster_version (
username varchar(250) PRIMARY KEY,
version text NOT NULL
) CHARACTER SET utf8;
-- To update from 1.x:
-- ALTER TABLE rosterusers ADD COLUMN askmessage text AFTER ask;
-- UPDATE rosterusers SET askmessage = '';

16
src/odbc/odbc_queries.erl
View File

@ -78,7 +78,9 @@
set_vcard/26,
get_vcard/2,
escape/1,
count_records_where/3]).
count_records_where/3,
get_roster_version/2,
set_roster_version/2]).
%% We have only two compile time options for db queries:
%-define(generic, true).
@ -555,6 +557,12 @@ count_records_where(LServer, Table, WhereClause) ->
ejabberd_odbc:sql_query(
LServer,
["select count(*) from ", Table, " ", WhereClause, ";"]).
get_roster_version(LServer, LUser) ->
ejabberd_odbc:sql_query(LServer,
["select version from roster_version where username = '", LUser, "'"]).
set_roster_version(LUser, Version) ->
update_t("roster_version", ["username", "version"], [LUser, Version], ["username = '", LUser, "'"]).
-endif.
%% -----------------
@ -791,4 +799,10 @@ count_records_where(LServer, Table, WhereClause) ->
ejabberd_odbc:sql_query(
LServer,
["select count(*) from ", Table, " ", WhereClause, " with (nolock)"]).
get_roster_version(LServer, LUser) ->
ejabberd_odbc:sql_query(LServer,
["select version from dbo.roster_version where username = '", LUser, "'"]).
set_roster_version(LUser, Version) ->
update_t("dbo.roster_version", ["username", "version"], [LUser, Version], ["username = '", LUser, "'"]).
-endif.

5
src/odbc/pg.sql
View File

@ -145,7 +145,10 @@ CREATE TABLE private_storage (
CREATE INDEX i_private_storage_username ON private_storage USING btree (username);
CREATE UNIQUE INDEX i_private_storage_username_namespace ON private_storage USING btree (username, namespace);
CREATE TABLE roster_version (
username text PRIMARY KEY,
version text NOT NULL
);
-- To update from 0.9.8:
-- CREATE SEQUENCE spool_seq_seq;
-- ALTER TABLE spool ADD COLUMN seq integer;

75
src/roster_versioning.erl Normal file
View File

@ -0,0 +1,75 @@
%%%----------------------------------------------------------------------
%%% File : mod_roster.erl
%%% Author : Pablo Polvorin <pablo.polvorin@process-one.net>
%%% Purpose : Common utility functions for XEP-0237 (Roster Versioning)
%%% Created : 19 Jul 2009 by Pablo Polvorin <pablo.polvorin@process-one.net>
%%%
%%%
%%% ejabberd, Copyright (C) 2009 ProcessOne
%%%
%%% This program is free software; you can redistribute it and/or
%%% modify it under the terms of the GNU General Public License as
%%% published by the Free Software Foundation; either version 2 of the
%%% License, or (at your option) any later version.
%%%
%%% This program is distributed in the hope that it will be useful,
%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
%%% General Public License for more details.
%%%
%%% You should have received a copy of the GNU General Public License
%%% along with this program; if not, write to the Free Software
%%% Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
%%% 02111-1307 USA
%%%
%%%
%%% @doc The roster versioning follows an all-or-nothing strategy:
%%% - If the version supplied by the client is the lastest, return an empty response
%%% - If not, return the entire new roster (with updated version string).
%%% Roster version is a hash digest of the entire roster.
%%% No additional data is stored in DB.
%%%----------------------------------------------------------------------
-module(roster_versioning).
-author('pablo.polvorin@process-one.net').
%%API
-export([is_enabled/1,
stream_feature/0,
push_item/5]).
-include("mod_roster.hrl").
-include_lib("exmpp/include/exmpp.hrl").
-define(NS_ROSTER_VER, "urn:xmpp:features:rosterver").
%%@doc is roster versioning enabled?
is_enabled(Host) ->
case gen_mod:is_loaded(binary_to_list(Host), mod_roster) of
true -> mod_roster:roster_versioning_enabled(Host);
false -> mod_roster_odbc:roster_versioning_enabled(Host)
end.
stream_feature() ->
exmpp_xml:element(?NS_ROSTER_VER, 'ver', [], [exmpp_xml:element(?NS_ROSTER_VER, 'optional')]).
%% @doc Roster push, calculate and include the version attribute.
%% TODO: don't push to those who didn't load roster
push_item(Server, User, From, Item, RosterVersion) ->
lists:foreach(fun(Resource) ->
push_item(User, Server, Resource, From, Item, RosterVersion)
end, ejabberd_sm:get_user_resources(User, Server)).
push_item(User, Server, Resource, From, Item, RosterVersion) ->
Request = #xmlel{ns = ?NS_ROSTER, name = 'query', attrs = [?XMLATTR('ver', RosterVersion)],
children = [mod_roster:item_to_xml(Item)]},
ResIQ = exmpp_iq:set(?NS_JABBER_CLIENT, Request,
"push" ++ randoms:get_string()),
ejabberd_router:route(
From,
exmpp_jid:make(User, Server, Resource),
ResIQ).