Chapril
/
xmpp.chapril.org-ejabberd
mirror of https://github.com/processone/ejabberd.git
24
1
Fork 0
Code Releases Activity

Support for roster versioning (EJAB-964) 

Introduces two options for mod_roster and mod_roster_odbc:
- {versioning, true | false}   Enable or disable roster versioning on ejabberd.
- {store_current_id, true | false}   If true, the current roster version is stored on DB (internal or odbc). Otherwise it is calculated on the fly each time.

Performance:
Setting store_current_id to true should help in reducing the load for both ejabberd and the DB.

Details: 
If store_current_id is false,  the roster version is a hash of the entire roster. If store_current_id is true, the roster version is a hash, but of the current time
(this has to do with transactional semantics; we need to perform both the roster update and the version update on the same transaction, but we don't   
have the entire roster when we are changing a single item on DB. Loading it there requires significant changes to be introduced, so I opted for this simpler approach).

In either case, there is no difference for the clients, the roster version ID is opaque.

IMPORTANT:
mod_shared_roster is not compatible with the option 'store_current_id'.  Shared roster and roster versioning can be both enabled, but store_current_id MUST be set to false.

SVN Revision: 2428
This commit is contained in:
Pablo Polvorin 2009-08-06 15:45:13 +00:00
parent 59c88fcfe7
commit 53626d16e3
10 changed files with 322 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/ejabberd_c2s.erl
View File

@ -326,13 +326,19 @@ wait_for_stream({xmlstreamstart, _Name, Attrs}, StateData) ->
_ ->
case StateData#state.resource of
"" ->
RosterVersioningFeature =
case roster_versioning:is_enabled(Server) of
true -> [roster_versioning:stream_feature()];
false -> []
end,
StreamFeatures = [{xmlelement, "bind",
[{"xmlns", ?NS_BIND}], []},
{xmlelement, "session",
[{"xmlns", ?NS_SESSION}], []} | RosterVersioningFeature],
send_element(
StateData,
{xmlelement, "stream:features", [],
[{xmlelement, "bind",
[{"xmlns", ?NS_BIND}], []},
{xmlelement, "session",
[{"xmlns", ?NS_SESSION}], []}]}),
StreamFeatures}),
fsm_next_state(wait_for_bind,
StateData#state{
server = Server,

1
src/jlib.hrl
View File

@ -27,6 +27,7 @@
-define(NS_REGISTER, "jabber:iq:register").
-define(NS_SEARCH, "jabber:iq:search").
-define(NS_ROSTER, "jabber:iq:roster").
-define(NS_ROSTER_VER, "urn:xmpp:features:rosterver").
-define(NS_PRIVACY, "jabber:iq:privacy").
-define(NS_PRIVATE, "jabber:iq:private").
-define(NS_VERSION, "jabber:iq:version").

107
src/mod_roster.erl
View File

@ -42,7 +42,9 @@
get_jid_info/4,
item_to_xml/1,
webadmin_page/3,
webadmin_user/4]).
webadmin_user/4,
roster_versioning_enabled/1,
roster_version/2]).
-include("ejabberd.hrl").
-include("jlib.hrl").
@ -55,8 +57,12 @@ start(Host, Opts) ->
IQDisc = gen_mod:get_opt(iqdisc, Opts, one_queue),
mnesia:create_table(roster,[{disc_copies, [node()]},
{attributes, record_info(fields, roster)}]),
mnesia:create_table(roster_version, [{disc_copies, [node()]},
{attributes, record_info(fields, roster_version)}]),
update_table(),
mnesia:add_table_index(roster, us),
mnesia:add_table_index(roster_version, us),
ejabberd_hooks:add(roster_get, Host,
?MODULE, get_user_roster, 50),
ejabberd_hooks:add(roster_in_subscription, Host,
@ -104,6 +110,12 @@ stop(Host) ->
gen_iq_handler:remove_iq_handler(ejabberd_sm, Host, ?NS_ROSTER).
roster_versioning_enabled(Host) ->
gen_mod:get_module_opt(Host, ?MODULE, versioning, false).
roster_version_on_db(Host) ->
gen_mod:get_module_opt(Host, ?MODULE, store_current_id, false).
process_iq(From, To, IQ) ->
#iq{sub_el = SubEl} = IQ,
#jid{lserver = LServer} = From,
@ -122,23 +134,79 @@ process_local_iq(From, To, #iq{type = Type} = IQ) ->
process_iq_get(From, To, IQ)
end.
roster_hash(Items) ->
sha:sha(term_to_binary(
lists:sort(
[R#roster{groups = lists:sort(Grs)} ||
R = #roster{groups = Grs} <- Items]))).
roster_version(LServer ,LUser) ->
US = {LUser, LServer},
case roster_version_on_db(LServer) of
true ->
case mnesia:dirty_read(roster_version, US) of
[#roster_version{version = V}] -> V;
[] -> not_found
end;
false ->
roster_hash(ejabberd_hooks:run_fold(roster_get, LServer, [], [US]))
end.
%% Load roster from DB only if neccesary.
%% It is neccesary if
%% - roster versioning is disabled in server OR
%% - roster versioning is not used by the client OR
%% - roster versioning is used by server and client, BUT the server isn't storing versions on db OR
%% - the roster version from client don't match current version.
process_iq_get(From, To, #iq{sub_el = SubEl} = IQ) ->
LUser = From#jid.luser,
LServer = From#jid.lserver,
US = {LUser, LServer},
case catch ejabberd_hooks:run_fold(roster_get, To#jid.lserver, [], [US]) of
Items when is_list(Items) ->
XItems = lists:map(fun item_to_xml/1, Items),
IQ#iq{type = result,
sub_el = [{xmlelement, "query",
[{"xmlns", ?NS_ROSTER}],
XItems}]};
_ ->
IQ#iq{type = error, sub_el = [SubEl, ?ERR_INTERNAL_SERVER_ERROR]}
try
{ItemsToSend, VersionToSend} =
case {xml:get_tag_attr("ver", SubEl),
roster_versioning_enabled(LServer),
roster_version_on_db(LServer)} of
{{value, RequestedVersion}, true, true} ->
%% Retrieve version from DB. Only load entire roster
%% when neccesary.
case mnesia:dirty_read(roster_version, US) of
[#roster_version{version = RequestedVersion}] ->
{false, false};
[#roster_version{version = NewVersion}] ->
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, To#jid.lserver, [], [US])), NewVersion};
[] ->
RosterVersion = sha:sha(term_to_binary(now())),
mnesia:dirty_write(#roster_version{us = US, version = RosterVersion}),
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, To#jid.lserver, [], [US])), RosterVersion}
end;
{{value, RequestedVersion}, true, false} ->
RosterItems = ejabberd_hooks:run_fold(roster_get, To#jid.lserver, [] , [US]),
case roster_hash(RosterItems) of
RequestedVersion ->
{false, false};
New ->
{lists:map(fun item_to_xml/1, RosterItems), New}
end;
_ ->
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, To#jid.lserver, [], [US])), false}
end,
IQ#iq{type = result, sub_el = case {ItemsToSend, VersionToSend} of
{false, false} -> [];
{Items, false} -> [{xmlelement, "query", [{"xmlns", ?NS_ROSTER}], Items}];
{Items, Version} -> [{xmlelement, "query", [{"xmlns", ?NS_ROSTER}, {"ver", Version}], Items}]
end}
catch
_:_ ->
IQ#iq{type =error, sub_el = [SubEl, ?ERR_INTERNAL_SERVER_ERROR]}
end.
get_user_roster(Acc, US) ->
case catch mnesia:dirty_index_read(roster, US, #roster.us) of
Items when is_list(Items) ->
@ -226,6 +294,10 @@ process_item_set(From, To, {xmlelement, _Name, Attrs, Els}) ->
%% subscription information from there:
Item3 = ejabberd_hooks:run_fold(roster_process_item,
LServer, Item2, [LServer]),
case roster_version_on_db(LServer) of
true -> mnesia:write(#roster_version{us = {LUser, LServer}, version = sha:sha(term_to_binary(now()))});
false -> ok
end,
{Item, Item3}
end,
case mnesia:transaction(F) of
@ -302,9 +374,14 @@ push_item(User, Server, From, Item) ->
[{item,
Item#roster.jid,
Item#roster.subscription}]}),
lists:foreach(fun(Resource) ->
case roster_versioning_enabled(Server) of
true ->
roster_versioning:push_item(Server, User, From, Item, roster_version(Server, User));
false ->
lists:foreach(fun(Resource) ->
push_item(User, Server, Resource, From, Item)
end, ejabberd_sm:get_user_resources(User, Server)).
end, ejabberd_sm:get_user_resources(User, Server))
end.
% TODO: don't push to those who didn't load roster
push_item(User, Server, Resource, From, Item) ->
@ -408,6 +485,10 @@ process_subscription(Direction, User, Server, JID1, Type, Reason) ->
ask = Pending,
askmessage = list_to_binary(AskMessage)},
mnesia:write(NewItem),
case roster_version_on_db(LServer) of
true -> mnesia:write(#roster_version{us = {LUser, LServer}, version = sha:sha(term_to_binary(now()))});
false -> ok
end,
{{push, NewItem}, AutoReply}
end
end,

2
src/mod_roster.hrl
View File

@ -29,3 +29,5 @@
askmessage = [],
xs = []}).
-record(roster_version, {us,
version}).

106
src/mod_roster_odbc.erl
View File

@ -41,7 +41,8 @@
remove_user/2,
get_jid_info/4,
webadmin_page/3,
webadmin_user/4]).
webadmin_user/4,
roster_versioning_enabled/1]).
-include("ejabberd.hrl").
-include("jlib.hrl").
@ -99,6 +100,12 @@ stop(Host) ->
gen_iq_handler:remove_iq_handler(ejabberd_sm, Host, ?NS_ROSTER).
roster_versioning_enabled(Host) ->
gen_mod:get_module_opt(Host, ?MODULE, versioning, false).
roster_version_on_db(Host) ->
gen_mod:get_module_opt(Host, ?MODULE, store_current_id, false).
process_iq(From, To, IQ) ->
#iq{sub_el = SubEl} = IQ,
#jid{lserver = LServer} = From,
@ -118,22 +125,83 @@ process_local_iq(From, To, #iq{type = Type} = IQ) ->
end.
roster_hash(Items) ->
sha:sha(term_to_binary(
lists:sort(
[R#roster{groups = lists:sort(Grs)} ||
R = #roster{groups = Grs} <- Items]))).
roster_version(LServer ,LUser) ->
US = {LUser, LServer},
case roster_version_on_db(LServer) of
true ->
case odbc_queries:get_roster_version(ejabberd_odbc:escape(LServer), ejabberd_odbc:escape(LUser)) of
{selected, ["version"], [{Version}]} -> Version;
{selected, ["version"], []} -> not_found
end;
false ->
roster_hash(ejabberd_hooks:run_fold(roster_get, LServer, [], [US]))
end.
%% Load roster from DB only if neccesary.
%% It is neccesary if
%% - roster versioning is disabled in server OR
%% - roster versioning is not used by the client OR
%% - roster versioning is used by server and client, BUT the server isn't storing versions on db OR
%% - the roster version from client don't match current version.
process_iq_get(From, To, #iq{sub_el = SubEl} = IQ) ->
LUser = From#jid.luser,
LServer = From#jid.lserver,
US = {LUser, LServer},
case catch ejabberd_hooks:run_fold(roster_get, To#jid.lserver, [], [US]) of
Items when is_list(Items) ->
XItems = lists:map(fun item_to_xml/1, Items),
IQ#iq{type = result,
sub_el = [{xmlelement, "query",
[{"xmlns", ?NS_ROSTER}],
XItems}]};
_ ->
IQ#iq{type = error, sub_el = [SubEl, ?ERR_INTERNAL_SERVER_ERROR]}
try
{ItemsToSend, VersionToSend} =
case {xml:get_tag_attr("ver", SubEl),
roster_versioning_enabled(LServer),
roster_version_on_db(LServer)} of
{{value, RequestedVersion}, true, true} ->
%% Retrieve version from DB. Only load entire roster
%% when neccesary.
case odbc_queries:get_roster_version(ejabberd_odbc:escape(LServer), ejabberd_odbc:escape(LUser)) of
{selected, ["version"], [{RequestedVersion}]} ->
{false, false};
{selected, ["version"], [{NewVersion}]} ->
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, To#jid.lserver, [], [US])), NewVersion};
{selected, ["version"], []} ->
RosterVersion = sha:sha(term_to_binary(now())),
{atomic, {updated,1}} = odbc_queries:sql_transaction(LServer, fun() ->
odbc_queries:set_roster_version(ejabberd_odbc:escape(LUser), RosterVersion)
end),
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, To#jid.lserver, [], [US])), RosterVersion}
end;
{{value, RequestedVersion}, true, false} ->
RosterItems = ejabberd_hooks:run_fold(roster_get, To#jid.lserver, [] , [US]),
case roster_hash(RosterItems) of
RequestedVersion ->
{false, false};
New ->
{lists:map(fun item_to_xml/1, RosterItems), New}
end;
_ ->
{lists:map(fun item_to_xml/1,
ejabberd_hooks:run_fold(roster_get, To#jid.lserver, [], [US])), false}
end,
IQ#iq{type = result, sub_el = case {ItemsToSend, VersionToSend} of
{false, false} -> [];
{Items, false} -> [{xmlelement, "query", [{"xmlns", ?NS_ROSTER}], Items}];
{Items, Version} -> [{xmlelement, "query", [{"xmlns", ?NS_ROSTER}, {"ver", Version}], Items}]
end}
catch
_:_ ->
IQ#iq{type =error, sub_el = [SubEl, ?ERR_INTERNAL_SERVER_ERROR]}
end.
get_user_roster(Acc, {LUser, LServer}) ->
Items = get_roster(LUser, LServer),
lists:filter(fun(#roster{subscription = none, ask = in}) ->
@ -257,6 +325,7 @@ process_item_set(From, To, {xmlelement, _Name, Attrs, Els}) ->
Item2 = process_item_els(Item1, Els),
case Item2#roster.subscription of
remove ->
io:format("del_roster: ~p ~p ~p \n", [LServer, Username, SJID]),
odbc_queries:del_roster(LServer, Username, SJID);
_ ->
ItemVals = record_to_string(Item2),
@ -267,6 +336,10 @@ process_item_set(From, To, {xmlelement, _Name, Attrs, Els}) ->
%% subscription information from there:
Item3 = ejabberd_hooks:run_fold(roster_process_item,
LServer, Item2, [LServer]),
case roster_version_on_db(LServer) of
true -> odbc_queries:set_roster_version(ejabberd_odbc:escape(LUser), sha:sha(term_to_binary(now())));
false -> ok
end,
{Item, Item3}
end,
case odbc_queries:sql_transaction(LServer, F) of
@ -337,9 +410,14 @@ push_item(User, Server, From, Item) ->
[{item,
Item#roster.jid,
Item#roster.subscription}]}),
lists:foreach(fun(Resource) ->
case roster_versioning_enabled(Server) of
true ->
roster_versioning:push_item(Server, User, From, Item, roster_version(Server, User));
false ->
lists:foreach(fun(Resource) ->
push_item(User, Server, Resource, From, Item)
end, ejabberd_sm:get_user_resources(User, Server)).
end, ejabberd_sm:get_user_resources(User, Server))
end.
% TODO: don't push to those who not load roster
push_item(User, Server, Resource, From, Item) ->
@ -468,6 +546,10 @@ process_subscription(Direction, User, Server, JID1, Type, Reason) ->
askmessage = AskMessage},
ItemVals = record_to_string(NewItem),
odbc_queries:roster_subscribe(LServer, Username, SJID, ItemVals),
case roster_version_on_db(LServer) of
true -> odbc_queries:set_roster_version(ejabberd_odbc:escape(LUser), sha:sha(term_to_binary(now())));
false -> ok
end,
{{push, NewItem}, AutoReply}
end
end,

17
src/odbc/mssql2000.sql
View File

@ -209,6 +209,14 @@ CREATE TABLE [dbo].[privacy_list_data] (
) ON [PRIMARY]
GO
/* Not tested on mssql */
CREATE TABLE [dbo].[roster_version] (
[username] [varchar] (250) NOT NULL ,
[version] [varchar] (64) NOT NULL
) ON [PRIMARY]
GO
/* Constraints to add:
- id in privacy_list is a SERIAL autogenerated number
- id in privacy_list_data must exist in the table privacy_list */
@ -244,6 +252,13 @@ ALTER TABLE [dbo].[users] WITH NOCHECK ADD
) WITH FILLFACTOR = 90 ON [PRIMARY]
GO
ALTER TABLE [dbo].[roster_version] WITH NOCHECK ADD
CONSTRAINT [PK_roster_version] PRIMARY KEY CLUSTERED
(
[username]
) WITH FILLFACTOR = 90 ON [PRIMARY]
GO
ALTER TABLE [dbo].[vcard] WITH NOCHECK ADD
CONSTRAINT [PK_vcard] PRIMARY KEY CLUSTERED
(
@ -275,6 +290,8 @@ ALTER TABLE [dbo].[privacy_default_list] WITH NOCHECK ADD
) WITH FILLFACTOR = 90 ON [PRIMARY]
GO
CREATE INDEX [IX_rostergroups_jid] ON [dbo].[rostergroups]([jid]) WITH FILLFACTOR = 90 ON [PRIMARY]
GO

6
src/odbc/mysql.sql
View File

@ -148,6 +148,12 @@ CREATE TABLE private_storage (
CREATE INDEX i_private_storage_username USING BTREE ON private_storage(username);
CREATE UNIQUE INDEX i_private_storage_username_namespace USING BTREE ON private_storage(username(75), namespace(75));
-- Not tested in mysql
CREATE TABLE roster_version (
username varchar(250) PRIMARY KEY,
version text NOT NULL
) CHARACTER SET utf8;
-- To update from 1.x:
-- ALTER TABLE rosterusers ADD COLUMN askmessage text AFTER ask;
-- UPDATE rosterusers SET askmessage = '';

17
src/odbc/odbc_queries.erl
View File

@ -78,7 +78,9 @@
set_vcard/26,
get_vcard/2,
escape/1,
count_records_where/3]).
count_records_where/3,
get_roster_version/2,
set_roster_version/2]).
%% We have only two compile time options for db queries:
%-define(generic, true).
@ -555,6 +557,13 @@ count_records_where(LServer, Table, WhereClause) ->
ejabberd_odbc:sql_query(
LServer,
["select count(*) from ", Table, " ", WhereClause, ";"]).
get_roster_version(LServer, LUser) ->
ejabberd_odbc:sql_query(LServer,
["select version from roster_version where username = '", LUser, "'"]).
set_roster_version(LUser, Version) ->
update_t("roster_version", ["username", "version"], [LUser, Version], ["username = '", LUser, "'"]).
-endif.
%% -----------------
@ -791,4 +800,10 @@ count_records_where(LServer, Table, WhereClause) ->
ejabberd_odbc:sql_query(
LServer,
["select count(*) from ", Table, " ", WhereClause, " with (nolock)"]).
get_roster_version(LServer, LUser) ->
ejabberd_odbc:sql_query(LServer,
["select version from dbo.roster_version where username = '", LUser, "'"]).
set_roster_version(LUser, Version) ->
update_t("dbo.roster_version", ["username", "version"], [LUser, Version], ["username = '", LUser, "'"]).
-endif.

5
src/odbc/pg.sql
View File

@ -146,6 +146,11 @@ CREATE INDEX i_private_storage_username ON private_storage USING btree (username
CREATE UNIQUE INDEX i_private_storage_username_namespace ON private_storage USING btree (username, namespace);
CREATE TABLE roster_version (
username text PRIMARY KEY,
version text NOT NULL
);
-- To update from 0.9.8:
-- CREATE SEQUENCE spool_seq_seq;
-- ALTER TABLE spool ADD COLUMN seq integer;

77
src/roster_versioning.erl Normal file
View File

@ -0,0 +1,77 @@
%%%----------------------------------------------------------------------
%%% File : mod_roster.erl
%%% Author : Pablo Polvorin <pablo.polvorin@process-one.net>
%%% Purpose : Common utility functions for XEP-0237 (Roster Versioning)
%%% Created : 19 Jul 2009 by Pablo Polvorin <pablo.polvorin@process-one.net>
%%%
%%%
%%% ejabberd, Copyright (C) 2009 ProcessOne
%%%
%%% This program is free software; you can redistribute it and/or
%%% modify it under the terms of the GNU General Public License as
%%% published by the Free Software Foundation; either version 2 of the
%%% License, or (at your option) any later version.
%%%
%%% This program is distributed in the hope that it will be useful,
%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
%%% General Public License for more details.
%%%
%%% You should have received a copy of the GNU General Public License
%%% along with this program; if not, write to the Free Software
%%% Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
%%% 02111-1307 USA
%%%
%%%
%%% @doc The roster versioning follows an all-or-nothing strategy:
%%% - If the version supplied by the client is the lastest, return an empty response
%%% - If not, return the entire new roster (with updated version string).
%%% Roster version is a hash digest of the entire roster.
%%% No additional data is stored in DB.
%%%----------------------------------------------------------------------
-module(roster_versioning).
-author('pablo.polvorin@process-one.net').
%%API
-export([is_enabled/1,
stream_feature/0,
push_item/5]).
-include("mod_roster.hrl").
-include("jlib.hrl").
%%@doc is roster versioning enabled?
is_enabled(Host) ->
case gen_mod:is_loaded(Host, mod_roster) of
true -> mod_roster:roster_versioning_enabled(Host);
false -> mod_roster_odbc:roster_versioning_enabled(Host)
end.
stream_feature() ->
{xmlelement,
"ver",
[{"xmlns", ?NS_ROSTER_VER}],
[{xmlelement, "optional", [], []}]}.
%% @doc Roster push, calculate and include the version attribute.
%% TODO: don't push to those who didn't load roster
push_item(Server, User, From, Item, RosterVersion) ->
lists:foreach(fun(Resource) ->
push_item(User, Server, Resource, From, Item, RosterVersion)
end, ejabberd_sm:get_user_resources(User, Server)).
push_item(User, Server, Resource, From, Item, RosterVersion) ->
IQPush = #iq{type = 'set', xmlns = ?NS_ROSTER,
id = "push" ++ randoms:get_string(),
sub_el = [{xmlelement, "query",
[{"xmlns", ?NS_ROSTER},
{"ver", RosterVersion}],
[mod_roster:item_to_xml(Item)]}]},
ejabberd_router:route(
From,
jlib:make_jid(User, Server, Resource),
jlib:iq_to_xml(IQPush)).