25
1
mirror of https://github.com/processone/ejabberd.git synced 2024-12-22 17:28:25 +01:00

Clean mod_offline.erl from DB specific code

This commit is contained in:
Evgeniy Khramtsov 2016-04-15 13:44:33 +03:00
parent 860db2ddca
commit 901d2e0aed
4 changed files with 808 additions and 763 deletions

File diff suppressed because it is too large Load Diff

232
src/mod_offline_mnesia.erl Normal file
View File

@ -0,0 +1,232 @@
%%%-------------------------------------------------------------------
%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
%%% @copyright (C) 2016, Evgeny Khramtsov
%%% @doc
%%%
%%% @end
%%% Created : 15 Apr 2016 by Evgeny Khramtsov <ekhramtsov@process-one.net>
%%%-------------------------------------------------------------------
-module(mod_offline_mnesia).
-behaviour(mod_offline).
-export([init/2, store_messages/5, pop_messages/2, remove_expired_messages/1,
remove_old_messages/2, remove_user/2, read_message_headers/2,
read_message/3, remove_message/3, read_all_messages/2,
remove_all_messages/2, count_messages/2, import/2]).
-include("jlib.hrl").
-include("mod_offline.hrl").
-include("logger.hrl").
-define(OFFLINE_TABLE_LOCK_THRESHOLD, 1000).
%%%===================================================================
%%% API
%%%===================================================================
init(_Host, _Opts) ->
mnesia:create_table(offline_msg,
[{disc_only_copies, [node()]}, {type, bag},
{attributes, record_info(fields, offline_msg)}]),
update_table().
store_messages(_Host, US, Msgs, Len, MaxOfflineMsgs) ->
F = fun () ->
Count = if MaxOfflineMsgs =/= infinity ->
Len + count_mnesia_records(US);
true -> 0
end,
if Count > MaxOfflineMsgs -> discard;
true ->
if Len >= (?OFFLINE_TABLE_LOCK_THRESHOLD) ->
mnesia:write_lock_table(offline_msg);
true -> ok
end,
lists:foreach(fun (M) -> mnesia:write(M) end, Msgs)
end
end,
mnesia:transaction(F).
pop_messages(LUser, LServer) ->
US = {LUser, LServer},
F = fun () ->
Rs = mnesia:wread({offline_msg, US}),
mnesia:delete({offline_msg, US}),
Rs
end,
case mnesia:transaction(F) of
{atomic, L} ->
{ok, lists:keysort(#offline_msg.timestamp, L)};
{aborted, Reason} ->
{error, Reason}
end.
remove_expired_messages(_LServer) ->
TimeStamp = p1_time_compat:timestamp(),
F = fun () ->
mnesia:write_lock_table(offline_msg),
mnesia:foldl(fun (Rec, _Acc) ->
case Rec#offline_msg.expire of
never -> ok;
TS ->
if TS < TimeStamp ->
mnesia:delete_object(Rec);
true -> ok
end
end
end,
ok, offline_msg)
end,
mnesia:transaction(F).
remove_old_messages(Days, _LServer) ->
S = p1_time_compat:system_time(seconds) - 60 * 60 * 24 * Days,
MegaSecs1 = S div 1000000,
Secs1 = S rem 1000000,
TimeStamp = {MegaSecs1, Secs1, 0},
F = fun () ->
mnesia:write_lock_table(offline_msg),
mnesia:foldl(fun (#offline_msg{timestamp = TS} = Rec,
_Acc)
when TS < TimeStamp ->
mnesia:delete_object(Rec);
(_Rec, _Acc) -> ok
end,
ok, offline_msg)
end,
mnesia:transaction(F).
remove_user(LUser, LServer) ->
US = {LUser, LServer},
F = fun () -> mnesia:delete({offline_msg, US}) end,
mnesia:transaction(F).
read_message_headers(LUser, LServer) ->
Msgs = mnesia:dirty_read({offline_msg, {LUser, LServer}}),
Hdrs = lists:map(
fun(#offline_msg{from = From, to = To, packet = Pkt,
timestamp = TS}) ->
Seq = now_to_integer(TS),
NewPkt = jlib:add_delay_info(Pkt, LServer, TS,
<<"Offline Storage">>),
{Seq, From, To, NewPkt}
end, Msgs),
lists:keysort(1, Hdrs).
read_message(LUser, LServer, I) ->
US = {LUser, LServer},
TS = integer_to_now(I),
case mnesia:dirty_match_object(
offline_msg, #offline_msg{us = US, timestamp = TS, _ = '_'}) of
[Msg|_] ->
{ok, Msg};
_ ->
error
end.
remove_message(LUser, LServer, I) ->
US = {LUser, LServer},
TS = integer_to_now(I),
Msgs = mnesia:dirty_match_object(
offline_msg, #offline_msg{us = US, timestamp = TS, _ = '_'}),
lists:foreach(
fun(Msg) ->
mnesia:dirty_delete_object(Msg)
end, Msgs).
read_all_messages(LUser, LServer) ->
US = {LUser, LServer},
lists:keysort(#offline_msg.timestamp,
mnesia:dirty_read({offline_msg, US})).
remove_all_messages(LUser, LServer) ->
US = {LUser, LServer},
F = fun () ->
mnesia:write_lock_table(offline_msg),
lists:foreach(fun (Msg) -> mnesia:delete_object(Msg) end,
mnesia:dirty_read({offline_msg, US}))
end,
mnesia:transaction(F).
count_messages(LUser, LServer) ->
US = {LUser, LServer},
F = fun () ->
count_mnesia_records(US)
end,
case catch mnesia:async_dirty(F) of
I when is_integer(I) -> I;
_ -> 0
end.
import(_LServer, #offline_msg{} = Msg) ->
mnesia:dirty_write(Msg).
%%%===================================================================
%%% Internal functions
%%%===================================================================
%% Return the number of records matching a given match expression.
%% This function is intended to be used inside a Mnesia transaction.
%% The count has been written to use the fewest possible memory by
%% getting the record by small increment and by using continuation.
-define(BATCHSIZE, 100).
count_mnesia_records(US) ->
MatchExpression = #offline_msg{us = US, _ = '_'},
case mnesia:select(offline_msg, [{MatchExpression, [], [[]]}],
?BATCHSIZE, read) of
{Result, Cont} ->
Count = length(Result),
count_records_cont(Cont, Count);
'$end_of_table' ->
0
end.
count_records_cont(Cont, Count) ->
case mnesia:select(Cont) of
{Result, Cont} ->
NewCount = Count + length(Result),
count_records_cont(Cont, NewCount);
'$end_of_table' ->
Count
end.
jid_to_binary(#jid{user = U, server = S, resource = R,
luser = LU, lserver = LS, lresource = LR}) ->
#jid{user = iolist_to_binary(U),
server = iolist_to_binary(S),
resource = iolist_to_binary(R),
luser = iolist_to_binary(LU),
lserver = iolist_to_binary(LS),
lresource = iolist_to_binary(LR)}.
now_to_integer({MS, S, US}) ->
(MS * 1000000 + S) * 1000000 + US.
integer_to_now(Int) ->
Secs = Int div 1000000,
USec = Int rem 1000000,
MSec = Secs div 1000000,
Sec = Secs rem 1000000,
{MSec, Sec, USec}.
update_table() ->
Fields = record_info(fields, offline_msg),
case mnesia:table_info(offline_msg, attributes) of
Fields ->
ejabberd_config:convert_table_to_binary(
offline_msg, Fields, bag,
fun(#offline_msg{us = {U, _}}) -> U end,
fun(#offline_msg{us = {U, S},
from = From,
to = To,
packet = El} = R) ->
R#offline_msg{us = {iolist_to_binary(U),
iolist_to_binary(S)},
from = jid_to_binary(From),
to = jid_to_binary(To),
packet = fxml:to_xmlel(El)}
end);
_ ->
?INFO_MSG("Recreating offline_msg table", []),
mnesia:transform_table(offline_msg, ignore, Fields)
end.

153
src/mod_offline_riak.erl Normal file
View File

@ -0,0 +1,153 @@
%%%-------------------------------------------------------------------
%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
%%% @copyright (C) 2016, Evgeny Khramtsov
%%% @doc
%%%
%%% @end
%%% Created : 15 Apr 2016 by Evgeny Khramtsov <ekhramtsov@process-one.net>
%%%-------------------------------------------------------------------
-module(mod_offline_riak).
-behaviour(mod_offline).
-export([init/2, store_messages/5, pop_messages/2, remove_expired_messages/1,
remove_old_messages/2, remove_user/2, read_message_headers/2,
read_message/3, remove_message/3, read_all_messages/2,
remove_all_messages/2, count_messages/2, import/2]).
-include("jlib.hrl").
-include("mod_offline.hrl").
%%%===================================================================
%%% API
%%%===================================================================
init(_Host, _Opts) ->
ok.
store_messages(Host, {User, _}, Msgs, Len, MaxOfflineMsgs) ->
Count = if MaxOfflineMsgs =/= infinity ->
Len + count_messages(User, Host);
true -> 0
end,
if
Count > MaxOfflineMsgs ->
{atomic, discard};
true ->
try
lists:foreach(
fun(#offline_msg{us = US,
timestamp = TS} = M) ->
ok = ejabberd_riak:put(
M, offline_msg_schema(),
[{i, TS}, {'2i', [{<<"us">>, US}]}])
end, Msgs),
{atomic, ok}
catch _:{badmatch, Err} ->
{atomic, Err}
end
end.
pop_messages(LUser, LServer) ->
case ejabberd_riak:get_by_index(offline_msg, offline_msg_schema(),
<<"us">>, {LUser, LServer}) of
{ok, Rs} ->
try
lists:foreach(
fun(#offline_msg{timestamp = T}) ->
ok = ejabberd_riak:delete(offline_msg, T)
end, Rs),
{ok, lists:keysort(#offline_msg.timestamp, Rs)}
catch _:{badmatch, Err} ->
Err
end;
Err ->
Err
end.
remove_expired_messages(_LServer) ->
%% TODO
{atomic, ok}.
remove_old_messages(_Days, _LServer) ->
%% TODO
{atomic, ok}.
remove_user(LUser, LServer) ->
{atomic, ejabberd_riak:delete_by_index(offline_msg,
<<"us">>, {LUser, LServer})}.
read_message_headers(LUser, LServer) ->
case ejabberd_riak:get_by_index(
offline_msg, offline_msg_schema(),
<<"us">>, {LUser, LServer}) of
{ok, Rs} ->
Hdrs = lists:map(
fun(#offline_msg{from = From, to = To, packet = Pkt,
timestamp = TS}) ->
Seq = now_to_integer(TS),
NewPkt = jlib:add_delay_info(
Pkt, LServer, TS, <<"Offline Storage">>),
{Seq, From, To, NewPkt}
end, Rs),
lists:keysort(1, Hdrs);
_Err ->
[]
end.
read_message(_LUser, _LServer, I) ->
TS = integer_to_now(I),
case ejabberd_riak:get(offline_msg, offline_msg_schema(), TS) of
{ok, Msg} ->
{ok, Msg};
_ ->
error
end.
remove_message(_LUser, _LServer, I) ->
TS = integer_to_now(I),
ejabberd_riak:delete(offline_msg, TS),
ok.
read_all_messages(LUser, LServer) ->
case ejabberd_riak:get_by_index(
offline_msg, offline_msg_schema(),
<<"us">>, {LUser, LServer}) of
{ok, Rs} ->
lists:keysort(#offline_msg.timestamp, Rs);
_Err ->
[]
end.
remove_all_messages(LUser, LServer) ->
Res = ejabberd_riak:delete_by_index(offline_msg,
<<"us">>, {LUser, LServer}),
{atomic, Res}.
count_messages(LUser, LServer) ->
case ejabberd_riak:count_by_index(
offline_msg, <<"us">>, {LUser, LServer}) of
{ok, Res} ->
Res;
_ ->
0
end.
import(_LServer, #offline_msg{us = US, timestamp = TS} = M) ->
ejabberd_riak:put(M, offline_msg_schema(),
[{i, TS}, {'2i', [{<<"us">>, US}]}]).
%%%===================================================================
%%% Internal functions
%%%===================================================================
offline_msg_schema() ->
{record_info(fields, offline_msg), #offline_msg{}}.
now_to_integer({MS, S, US}) ->
(MS * 1000000 + S) * 1000000 + US.
integer_to_now(Int) ->
Secs = Int div 1000000,
USec = Int rem 1000000,
MSec = Secs div 1000000,
Sec = Secs rem 1000000,
{MSec, Sec, USec}.

252
src/mod_offline_sql.erl Normal file
View File

@ -0,0 +1,252 @@
%%%-------------------------------------------------------------------
%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
%%% @copyright (C) 2016, Evgeny Khramtsov
%%% @doc
%%%
%%% @end
%%% Created : 15 Apr 2016 by Evgeny Khramtsov <ekhramtsov@process-one.net>
%%%-------------------------------------------------------------------
-module(mod_offline_sql).
-compile([{parse_transform, ejabberd_sql_pt}]).
-behaviour(mod_offline).
-export([init/2, store_messages/5, pop_messages/2, remove_expired_messages/1,
remove_old_messages/2, remove_user/2, read_message_headers/2,
read_message/3, remove_message/3, read_all_messages/2,
remove_all_messages/2, count_messages/2, import/1, import/2,
export/1]).
-include("jlib.hrl").
-include("mod_offline.hrl").
-include("logger.hrl").
-include("ejabberd_sql_pt.hrl").
%%%===================================================================
%%% API
%%%===================================================================
init(_Host, _Opts) ->
ok.
store_messages(Host, {User, _Server}, Msgs, Len, MaxOfflineMsgs) ->
Count = if MaxOfflineMsgs =/= infinity ->
Len + count_messages(User, Host);
true -> 0
end,
if Count > MaxOfflineMsgs -> {atomic, discard};
true ->
Query = lists:map(
fun(M) ->
Username =
ejabberd_odbc:escape((M#offline_msg.to)#jid.luser),
From = M#offline_msg.from,
To = M#offline_msg.to,
Packet =
jlib:replace_from_to(From, To,
M#offline_msg.packet),
NewPacket =
jlib:add_delay_info(Packet, Host,
M#offline_msg.timestamp,
<<"Offline Storage">>),
XML =
ejabberd_odbc:escape(fxml:element_to_binary(NewPacket)),
odbc_queries:add_spool_sql(Username, XML)
end,
Msgs),
odbc_queries:add_spool(Host, Query)
end.
pop_messages(LUser, LServer) ->
case odbc_queries:get_and_del_spool_msg_t(LServer, LUser) of
{atomic, {selected, Rs}} ->
{ok, lists:flatmap(
fun({_, XML}) ->
case xml_to_offline_msg(XML) of
{ok, Msg} ->
[Msg];
_Err ->
[]
end
end, Rs)};
Err ->
{error, Err}
end.
remove_expired_messages(_LServer) ->
%% TODO
{atomic, ok}.
remove_old_messages(Days, LServer) ->
case catch ejabberd_odbc:sql_query(
LServer,
[<<"DELETE FROM spool"
" WHERE created_at < "
"DATE_SUB(CURDATE(), INTERVAL ">>,
integer_to_list(Days), <<" DAY);">>]) of
{updated, N} ->
?INFO_MSG("~p message(s) deleted from offline spool", [N]);
_Error ->
?ERROR_MSG("Cannot delete message in offline spool: ~p", [_Error])
end,
{atomic, ok}.
remove_user(LUser, LServer) ->
odbc_queries:del_spool_msg(LServer, LUser).
read_message_headers(LUser, LServer) ->
Username = ejabberd_odbc:escape(LUser),
case catch ejabberd_odbc:sql_query(
LServer, [<<"select xml, seq from spool where username ='">>,
Username, <<"' order by seq;">>]) of
{selected, [<<"xml">>, <<"seq">>], Rows} ->
lists:flatmap(
fun([XML, Seq]) ->
case xml_to_offline_msg(XML) of
{ok, #offline_msg{from = From,
to = To,
packet = El}} ->
Seq0 = binary_to_integer(Seq),
[{Seq0, From, To, El}];
_ ->
[]
end
end, Rows);
_Err ->
[]
end.
read_message(LUser, LServer, Seq) ->
Username = ejabberd_odbc:escape(LUser),
SSeq = ejabberd_odbc:escape(integer_to_binary(Seq)),
case ejabberd_odbc:sql_query(
LServer,
[<<"select xml from spool where username='">>, Username,
<<"' and seq='">>, SSeq, <<"';">>]) of
{selected, [<<"xml">>], [[RawXML]|_]} ->
case xml_to_offline_msg(RawXML) of
{ok, Msg} ->
{ok, Msg};
_ ->
error
end;
_ ->
error
end.
remove_message(LUser, LServer, Seq) ->
Username = ejabberd_odbc:escape(LUser),
SSeq = ejabberd_odbc:escape(integer_to_binary(Seq)),
ejabberd_odbc:sql_query(
LServer,
[<<"delete from spool where username='">>, Username,
<<"' and seq='">>, SSeq, <<"';">>]),
ok.
read_all_messages(LUser, LServer) ->
case catch ejabberd_odbc:sql_query(
LServer,
?SQL("select @(xml)s from spool where "
"username=%(LUser)s order by seq")) of
{selected, Rs} ->
lists:flatmap(
fun({XML}) ->
case xml_to_offline_msg(XML) of
{ok, Msg} -> [Msg];
_ -> []
end
end, Rs);
_ ->
[]
end.
remove_all_messages(LUser, LServer) ->
odbc_queries:del_spool_msg(LServer, LUser),
{atomic, ok}.
count_messages(LUser, LServer) ->
case catch ejabberd_odbc:sql_query(
LServer,
?SQL("select @(count(*))d from spool "
"where username=%(LUser)s")) of
{selected, [{Res}]} ->
Res;
_ -> 0
end.
export(_Server) ->
[{offline_msg,
fun(Host, #offline_msg{us = {LUser, LServer},
timestamp = TimeStamp, from = From, to = To,
packet = Packet})
when LServer == Host ->
Username = ejabberd_odbc:escape(LUser),
Packet1 = jlib:replace_from_to(From, To, Packet),
Packet2 = jlib:add_delay_info(Packet1, LServer, TimeStamp,
<<"Offline Storage">>),
XML = ejabberd_odbc:escape(fxml:element_to_binary(Packet2)),
[[<<"delete from spool where username='">>, Username, <<"';">>],
[<<"insert into spool(username, xml) values ('">>,
Username, <<"', '">>, XML, <<"');">>]];
(_Host, _R) ->
[]
end}].
import(LServer) ->
[{<<"select username, xml from spool;">>,
fun([LUser, XML]) ->
El = #xmlel{} = fxml_stream:parse_element(XML),
From = #jid{} = jid:from_string(
fxml:get_attr_s(<<"from">>, El#xmlel.attrs)),
To = #jid{} = jid:from_string(
fxml:get_attr_s(<<"to">>, El#xmlel.attrs)),
Stamp = fxml:get_path_s(El, [{elem, <<"delay">>},
{attr, <<"stamp">>}]),
TS = case jlib:datetime_string_to_timestamp(Stamp) of
{_, _, _} = Now ->
Now;
undefined ->
p1_time_compat:timestamp()
end,
Expire = mod_offline:find_x_expire(TS, El#xmlel.children),
#offline_msg{us = {LUser, LServer},
from = From, to = To,
packet = El,
timestamp = TS, expire = Expire}
end}].
import(_, _) ->
pass.
%%%===================================================================
%%% Internal functions
%%%===================================================================
xml_to_offline_msg(XML) ->
case fxml_stream:parse_element(XML) of
#xmlel{} = El ->
el_to_offline_msg(El);
Err ->
?ERROR_MSG("got ~p when parsing XML packet ~s",
[Err, XML]),
Err
end.
el_to_offline_msg(El) ->
To_s = fxml:get_tag_attr_s(<<"to">>, El),
From_s = fxml:get_tag_attr_s(<<"from">>, El),
To = jid:from_string(To_s),
From = jid:from_string(From_s),
if To == error ->
?ERROR_MSG("failed to get 'to' JID from offline XML ~p", [El]),
{error, bad_jid_to};
From == error ->
?ERROR_MSG("failed to get 'from' JID from offline XML ~p", [El]),
{error, bad_jid_from};
true ->
{ok, #offline_msg{us = {To#jid.luser, To#jid.lserver},
from = From,
to = To,
timestamp = undefined,
expire = undefined,
packet = El}}
end.