2005-08-29 21:00:10 +02:00
|
|
|
%%%----------------------------------------------------------------------
|
2016-04-20 11:27:32 +02:00
|
|
|
%%% File : ejd2sql.erl
|
2007-12-24 12:41:41 +01:00
|
|
|
%%% Author : Alexey Shchepin <alexey@process-one.net>
|
2005-09-10 20:03:48 +02:00
|
|
|
%%% Purpose : Export some mnesia tables to SQL DB
|
2007-12-24 12:41:41 +01:00
|
|
|
%%% Created : 22 Aug 2005 by Alexey Shchepin <alexey@process-one.net>
|
|
|
|
%%%
|
|
|
|
%%%
|
2018-01-05 21:18:58 +01:00
|
|
|
%%% ejabberd, Copyright (C) 2002-2018 ProcessOne
|
2007-12-24 12:41:41 +01:00
|
|
|
%%%
|
|
|
|
%%% This program is free software; you can redistribute it and/or
|
|
|
|
%%% modify it under the terms of the GNU General Public License as
|
|
|
|
%%% published by the Free Software Foundation; either version 2 of the
|
|
|
|
%%% License, or (at your option) any later version.
|
|
|
|
%%%
|
|
|
|
%%% This program is distributed in the hope that it will be useful,
|
|
|
|
%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
%%% General Public License for more details.
|
2009-01-12 15:44:42 +01:00
|
|
|
%%%
|
2014-02-22 11:27:40 +01:00
|
|
|
%%% You should have received a copy of the GNU General Public License along
|
|
|
|
%%% with this program; if not, write to the Free Software Foundation, Inc.,
|
|
|
|
%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
2007-12-24 12:41:41 +01:00
|
|
|
%%%
|
2005-08-29 21:00:10 +02:00
|
|
|
%%%----------------------------------------------------------------------
|
|
|
|
|
2016-04-20 11:27:32 +02:00
|
|
|
-module(ejd2sql).
|
2005-08-29 21:00:10 +02:00
|
|
|
|
2013-03-14 10:33:02 +01:00
|
|
|
-author('alexey@process-one.net').
|
2005-08-29 21:00:10 +02:00
|
|
|
|
2013-07-21 12:24:36 +02:00
|
|
|
-include("logger.hrl").
|
2016-05-04 20:01:05 +02:00
|
|
|
-include("ejabberd_sql_pt.hrl").
|
2013-07-21 12:24:36 +02:00
|
|
|
|
2016-11-22 14:48:01 +01:00
|
|
|
-export([export/2, export/3, import/3, import/4, delete/1, import_info/1]).
|
|
|
|
|
2005-08-29 21:00:10 +02:00
|
|
|
|
2013-03-14 10:33:02 +01:00
|
|
|
-define(MAX_RECORDS_PER_TRANSACTION, 100).
|
2005-08-29 21:00:10 +02:00
|
|
|
|
2016-11-22 14:48:01 +01:00
|
|
|
-record(sql_dump, {fd, type}).
|
2013-07-21 12:24:36 +02:00
|
|
|
|
2005-08-29 21:00:10 +02:00
|
|
|
%%%----------------------------------------------------------------------
|
|
|
|
%%% API
|
|
|
|
%%%----------------------------------------------------------------------
|
2006-11-20 14:20:47 +01:00
|
|
|
%%% How to use:
|
|
|
|
%%% A table can be converted from Mnesia to an ODBC database by calling
|
|
|
|
%%% one of the API function with the following parameters:
|
|
|
|
%%% - Server is the server domain you want to convert
|
2016-04-20 11:27:32 +02:00
|
|
|
%%% - Output can be either sql to export to the configured relational
|
2006-11-20 14:20:47 +01:00
|
|
|
%%% database or "Filename" to export to text file.
|
2005-08-29 21:00:10 +02:00
|
|
|
|
2013-07-21 12:24:36 +02:00
|
|
|
modules() ->
|
|
|
|
[ejabberd_auth,
|
|
|
|
mod_announce,
|
2016-11-22 14:48:01 +01:00
|
|
|
mod_caps,
|
2013-07-21 12:24:36 +02:00
|
|
|
mod_last,
|
2017-05-07 14:58:11 +02:00
|
|
|
mod_mam,
|
2013-07-21 12:24:36 +02:00
|
|
|
mod_muc,
|
|
|
|
mod_offline,
|
|
|
|
mod_privacy,
|
|
|
|
mod_private,
|
2017-08-03 10:52:44 +02:00
|
|
|
mod_pubsub,
|
2017-10-26 20:05:09 +02:00
|
|
|
mod_push,
|
2013-07-21 12:24:36 +02:00
|
|
|
mod_roster,
|
|
|
|
mod_shared_roster,
|
2017-05-17 18:29:19 +02:00
|
|
|
mod_vcard].
|
2013-07-21 12:24:36 +02:00
|
|
|
|
2013-03-14 10:33:02 +01:00
|
|
|
export(Server, Output) ->
|
2015-11-24 16:44:13 +01:00
|
|
|
LServer = jid:nameprep(iolist_to_binary(Server)),
|
2013-07-21 12:24:36 +02:00
|
|
|
Modules = modules(),
|
2013-03-14 10:33:02 +01:00
|
|
|
IO = prepare_output(Output),
|
|
|
|
lists:foreach(
|
|
|
|
fun(Module) ->
|
|
|
|
export(LServer, IO, Module)
|
|
|
|
end, Modules),
|
|
|
|
close_output(Output, IO).
|
|
|
|
|
2017-10-25 20:21:52 +02:00
|
|
|
export(Server, Output, Module1) ->
|
|
|
|
Module = case Module1 of
|
|
|
|
mod_pubsub -> pubsub_db;
|
|
|
|
_ -> Module1
|
|
|
|
end,
|
|
|
|
SQLMod = gen_mod:db_mod(sql, Module),
|
2015-11-24 16:44:13 +01:00
|
|
|
LServer = jid:nameprep(iolist_to_binary(Server)),
|
2013-03-14 10:33:02 +01:00
|
|
|
IO = prepare_output(Output),
|
|
|
|
lists:foreach(
|
|
|
|
fun({Table, ConvertFun}) ->
|
2016-05-04 20:01:05 +02:00
|
|
|
case export(LServer, Table, IO, ConvertFun) of
|
|
|
|
{atomic, ok} -> ok;
|
2017-10-27 07:59:49 +02:00
|
|
|
{aborted, {no_exists, _}} ->
|
|
|
|
?WARNING_MSG("Ignoring export for module ~s: "
|
|
|
|
"Mnesia table ~s doesn't exist (most likely "
|
|
|
|
"because the module is unused)",
|
|
|
|
[Module1, Table]);
|
2016-05-04 20:01:05 +02:00
|
|
|
{aborted, Reason} ->
|
2017-08-03 10:52:44 +02:00
|
|
|
?ERROR_MSG("Failed export for module ~p and table ~p: ~p",
|
|
|
|
[Module, Table, Reason])
|
2016-05-04 20:01:05 +02:00
|
|
|
end
|
2017-10-25 20:21:52 +02:00
|
|
|
end, SQLMod:export(Server)),
|
2013-03-14 10:33:02 +01:00
|
|
|
close_output(Output, IO).
|
2012-04-02 06:49:13 +02:00
|
|
|
|
2016-03-21 16:19:06 +01:00
|
|
|
delete(Server) ->
|
|
|
|
Modules = modules(),
|
|
|
|
lists:foreach(
|
|
|
|
fun(Module) ->
|
|
|
|
delete(Server, Module)
|
|
|
|
end, Modules).
|
|
|
|
|
|
|
|
delete(Server, Module) ->
|
|
|
|
LServer = jid:nameprep(iolist_to_binary(Server)),
|
|
|
|
lists:foreach(
|
|
|
|
fun({Table, ConvertFun}) ->
|
|
|
|
delete(LServer, Table, ConvertFun)
|
|
|
|
end, Module:export(Server)).
|
|
|
|
|
2016-11-22 14:48:01 +01:00
|
|
|
import(Server, Dir, ToType) ->
|
2013-07-21 12:24:36 +02:00
|
|
|
lists:foreach(
|
2016-11-22 14:48:01 +01:00
|
|
|
fun(Mod) ->
|
|
|
|
?INFO_MSG("importing ~p...", [Mod]),
|
|
|
|
import(Mod, Server, Dir, ToType)
|
|
|
|
end, modules()).
|
2013-07-21 12:24:36 +02:00
|
|
|
|
2016-11-22 14:48:01 +01:00
|
|
|
import(Mod, Server, Dir, ToType) ->
|
2015-11-24 16:44:13 +01:00
|
|
|
LServer = jid:nameprep(iolist_to_binary(Server)),
|
2016-11-22 14:48:01 +01:00
|
|
|
try Mod:import_start(LServer, ToType)
|
|
|
|
catch error:undef -> ok end,
|
2013-07-21 12:24:36 +02:00
|
|
|
lists:foreach(
|
2016-11-22 14:48:01 +01:00
|
|
|
fun({File, Tab, _Mod, FieldsNumber}) ->
|
|
|
|
FileName = filename:join([Dir, File]),
|
|
|
|
case open_sql_dump(FileName) of
|
|
|
|
{ok, #sql_dump{type = FromType} = Dump} ->
|
|
|
|
import_rows(LServer, {sql, FromType}, ToType,
|
|
|
|
Tab, Mod, Dump, FieldsNumber),
|
|
|
|
close_sql_dump(Dump);
|
|
|
|
{error, enoent} ->
|
|
|
|
ok;
|
|
|
|
eof ->
|
|
|
|
?INFO_MSG("It seems like SQL dump ~s is empty", [FileName]);
|
|
|
|
Err ->
|
|
|
|
?ERROR_MSG("Failed to open SQL dump ~s: ~s",
|
|
|
|
[FileName, format_error(Err)])
|
|
|
|
end
|
|
|
|
end, import_info(Mod)),
|
|
|
|
try Mod:import_stop(LServer, ToType)
|
|
|
|
catch error:undef -> ok end.
|
|
|
|
|
|
|
|
import_info(Mod) ->
|
|
|
|
Info = Mod:import_info(),
|
|
|
|
lists:map(
|
|
|
|
fun({Tab, FieldsNum}) ->
|
|
|
|
FileName = <<Tab/binary, ".txt">>,
|
|
|
|
{FileName, Tab, Mod, FieldsNum}
|
|
|
|
end, Info).
|
2013-07-21 12:24:36 +02:00
|
|
|
|
2005-08-29 21:00:10 +02:00
|
|
|
%%%----------------------------------------------------------------------
|
|
|
|
%%% Internal functions
|
|
|
|
%%%----------------------------------------------------------------------
|
2013-03-14 10:33:02 +01:00
|
|
|
export(LServer, Table, IO, ConvertFun) ->
|
|
|
|
F = fun () ->
|
|
|
|
mnesia:read_lock_table(Table),
|
|
|
|
{_N, SQLs} =
|
|
|
|
mnesia:foldl(
|
|
|
|
fun(R, {N, SQLs} = Acc) ->
|
|
|
|
case ConvertFun(LServer, R) of
|
|
|
|
[] ->
|
|
|
|
Acc;
|
2016-05-04 20:01:05 +02:00
|
|
|
SQL1 ->
|
|
|
|
SQL = format_queries(SQL1),
|
2013-03-14 10:33:02 +01:00
|
|
|
if N < (?MAX_RECORDS_PER_TRANSACTION) - 1 ->
|
|
|
|
{N + 1, [SQL | SQLs]};
|
|
|
|
true ->
|
|
|
|
output(LServer,
|
|
|
|
Table, IO,
|
|
|
|
flatten([SQL | SQLs])),
|
|
|
|
{0, []}
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end,
|
|
|
|
{0, []}, Table),
|
|
|
|
output(LServer, Table, IO, flatten(SQLs))
|
|
|
|
end,
|
|
|
|
mnesia:transaction(F).
|
|
|
|
|
|
|
|
output(_LServer, _Table, _IO, []) ->
|
|
|
|
ok;
|
2016-04-20 11:27:32 +02:00
|
|
|
output(LServer, _Table, sql, SQLs) ->
|
2017-03-08 16:22:13 +01:00
|
|
|
{atomic, ok} = ejabberd_sql:sql_transaction(LServer, SQLs),
|
|
|
|
ok;
|
2013-03-14 10:33:02 +01:00
|
|
|
output(_LServer, Table, Fd, SQLs) ->
|
|
|
|
file:write(Fd, ["-- \n-- Mnesia table: ", atom_to_list(Table),
|
|
|
|
"\n--\n", SQLs]).
|
|
|
|
|
2016-03-21 16:19:06 +01:00
|
|
|
delete(LServer, Table, ConvertFun) ->
|
|
|
|
F = fun () ->
|
|
|
|
mnesia:write_lock_table(Table),
|
|
|
|
{_N, SQLs} =
|
|
|
|
mnesia:foldl(
|
2016-07-07 11:17:38 +02:00
|
|
|
fun(R, Acc) ->
|
2016-03-21 16:19:06 +01:00
|
|
|
case ConvertFun(LServer, R) of
|
|
|
|
[] ->
|
|
|
|
Acc;
|
|
|
|
_SQL ->
|
|
|
|
mnesia:delete_object(R),
|
|
|
|
Acc
|
|
|
|
end
|
|
|
|
end,
|
|
|
|
{0, []}, Table),
|
|
|
|
delete(LServer, Table, SQLs)
|
|
|
|
end,
|
|
|
|
mnesia:transaction(F).
|
|
|
|
|
2013-07-21 12:24:36 +02:00
|
|
|
prepare_output(FileName) ->
|
|
|
|
prepare_output(FileName, normal).
|
|
|
|
|
|
|
|
prepare_output(FileName, Type) when is_binary(FileName) ->
|
|
|
|
prepare_output(binary_to_list(FileName), Type);
|
|
|
|
prepare_output(FileName, normal) when is_list(FileName) ->
|
2013-03-14 10:33:02 +01:00
|
|
|
case file:open(FileName, [write, raw]) of
|
|
|
|
{ok, Fd} ->
|
|
|
|
Fd;
|
2018-08-13 13:56:29 +02:00
|
|
|
{error, eacces} ->
|
|
|
|
exit({"Not enough permission to the file or path", FileName});
|
|
|
|
{error, enoent} ->
|
|
|
|
exit({"Path does not exist", FileName});
|
2013-03-14 10:33:02 +01:00
|
|
|
Err ->
|
|
|
|
exit(Err)
|
|
|
|
end;
|
2013-07-21 12:24:36 +02:00
|
|
|
prepare_output(Output, _Type) ->
|
2013-03-14 10:33:02 +01:00
|
|
|
Output.
|
|
|
|
|
|
|
|
close_output(FileName, Fd) when FileName /= Fd ->
|
2016-11-22 14:48:01 +01:00
|
|
|
file:close(Fd),
|
2013-03-14 10:33:02 +01:00
|
|
|
ok;
|
|
|
|
close_output(_, _) ->
|
|
|
|
ok.
|
|
|
|
|
|
|
|
flatten(SQLs) ->
|
|
|
|
flatten(SQLs, []).
|
|
|
|
|
|
|
|
flatten([L|Ls], Acc) ->
|
|
|
|
flatten(Ls, flatten1(lists:reverse(L), Acc));
|
|
|
|
flatten([], Acc) ->
|
|
|
|
Acc.
|
|
|
|
|
|
|
|
flatten1([H|T], Acc) ->
|
|
|
|
flatten1(T, [[H, $\n]|Acc]);
|
|
|
|
flatten1([], Acc) ->
|
|
|
|
Acc.
|
2016-05-04 20:01:05 +02:00
|
|
|
|
2016-11-22 14:48:01 +01:00
|
|
|
import_rows(LServer, FromType, ToType, Tab, Mod, Dump, FieldsNumber) ->
|
|
|
|
case read_row_from_sql_dump(Dump, FieldsNumber) of
|
|
|
|
{ok, Fields} ->
|
|
|
|
case catch Mod:import(LServer, FromType, ToType, Tab, Fields) of
|
|
|
|
ok ->
|
|
|
|
ok;
|
|
|
|
Err ->
|
|
|
|
?ERROR_MSG("Failed to import fields ~p for tab ~p: ~p",
|
|
|
|
[Fields, Tab, Err])
|
|
|
|
end,
|
|
|
|
import_rows(LServer, FromType, ToType,
|
|
|
|
Tab, Mod, Dump, FieldsNumber);
|
|
|
|
eof ->
|
|
|
|
ok;
|
|
|
|
Err ->
|
|
|
|
?ERROR_MSG("Failed to read row from SQL dump: ~s",
|
|
|
|
[format_error(Err)])
|
|
|
|
end.
|
|
|
|
|
|
|
|
open_sql_dump(FileName) ->
|
|
|
|
case file:open(FileName, [raw, read, binary, read_ahead]) of
|
|
|
|
{ok, Fd} ->
|
|
|
|
case file:read(Fd, 11) of
|
|
|
|
{ok, <<"PGCOPY\n", 16#ff, "\r\n", 0>>} ->
|
|
|
|
case skip_pgcopy_header(Fd) of
|
|
|
|
ok ->
|
|
|
|
{ok, #sql_dump{fd = Fd, type = pgsql}};
|
|
|
|
Err ->
|
|
|
|
Err
|
|
|
|
end;
|
|
|
|
{ok, _} ->
|
|
|
|
file:position(Fd, 0),
|
|
|
|
{ok, #sql_dump{fd = Fd, type = mysql}};
|
|
|
|
Err ->
|
|
|
|
Err
|
|
|
|
end;
|
|
|
|
Err ->
|
|
|
|
Err
|
|
|
|
end.
|
|
|
|
|
|
|
|
close_sql_dump(#sql_dump{fd = Fd}) ->
|
|
|
|
file:close(Fd).
|
|
|
|
|
|
|
|
read_row_from_sql_dump(#sql_dump{fd = Fd, type = pgsql}, _) ->
|
|
|
|
case file:read(Fd, 2) of
|
|
|
|
{ok, <<(-1):16/signed>>} ->
|
|
|
|
eof;
|
|
|
|
{ok, <<FieldsNum:16>>} ->
|
|
|
|
read_fields(Fd, FieldsNum, []);
|
|
|
|
{ok, _} ->
|
|
|
|
{error, eof};
|
|
|
|
eof ->
|
|
|
|
{error, eof};
|
|
|
|
{error, _} = Err ->
|
|
|
|
Err
|
|
|
|
end;
|
|
|
|
read_row_from_sql_dump(#sql_dump{fd = Fd, type = mysql}, FieldsNum) ->
|
|
|
|
read_lines(Fd, FieldsNum, <<"">>, []).
|
|
|
|
|
|
|
|
skip_pgcopy_header(Fd) ->
|
|
|
|
try
|
|
|
|
{ok, <<_:4/binary, ExtSize:32>>} = file:read(Fd, 8),
|
|
|
|
{ok, <<_:ExtSize/binary>>} = file:read(Fd, ExtSize),
|
|
|
|
ok
|
|
|
|
catch error:{badmatch, {error, _} = Err} ->
|
|
|
|
Err;
|
|
|
|
error:{badmatch, _} ->
|
|
|
|
{error, eof}
|
|
|
|
end.
|
|
|
|
|
|
|
|
read_fields(_Fd, 0, Acc) ->
|
|
|
|
{ok, lists:reverse(Acc)};
|
|
|
|
read_fields(Fd, N, Acc) ->
|
|
|
|
case file:read(Fd, 4) of
|
|
|
|
{ok, <<(-1):32/signed>>} ->
|
|
|
|
read_fields(Fd, N-1, [null|Acc]);
|
|
|
|
{ok, <<ValSize:32>>} ->
|
|
|
|
case file:read(Fd, ValSize) of
|
|
|
|
{ok, <<Val:ValSize/binary>>} ->
|
|
|
|
read_fields(Fd, N-1, [Val|Acc]);
|
|
|
|
{ok, _} ->
|
|
|
|
{error, eof};
|
|
|
|
Err ->
|
|
|
|
Err
|
|
|
|
end;
|
|
|
|
{ok, _} ->
|
|
|
|
{error, eof};
|
|
|
|
eof ->
|
|
|
|
{error, eof};
|
|
|
|
{error, _} = Err ->
|
|
|
|
Err
|
|
|
|
end.
|
|
|
|
|
|
|
|
read_lines(_Fd, 0, <<"">>, Acc) ->
|
|
|
|
{ok, lists:reverse(Acc)};
|
|
|
|
read_lines(Fd, N, Buf, Acc) ->
|
|
|
|
case file:read_line(Fd) of
|
|
|
|
{ok, Data} when size(Data) >= 2 ->
|
|
|
|
Size = size(Data) - 2,
|
|
|
|
case Data of
|
|
|
|
<<Val:Size/binary, 0, $\n>> ->
|
|
|
|
NewBuf = <<Buf/binary, Val/binary>>,
|
|
|
|
read_lines(Fd, N-1, <<"">>, [NewBuf|Acc]);
|
|
|
|
_ ->
|
|
|
|
NewBuf = <<Buf/binary, Data/binary>>,
|
|
|
|
read_lines(Fd, N, NewBuf, Acc)
|
|
|
|
end;
|
|
|
|
{ok, Data} ->
|
|
|
|
NewBuf = <<Buf/binary, Data/binary>>,
|
|
|
|
read_lines(Fd, N, NewBuf, Acc);
|
|
|
|
eof when Buf == <<"">>, Acc == [] ->
|
|
|
|
eof;
|
|
|
|
eof ->
|
|
|
|
{error, eof};
|
|
|
|
{error, _} = Err ->
|
|
|
|
Err
|
|
|
|
end.
|
|
|
|
|
|
|
|
format_error({error, eof}) ->
|
|
|
|
"unexpected end of file";
|
|
|
|
format_error({error, Posix}) ->
|
|
|
|
file:format_error(Posix).
|
|
|
|
|
2016-05-04 20:01:05 +02:00
|
|
|
format_queries(SQLs) ->
|
|
|
|
lists:map(
|
|
|
|
fun(#sql_query{} = SQL) ->
|
|
|
|
ejabberd_sql:sql_query_to_iolist(SQL);
|
|
|
|
(SQL) ->
|
|
|
|
SQL
|
|
|
|
end, SQLs).
|