diff --git a/src/ejabberd_auth_mnesia.erl b/src/ejabberd_auth_mnesia.erl index 457e4c1b8..f856c8a9c 100644 --- a/src/ejabberd_auth_mnesia.erl +++ b/src/ejabberd_auth_mnesia.erl @@ -42,6 +42,7 @@ get_password_s/2, is_user_exists/2, remove_user/2, remove_user/3, store_type/0, export/1, import/2, plain_password_required/0, opt_type/1]). +-export([need_transform/1, transform/1]). -include("ejabberd.hrl"). -include("logger.hrl"). @@ -60,9 +61,7 @@ %%%---------------------------------------------------------------------- start(Host) -> init_db(), - update_table(), update_reg_users_counter_table(Host), - maybe_alert_password_scrammed_without_option(), ok. stop(_Host) -> @@ -90,10 +89,8 @@ plain_password_required() -> is_scrammed(). store_type() -> - case is_scrammed() of - false -> plain; %% allows: PLAIN DIGEST-MD5 SCRAM - true -> scram %% allows: PLAIN SCRAM - end. + ejabberd_config:get_option({auth_password_format, ?MYNAME}, + opt_type(auth_password_format), plain). check_password(User, AuthzId, Server, Password) -> if AuthzId /= <<>> andalso AuthzId /= User -> @@ -374,100 +371,72 @@ remove_user(User, Server, Password) -> _ -> bad_request end. -update_table() -> - Fields = record_info(fields, passwd), - case mnesia:table_info(passwd, attributes) of - Fields -> - convert_to_binary(Fields), - maybe_scram_passwords(), - ok; - _ -> - ?INFO_MSG("Recreating passwd table", []), - mnesia:transform_table(passwd, ignore, Fields) +need_transform(#passwd{us = {U, S}, password = Pass}) -> + if is_binary(Pass) -> + IsScrammed = is_scrammed(), + if IsScrammed -> + ?INFO_MSG("Passwords in Mnesia table 'passwd' " + "will be SCRAM'ed", []); + true -> + ok + end, + IsScrammed; + is_record(Pass, scram) -> + case is_scrammed() of + true -> + next; + false -> + ?WARNING_MSG("Some passwords were stored in the database " + "as SCRAM, but 'auth_password_format' " + "is not configured as 'scram'.", []), + false + end; + is_list(U) orelse is_list(S) orelse is_list(Pass) -> + ?INFO_MSG("Mnesia table 'passwd' will be converted to binary", []), + true end. -convert_to_binary(Fields) -> - ejabberd_config:convert_table_to_binary( - passwd, Fields, set, - fun(#passwd{us = {U, _}}) -> U end, - fun(#passwd{us = {U, S}, password = Pass} = R) -> - NewUS = {iolist_to_binary(U), iolist_to_binary(S)}, - NewPass = case Pass of - #scram{storedkey = StoredKey, - serverkey = ServerKey, - salt = Salt} -> - Pass#scram{ - storedkey = iolist_to_binary(StoredKey), - serverkey = iolist_to_binary(ServerKey), - salt = iolist_to_binary(Salt)}; - _ -> - iolist_to_binary(Pass) - end, - R#passwd{us = NewUS, password = NewPass} - end). +transform(#passwd{us = {U, S}, password = Pass} = R) + when is_list(U) orelse is_list(S) orelse is_list(Pass) -> + NewUS = {iolist_to_binary(U), iolist_to_binary(S)}, + NewPass = case Pass of + #scram{storedkey = StoredKey, + serverkey = ServerKey, + salt = Salt} -> + Pass#scram{ + storedkey = iolist_to_binary(StoredKey), + serverkey = iolist_to_binary(ServerKey), + salt = iolist_to_binary(Salt)}; + _ -> + iolist_to_binary(Pass) + end, + transform(R#passwd{us = NewUS, password = NewPass}); +transform(#passwd{us = {U, S}, password = Password} = P) + when is_binary(Password) -> + case is_scrammed() of + true -> + case jid:resourceprep(Password) of + error -> + ?ERROR_MSG("SASLprep failed for password of user ~s@~s", + [U, S]), + P; + _ -> + Scram = password_to_scram(Password), + P#passwd{password = Scram} + end; + false -> + P + end; +transform(#passwd{password = Password} = P) + when is_record(Password, scram) -> + P. %%% %%% SCRAM %%% -%% The passwords are stored scrammed in the table either if the option says so, -%% or if at least the first password is scrammed. is_scrammed() -> - OptionScram = is_option_scram(), - FirstElement = mnesia:dirty_read(passwd, - mnesia:dirty_first(passwd)), - case {OptionScram, FirstElement} of - {true, _} -> true; - {false, [#passwd{password = Scram}]} - when is_record(Scram, scram) -> - true; - _ -> false - end. - -is_option_scram() -> - scram == - ejabberd_config:get_option({auth_password_format, ?MYNAME}, - fun(V) -> V end). - -maybe_alert_password_scrammed_without_option() -> - case is_scrammed() andalso not is_option_scram() of - true -> - ?ERROR_MSG("Some passwords were stored in the database " - "as SCRAM, but 'auth_password_format' " - "is not configured 'scram'. The option " - "will now be considered to be 'scram'.", - []); - false -> ok - end. - -maybe_scram_passwords() -> - case is_scrammed() of - true -> scram_passwords(); - false -> ok - end. - -scram_passwords() -> - ?INFO_MSG("Converting the stored passwords into " - "SCRAM bits", - []), - Fun = fun (#passwd{us = {U, S}, password = Password} = P) - when is_binary(Password) -> - case jid:resourceprep(Password) of - error -> - ?ERROR_MSG( - "SASLprep failed for " - "password of user ~s@~s", - [U, S]), - P; - _ -> - Scram = password_to_scram(Password), - P#passwd{password = Scram} - end; - (P) -> - P - end, - Fields = record_info(fields, passwd), - mnesia:transform_table(passwd, Fun, Fields). + scram == store_type(). password_to_scram(Password) -> password_to_scram(Password, @@ -526,5 +495,8 @@ import(LServer, [LUser, Password, _TimeStamp]) -> mnesia:dirty_write( #passwd{us = {LUser, LServer}, password = Password}). -opt_type(auth_password_format) -> fun (V) -> V end; +opt_type(auth_password_format) -> + fun (plain) -> plain; + (scram) -> scram + end; opt_type(_) -> [auth_password_format]. diff --git a/src/ejabberd_auth_riak.erl b/src/ejabberd_auth_riak.erl index c41e8f63a..9555fcad8 100644 --- a/src/ejabberd_auth_riak.erl +++ b/src/ejabberd_auth_riak.erl @@ -274,7 +274,7 @@ remove_user(User, Server, Password) -> is_scrammed() -> scram == ejabberd_config:get_option({auth_password_format, ?MYNAME}, - fun(V) -> V end). + opt_type(auth_password_format), plain). password_to_scram(Password) -> password_to_scram(Password, @@ -321,5 +321,8 @@ import(LServer, [LUser, Password, _TimeStamp]) -> Passwd = #passwd{us = {LUser, LServer}, password = Password}, ejabberd_riak:put(Passwd, passwd_schema(), [{'2i', [{<<"host">>, LServer}]}]). -opt_type(auth_password_format) -> fun (V) -> V end; +opt_type(auth_password_format) -> + fun (plain) -> plain; + (scram) -> scram + end; opt_type(_) -> [auth_password_format]. diff --git a/src/ejabberd_auth_sql.erl b/src/ejabberd_auth_sql.erl index d64909771..0d9665afc 100644 --- a/src/ejabberd_auth_sql.erl +++ b/src/ejabberd_auth_sql.erl @@ -410,7 +410,7 @@ remove_user(User, Server, Password) -> is_scrammed() -> scram == ejabberd_config:get_option({auth_password_format, ?MYNAME}, - fun(V) -> V end). + opt_type(auth_password_format), plain). password_to_scram(Password) -> password_to_scram(Password, @@ -510,5 +510,8 @@ convert_to_scram(Server) -> end end. -opt_type(auth_password_format) -> fun (V) -> V end; +opt_type(auth_password_format) -> + fun (plain) -> plain; + (scram) -> scram + end; opt_type(_) -> [auth_password_format]. diff --git a/src/ejabberd_config.erl b/src/ejabberd_config.erl index bbef755a9..139549413 100644 --- a/src/ejabberd_config.erl +++ b/src/ejabberd_config.erl @@ -31,8 +31,7 @@ get_vh_by_auth_method/1, is_file_readable/1, get_version/0, get_myhosts/0, get_mylang/0, get_ejabberd_config_path/0, is_using_elixir_config/0, - prepare_opt_val/4, convert_table_to_binary/5, - transform_options/1, collect_options/1, + prepare_opt_val/4, transform_options/1, collect_options/1, convert_to_yaml/1, convert_to_yaml/2, v_db/2, env_binary_to_list/2, opt_type/1, may_hide_data/1, is_elixir_enabled/0, v_dbs/1, v_dbs_mods/1, @@ -1339,94 +1338,6 @@ transform_options({include_config_file, _, _} = Opt, Opts) -> transform_options(Opt, Opts) -> [Opt|Opts]. --spec convert_table_to_binary(atom(), [atom()], atom(), - fun(), fun()) -> ok. - -convert_table_to_binary(Tab, Fields, Type, DetectFun, ConvertFun) -> - case is_table_still_list(Tab, DetectFun) of - true -> - ?INFO_MSG("Converting '~s' table from strings to binaries.", [Tab]), - TmpTab = list_to_atom(atom_to_list(Tab) ++ "_tmp_table"), - catch mnesia:delete_table(TmpTab), - case ejabberd_mnesia:create(?MODULE, TmpTab, - [{disc_only_copies, [node()]}, - {type, Type}, - {local_content, true}, - {record_name, Tab}, - {attributes, Fields}]) of - {atomic, ok} -> - mnesia:transform_table(Tab, ignore, Fields), - case mnesia:transaction( - fun() -> - mnesia:write_lock_table(TmpTab), - mnesia:foldl( - fun(R, _) -> - NewR = ConvertFun(R), - mnesia:dirty_write(TmpTab, NewR) - end, ok, Tab) - end) of - {atomic, ok} -> - mnesia:clear_table(Tab), - case mnesia:transaction( - fun() -> - mnesia:write_lock_table(Tab), - mnesia:foldl( - fun(R, _) -> - mnesia:dirty_write(R) - end, ok, TmpTab) - end) of - {atomic, ok} -> - mnesia:delete_table(TmpTab); - Err -> - report_and_stop(Tab, Err) - end; - Err -> - report_and_stop(Tab, Err) - end; - Err -> - report_and_stop(Tab, Err) - end; - false -> - ok - end. - -is_table_still_list(Tab, DetectFun) -> - is_table_still_list(Tab, DetectFun, mnesia:dirty_first(Tab)). - -is_table_still_list(_Tab, _DetectFun, '$end_of_table') -> - false; -is_table_still_list(Tab, DetectFun, Key) -> - Rs = mnesia:dirty_read(Tab, Key), - Res = lists:foldl(fun(_, true) -> - true; - (_, false) -> - false; - (R, _) -> - case DetectFun(R) of - '$next' -> - '$next'; - El -> - is_list(El) - end - end, '$next', Rs), - case Res of - true -> - true; - false -> - false; - '$next' -> - is_table_still_list(Tab, DetectFun, mnesia:dirty_next(Tab, Key)) - end. - -report_and_stop(Tab, Err) -> - ErrTxt = lists:flatten( - io_lib:format( - "Failed to convert '~s' table to binary: ~p", - [Tab, Err])), - ?CRITICAL_MSG(ErrTxt, []), - timer:sleep(1000), - halt(string:substr(ErrTxt, 1, 199)). - emit_deprecation_warning(Module, NewModule, DBType) -> ?WARNING_MSG("Module ~s is deprecated, use ~s with 'db_type: ~s'" " instead", [Module, NewModule, DBType]). diff --git a/src/ejabberd_mnesia.erl b/src/ejabberd_mnesia.erl index 1a1a62a51..34691545a 100644 --- a/src/ejabberd_mnesia.erl +++ b/src/ejabberd_mnesia.erl @@ -33,7 +33,8 @@ -behaviour(gen_server). --export([start/0, create/3, reset/2, update/2]). +-export([start/0, create/3, update/2, transform/2, transform/3, + dump_schema/0]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). @@ -43,14 +44,18 @@ -include("logger.hrl"). --record(state, {tables = #{} :: map()}). +-record(state, {tables = #{} :: map(), + schema = [] :: [{atom(), [{atom(), any()}]}]}). start() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). +-spec create(module(), atom(), list()) -> any(). create(Module, Name, TabDef) -> gen_server:call(?MODULE, {create, Module, Name, TabDef}, - timer:seconds(60)). + %% Huge timeout is need to have enough + %% time to transform huge tables + timer:minutes(30)). init([]) -> ejabberd_config:env_binary_to_list(mnesia, dir), @@ -65,7 +70,8 @@ init([]) -> ejabberd:start_app(mnesia, permanent), ?DEBUG("Waiting for Mnesia tables synchronization...", []), mnesia:wait_for_tables(mnesia:system_info(local_tables), infinity), - {ok, #state{}}; + Schema = read_schema_file(), + {ok, #state{schema = Schema}}; false -> ?CRITICAL_MSG("Node name mismatch: I'm [~s], " "the database is owned by ~p", [MyNode, DbNodes]), @@ -79,9 +85,9 @@ handle_call({create, Module, Name, TabDef}, _From, State) -> {TabDef, Result} -> {reply, Result, State}; _ -> - Result = do_create(Module, Name, TabDef), + Result = do_create(Module, Name, TabDef, State#state.schema), Tables = maps:put(Name, {TabDef, Result}, State#state.tables), - {reply, Result, #state{tables = Tables}} + {reply, Result, State#state{tables = Tables}} end; handle_call(_Request, _From, State) -> {noreply, State}. @@ -98,90 +104,178 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -do_create(Module, Name, TabDef) - when is_atom(Module), is_atom(Name), is_list(TabDef) -> - Path = os:getenv("EJABBERD_SCHEMA_PATH"), - Schema = schema(Path, Module, Name, TabDef), +do_create(Module, Name, TabDef, TabDefs) -> + code:ensure_loaded(Module), + Schema = schema(Name, TabDef, TabDefs), {attributes, Attrs} = lists:keyfind(attributes, 1, Schema), case catch mnesia:table_info(Name, attributes) of {'EXIT', _} -> create(Name, TabDef); Attrs -> case need_reset(Name, Schema) of - true -> reset(Name, Schema); - false -> update(Name, Attrs, Schema) + true -> + reset(Name, Schema); + false -> + case update(Name, Attrs, Schema) of + {atomic, ok} -> + transform(Module, Name, Attrs, Attrs); + Err -> + Err + end end; OldAttrs -> - Fun = case lists:member({transform,1}, Module:module_info(exports)) of - true -> fun(Old) -> Module:transform(Old) end; - false -> fun(Old) -> transform(OldAttrs, Attrs, Old) end - end, - mnesia_op(transform_table, [Name, Fun, Attrs]) + transform(Module, Name, OldAttrs, Attrs) end. -reset(Name, TabDef) - when is_atom(Name), is_list(TabDef) -> +reset(Name, TabDef) -> + ?INFO_MSG("Deleting Mnesia table '~s'", [Name]), mnesia_op(delete_table, [Name]), create(Name, TabDef). -update(Name, TabDef) - when is_atom(Name), is_list(TabDef) -> +update(Name, TabDef) -> {attributes, Attrs} = lists:keyfind(attributes, 1, TabDef), update(Name, Attrs, TabDef). + update(Name, Attrs, TabDef) -> - Storage = case catch mnesia:table_info(Name, storage_type) of - {'EXIT', _} -> unknown; - Type -> Type - end, - NewStorage = lists:foldl( - fun({Key, _}, Acc) -> - case lists:member(Key, ?STORAGE_TYPES) of - true -> Key; - false -> Acc - end - end, Storage, TabDef), - R1 = [mnesia_op(change_table_copy_type, [Name, node(), NewStorage]) - || Storage=/=NewStorage], - CurIndexes = [lists:nth(N-1, Attrs) || N<-mnesia:table_info(Name, index)], - NewIndexes = proplists:get_value(index, TabDef, []), - R2 = [mnesia_op(del_table_index, [Name, Attr]) - || Attr <- CurIndexes--NewIndexes], - R3 = [mnesia_op(add_table_index, [Name, Attr]) - || Attr <- NewIndexes--CurIndexes], - lists:foldl( - fun({atomic, ok}, Acc) -> Acc; - (Error, _Acc) -> Error - end, {atomic, ok}, R1++R2++R3). + case change_table_copy_type(Name, TabDef) of + {atomic, ok} -> + CurrIndexes = [lists:nth(N-1, Attrs) || + N <- mnesia:table_info(Name, index)], + NewIndexes = proplists:get_value(index, TabDef, []), + case delete_indexes(Name, CurrIndexes -- NewIndexes) of + {atomic, ok} -> + add_indexes(Name, NewIndexes -- CurrIndexes); + Err -> + Err + end; + Err -> + Err + end. + +change_table_copy_type(Name, TabDef) -> + CurrType = mnesia:table_info(Name, storage_type), + NewType = case lists:filter(fun is_storage_type_option/1, TabDef) of + [{Type, _}|_] -> Type; + [] -> CurrType + end, + if NewType /= CurrType -> + ?INFO_MSG("Changing Mnesia table '~s' from ~s to ~s", + [Name, CurrType, NewType]), + mnesia_op(change_table_copy_type, [Name, node(), NewType]); + true -> + {atomic, ok} + end. + +delete_indexes(Name, [Index|Indexes]) -> + ?INFO_MSG("Deleting index '~s' from Mnesia table '~s'", [Index, Name]), + case mnesia_op(del_table_index, [Name, Index]) of + {atomic, ok} -> + delete_indexes(Name, Indexes); + Err -> + Err + end; +delete_indexes(_Name, []) -> + {atomic, ok}. + +add_indexes(Name, [Index|Indexes]) -> + ?INFO_MSG("Adding index '~s' to Mnesia table '~s'", [Index, Name]), + case mnesia_op(add_table_index, [Name, Index]) of + {atomic, ok} -> + add_indexes(Name, Indexes); + Err -> + Err + end; +add_indexes(_Name, []) -> + {atomic, ok}. % % utilities % -schema(false, Module, _Name, TabDef) -> - ?DEBUG("No custom ~s schema path", [Module]), - TabDef; -schema(Path, Module, Name, TabDef) -> - File = filename:join(Path, atom_to_list(Module)++".mnesia"), - case parse(File) of - {ok, CustomDefs} -> - case lists:keyfind(Name, 1, CustomDefs) of - {Name, CustomDef} -> - ?INFO_MSG("Using custom ~s schema for table ~s", - [Module, Name]), - merge(TabDef, CustomDef); - _ -> - TabDef - end; +schema(Name, Default, Schema) -> + case lists:keyfind(Name, 1, Schema) of + {_, Custom} -> + TabDefs = merge(Custom, Default), + ?DEBUG("Using custom schema for table '~s': ~p", + [Name, TabDefs]), + TabDefs; + false -> + ?DEBUG("No custom Mnesia schema for table '~s' found", + [Name]), + Default + end. + +read_schema_file() -> + File = schema_path(), + case fast_yaml:decode_from_file(File, [plain_as_atom]) of + {ok, [Defs|_]} -> + ?INFO_MSG("Using custom Mnesia schema from ~s", [File]), + lists:flatmap( + fun({Tab, Opts}) -> + case validate_schema_opts(File, Opts) of + {ok, NewOpts} -> + [{Tab, lists:ukeysort(1, NewOpts)}]; + error -> + [] + end + end, Defs); + {ok, []} -> + ?WARNING_MSG("Mnesia schema file ~s is empty", [File]), + []; {error, enoent} -> - ?DEBUG("No custom ~s schema path", [Module]), - TabDef; - {error, Error} -> - ?ERROR_MSG("Can not use custom ~s schema for table ~s: ~p", - [Module, Name, Error]), - TabDef + ?DEBUG("No custom Mnesia schema file found", []), + []; + {error, Reason} -> + ?ERROR_MSG("Failed to read Mnesia schema file ~s: ~s", + [File, fast_yaml:format_error(Reason)]), + [] + end. + +validate_schema_opts(File, Opts) -> + try {ok, lists:map( + fun({storage_type, Type}) when Type == ram_copies; + Type == disc_copies; + Type == disc_only_copies -> + {Type, [node()]}; + ({storage_type, _} = Opt) -> + erlang:error({invalid_value, Opt}); + ({local_content, Bool}) when is_boolean(Bool) -> + {local_content, Bool}; + ({local_content, _} = Opt) -> + erlang:error({invalid_value, Opt}); + ({type, Type}) when Type == set; + Type == ordered_set; + Type == bag -> + {type, Type}; + ({type, _} = Opt) -> + erlang:error({invalid_value, Opt}); + ({attributes, Attrs} = Opt) -> + try lists:all(fun is_atom/1, Attrs) of + true -> {attributes, Attrs}; + false -> erlang:error({invalid_value, Opt}) + catch _:_ -> erlang:error({invalid_value, Opt}) + end; + ({index, Indexes} = Opt) -> + try lists:all(fun is_atom/1, Indexes) of + true -> {index, Indexes}; + false -> erlang:error({invalid_value, Opt}) + catch _:_ -> erlang:error({invalid_value, Opt}) + end; + (Opt) -> + erlang:error({unknown_option, Opt}) + end, Opts)} + catch _:{invalid_value, {Opt, Val}} -> + ?ERROR_MSG("Mnesia schema ~s is incorrect: invalid value ~p of " + "option '~s'", [File, Val, Opt]), + error; + _:{unknown_option, Opt} -> + ?ERROR_MSG("Mnesia schema ~s is incorrect: unknown option ~p", + [File, Opt]), + error end. create(Name, TabDef) -> + ?INFO_MSG("Creating Mnesia table '~s'", [Name]), case mnesia_op(create_table, [Name, TabDef]) of {atomic, ok} -> add_table_copy(Name); @@ -200,40 +294,17 @@ add_table_copy(Name) -> mnesia_op(add_table_copy, [Name, node(), Type]) end. -merge(TabDef, CustomDef) -> - {CustomKeys, _} = lists:unzip(CustomDef), - CleanDef = lists:foldl( - fun(Elem, Acc) -> - case lists:member(Elem, ?STORAGE_TYPES) of - true -> - lists:foldl( - fun(Key, CleanAcc) -> - lists:keydelete(Key, 1, CleanAcc) - end, Acc, ?STORAGE_TYPES); - false -> - Acc - end - end, TabDef, CustomKeys), - lists:ukeymerge(1, - lists:ukeysort(1, CustomDef), - lists:ukeysort(1, CleanDef)). - -parse(File) -> - case file:consult(File) of - {ok, Terms} -> parse(Terms, []); - Error -> Error - end. -parse([], Acc) -> - {ok, lists:reverse(Acc)}; -parse([{Name, Storage, TabDef}|Tail], Acc) - when is_atom(Name), is_atom(Storage), is_list(TabDef) -> - NewDef = case lists:member(Storage, ?STORAGE_TYPES) of - true -> [{Storage, [node()]} | TabDef]; - false -> TabDef - end, - parse(Tail, [{Name, NewDef} | Acc]); -parse([Other|_], _) -> - {error, {invalid, Other}}. +merge(Custom, Default) -> + NewDefault = case lists:any(fun is_storage_type_option/1, Custom) of + true -> + lists:filter( + fun(O) -> + not is_storage_type_option(O) + end, Default); + false -> + Default + end, + lists:ukeymerge(1, Custom, lists:ukeysort(1, NewDefault)). need_reset(Table, TabDef) -> ValuesF = [mnesia:table_info(Table, Key) || Key <- ?NEED_RESET], @@ -244,7 +315,61 @@ need_reset(Table, TabDef) -> ({_, _}, _) -> true end, false, lists:zip(ValuesF, ValuesT)). -transform(OldAttrs, Attrs, Old) -> +transform(Module, Name) -> + try mnesia:table_info(Name, attributes) of + Attrs -> + transform(Module, Name, Attrs, Attrs) + catch _:{aborted, _} = Err -> + Err + end. + +transform(Module, Name, NewAttrs) -> + try mnesia:table_info(Name, attributes) of + OldAttrs -> + transform(Module, Name, OldAttrs, NewAttrs) + catch _:{aborted, _} = Err -> + Err + end. + +transform(Module, Name, Attrs, Attrs) -> + case need_transform(Module, Name) of + true -> + ?INFO_MSG("Transforming table '~s', this may take a while", [Name]), + transform_table(Module, Name); + false -> + {atomic, ok} + end; +transform(Module, Name, OldAttrs, NewAttrs) -> + Fun = case erlang:function_exported(Module, transform, 1) of + true -> transform_fun(Module, Name); + false -> fun(Old) -> do_transform(OldAttrs, NewAttrs, Old) end + end, + mnesia_op(transform_table, [Name, Fun, NewAttrs]). + +-spec need_transform(module(), atom()) -> boolean(). +need_transform(Module, Name) -> + case erlang:function_exported(Module, need_transform, 1) of + true -> + do_need_transform(Module, Name, mnesia:dirty_first(Name)); + false -> + false + end. + +do_need_transform(_Module, _Name, '$end_of_table') -> + false; +do_need_transform(Module, Name, Key) -> + Objs = mnesia:dirty_read(Name, Key), + case lists:foldl( + fun(_, true) -> true; + (Obj, _) -> Module:need_transform(Obj) + end, undefined, Objs) of + true -> true; + false -> false; + _ -> + do_need_transform(Module, Name, mnesia:dirty_next(Name, Key)) + end. + +do_transform(OldAttrs, Attrs, Old) -> [Name|OldValues] = tuple_to_list(Old), Before = lists:zip(OldAttrs, OldValues), After = lists:foldl( @@ -257,6 +382,56 @@ transform(OldAttrs, Attrs, Old) -> {Attrs, NewRecord} = lists:unzip(After), list_to_tuple([Name|NewRecord]). +transform_fun(Module, Name) -> + fun(Obj) -> + try Module:transform(Obj) + catch E:R -> + StackTrace = erlang:get_stacktrace(), + ?ERROR_MSG("Failed to transform Mnesia table ~s:~n" + "** Record: ~p~n" + "** Reason: ~p~n" + "** StackTrace: ~p", + [Name, Obj, R, StackTrace]), + erlang:raise(E, R, StackTrace) + end + end. + +transform_table(Module, Name) -> + Type = mnesia:table_info(Name, type), + Attrs = mnesia:table_info(Name, attributes), + TmpTab = list_to_atom(atom_to_list(Name) ++ "_backup"), + StorageType = if Type == ordered_set -> disc_copies; + true -> disc_only_copies + end, + mnesia:create_table(TmpTab, + [{StorageType, [node()]}, + {type, Type}, + {local_content, true}, + {record_name, Name}, + {attributes, Attrs}]), + mnesia:clear_table(TmpTab), + Fun = transform_fun(Module, Name), + Res = mnesia_op( + transaction, + [fun() -> do_transform_table(Name, Fun, TmpTab, mnesia:first(Name)) end]), + mnesia:delete_table(TmpTab), + Res. + +do_transform_table(Name, _Fun, TmpTab, '$end_of_table') -> + mnesia:foldl( + fun(Obj, _) -> + mnesia:write(Name, Obj, write) + end, ok, TmpTab); +do_transform_table(Name, Fun, TmpTab, Key) -> + Next = mnesia:next(Name, Key), + Objs = mnesia:read(Name, Key), + lists:foreach( + fun(Obj) -> + mnesia:write(TmpTab, Fun(Obj), write), + mnesia:delete_object(Obj) + end, Objs), + do_transform_table(Name, Fun, TmpTab, Next). + mnesia_op(Fun, Args) -> case apply(mnesia, Fun, Args) of {atomic, ok} -> @@ -266,3 +441,32 @@ mnesia_op(Fun, Args) -> [Fun, Args, Other]), Other end. + +schema_path() -> + Dir = case os:getenv("EJABBERD_MNESIA_SCHEMA") of + false -> mnesia:system_info(directory); + Path -> Path + end, + filename:join(Dir, "ejabberd.schema"). + +is_storage_type_option({O, _}) -> + O == ram_copies orelse O == disc_copies orelse O == disc_only_copies. + +dump_schema() -> + File = schema_path(), + Schema = lists:flatmap( + fun(schema) -> + []; + (Tab) -> + [{Tab, [{storage_type, + mnesia:table_info(Tab, storage_type)}, + {local_content, + mnesia:table_info(Tab, local_content)}]}] + end, mnesia:system_info(tables)), + case file:write_file(File, [fast_yaml:encode(Schema), io_lib:nl()]) of + ok -> + io:format("Mnesia schema is written to ~s~n", [File]); + {error, Reason} -> + io:format("Failed to write Mnesia schema to ~s: ~s", + [File, file:format_error(Reason)]) + end. diff --git a/src/mod_announce_mnesia.erl b/src/mod_announce_mnesia.erl index 09316189a..c298627eb 100644 --- a/src/mod_announce_mnesia.erl +++ b/src/mod_announce_mnesia.erl @@ -29,6 +29,7 @@ %% API -export([init/2, set_motd_users/2, set_motd/2, delete_motd/1, get_motd/1, is_motd_user/2, set_motd_user/2, import/3]). +-export([need_transform/1, transform/1]). -include("xmpp.hrl"). -include("mod_announce.hrl"). @@ -45,8 +46,7 @@ init(_Host, _Opts) -> ejabberd_mnesia:create(?MODULE, motd_users, [{disc_copies, [node()]}, {attributes, - record_info(fields, motd_users)}]), - update_tables(). + record_info(fields, motd_users)}]). set_motd_users(_LServer, USRs) -> F = fun() -> @@ -98,6 +98,23 @@ set_motd_user(LUser, LServer) -> end, mnesia:transaction(F). +need_transform(#motd{server = S}) when is_list(S) -> + ?INFO_MSG("Mnesia table 'motd' will be converted to binary", []), + true; +need_transform(#motd_users{us = {U, S}}) when is_list(U) orelse is_list(S) -> + ?INFO_MSG("Mnesia table 'motd_users' will be converted to binary", []), + true; +need_transform(_) -> + false. + +transform(#motd{server = S, packet = P} = R) -> + NewS = iolist_to_binary(S), + NewP = fxml:to_xmlel(P), + R#motd{server = NewS, packet = NewP}; +transform(#motd_users{us = {U, S}} = R) -> + NewUS = {iolist_to_binary(U), iolist_to_binary(S)}, + R#motd_users{us = NewUS}. + import(LServer, <<"motd">>, [<<>>, XML, _TimeStamp]) -> El = fxml_stream:parse_element(XML), mnesia:dirty_write(#motd{server = LServer, packet = El}); @@ -107,41 +124,3 @@ import(LServer, <<"motd">>, [LUser, <<>>, _TimeStamp]) -> %%%=================================================================== %%% Internal functions %%%=================================================================== -update_tables() -> - update_motd_table(), - update_motd_users_table(). - -update_motd_table() -> - Fields = record_info(fields, motd), - case mnesia:table_info(motd, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - motd, Fields, set, - fun(#motd{server = S}) -> S end, - fun(#motd{server = S, packet = P} = R) -> - NewS = iolist_to_binary(S), - NewP = fxml:to_xmlel(P), - R#motd{server = NewS, packet = NewP} - end); - _ -> - ?INFO_MSG("Recreating motd table", []), - mnesia:transform_table(motd, ignore, Fields) - end. - - -update_motd_users_table() -> - Fields = record_info(fields, motd_users), - case mnesia:table_info(motd_users, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - motd_users, Fields, set, - fun(#motd_users{us = {U, _}}) -> U end, - fun(#motd_users{us = {U, S}} = R) -> - NewUS = {iolist_to_binary(U), - iolist_to_binary(S)}, - R#motd_users{us = NewUS} - end); - _ -> - ?INFO_MSG("Recreating motd_users table", []), - mnesia:transform_table(motd_users, ignore, Fields) - end. diff --git a/src/mod_caps_mnesia.erl b/src/mod_caps_mnesia.erl index 30fbf766f..7362b1cd1 100644 --- a/src/mod_caps_mnesia.erl +++ b/src/mod_caps_mnesia.erl @@ -28,6 +28,7 @@ %% API -export([init/2, caps_read/2, caps_write/3, import/3]). +-export([need_transform/1, transform/1]). -include("mod_caps.hrl"). -include("logger.hrl"). @@ -36,20 +37,10 @@ %%% API %%%=================================================================== init(_Host, _Opts) -> - case catch mnesia:table_info(caps_features, storage_type) of - {'EXIT', _} -> - ok; - disc_only_copies -> - ok; - _ -> - mnesia:delete_table(caps_features) - end, ejabberd_mnesia:create(?MODULE, caps_features, - [{disc_only_copies, [node()]}, - {local_content, true}, - {attributes, - record_info(fields, caps_features)}]), - update_table(). + [{disc_only_copies, [node()]}, + {local_content, true}, + {attributes, record_info(fields, caps_features)}]). caps_read(_LServer, Node) -> case mnesia:dirty_read({caps_features, Node}) of @@ -68,28 +59,26 @@ import(_LServer, NodePair, Features) -> mnesia:dirty_write( #caps_features{node_pair = NodePair, features = Features}). +need_transform(#caps_features{node_pair = {N, P}, features = Fs}) -> + case is_list(N) orelse is_list(P) orelse + (is_list(Fs) andalso lists:any(fun is_list/1, Fs)) of + true -> + ?INFO_MSG("Mnesia table 'caps_features' will be " + "converted to binary", []), + true; + false -> + false + end. + +transform(#caps_features{node_pair = {N, P}, features = Fs} = R) -> + NewFs = if is_integer(Fs) -> + Fs; + true -> + [iolist_to_binary(F) || F <- Fs] + end, + R#caps_features{node_pair = {iolist_to_binary(N), iolist_to_binary(P)}, + features = NewFs}. + %%%=================================================================== %%% Internal functions %%%=================================================================== -update_table() -> - Fields = record_info(fields, caps_features), - case mnesia:table_info(caps_features, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - caps_features, Fields, set, - fun(#caps_features{node_pair = {N, _}}) -> N end, - fun(#caps_features{node_pair = {N, P}, - features = Fs} = R) -> - NewFs = if is_integer(Fs) -> - Fs; - true -> - [iolist_to_binary(F) || F <- Fs] - end, - R#caps_features{node_pair = {iolist_to_binary(N), - iolist_to_binary(P)}, - features = NewFs} - end); - _ -> - ?INFO_MSG("Recreating caps_features table", []), - mnesia:transform_table(caps_features, ignore, Fields) - end. diff --git a/src/mod_irc_mnesia.erl b/src/mod_irc_mnesia.erl index 25cdd6d4d..eb982e1fa 100644 --- a/src/mod_irc_mnesia.erl +++ b/src/mod_irc_mnesia.erl @@ -28,6 +28,7 @@ %% API -export([init/2, get_data/3, set_data/4, import/2]). +-export([need_transform/1, transform/1]). -include("jid.hrl"). -include("mod_irc.hrl"). @@ -38,9 +39,8 @@ %%%=================================================================== init(_Host, _Opts) -> ejabberd_mnesia:create(?MODULE, irc_custom, - [{disc_copies, [node()]}, - {attributes, record_info(fields, irc_custom)}]), - update_table(). + [{disc_copies, [node()]}, + {attributes, record_info(fields, irc_custom)}]). get_data(_LServer, Host, From) -> {U, S, _} = jid:tolower(From), @@ -61,25 +61,21 @@ set_data(_LServer, Host, From, Data) -> import(_LServer, #irc_custom{} = R) -> mnesia:dirty_write(R). +need_transform(#irc_custom{us_host = {{U, S}, H}}) + when is_list(U) orelse is_list(S) orelse is_list(H) -> + ?INFO_MSG("Mnesia table 'irc_custom' will be converted to binary", []), + true; +need_transform(_) -> + false. + +transform(#irc_custom{us_host = {{U, S}, H}, + data = Data} = R) -> + JID = jid:make(U, S), + R#irc_custom{us_host = {{iolist_to_binary(U), + iolist_to_binary(S)}, + iolist_to_binary(H)}, + data = mod_irc:data_to_binary(JID, Data)}. + %%%=================================================================== %%% Internal functions %%%=================================================================== -update_table() -> - Fields = record_info(fields, irc_custom), - case mnesia:table_info(irc_custom, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - irc_custom, Fields, set, - fun(#irc_custom{us_host = {_, H}}) -> H end, - fun(#irc_custom{us_host = {{U, S}, H}, - data = Data} = R) -> - JID = jid:make(U, S), - R#irc_custom{us_host = {{iolist_to_binary(U), - iolist_to_binary(S)}, - iolist_to_binary(H)}, - data = mod_irc:data_to_binary(JID, Data)} - end); - _ -> - ?INFO_MSG("Recreating irc_custom table", []), - mnesia:transform_table(irc_custom, ignore, Fields) - end. diff --git a/src/mod_last_mnesia.erl b/src/mod_last_mnesia.erl index 1ca9fb112..ab8f47478 100644 --- a/src/mod_last_mnesia.erl +++ b/src/mod_last_mnesia.erl @@ -28,6 +28,7 @@ %% API -export([init/2, import/2, get_last/2, store_last_info/4, remove_user/2]). +-export([need_transform/1, transform/1]). -include("mod_last.hrl"). -include("logger.hrl"). @@ -37,10 +38,8 @@ %%%=================================================================== init(_Host, _Opts) -> ejabberd_mnesia:create(?MODULE, last_activity, - [{disc_copies, [node()]}, - {attributes, - record_info(fields, last_activity)}]), - update_table(). + [{disc_copies, [node()]}, + {attributes, record_info(fields, last_activity)}]). get_last(LUser, LServer) -> case mnesia:dirty_read(last_activity, {LUser, LServer}) of @@ -68,22 +67,17 @@ remove_user(LUser, LServer) -> import(_LServer, #last_activity{} = LA) -> mnesia:dirty_write(LA). +need_transform(#last_activity{us = {U, S}, status = Status}) + when is_list(U) orelse is_list(S) orelse is_list(Status) -> + ?INFO_MSG("Mnesia table 'last_activity' will be converted to binary", []), + true; +need_transform(_) -> + false. + +transform(#last_activity{us = {U, S}, status = Status} = R) -> + R#last_activity{us = {iolist_to_binary(U), iolist_to_binary(S)}, + status = iolist_to_binary(Status)}. + %%%=================================================================== %%% Internal functions %%%=================================================================== -update_table() -> - Fields = record_info(fields, last_activity), - case mnesia:table_info(last_activity, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - last_activity, Fields, set, - fun(#last_activity{us = {U, _}}) -> U end, - fun(#last_activity{us = {U, S}, status = Status} = R) -> - R#last_activity{us = {iolist_to_binary(U), - iolist_to_binary(S)}, - status = iolist_to_binary(Status)} - end); - _ -> - ?INFO_MSG("Recreating last_activity table", []), - mnesia:transform_table(last_activity, ignore, Fields) - end. diff --git a/src/mod_muc_mnesia.erl b/src/mod_muc_mnesia.erl index 8cbdfe6bd..53f31cb9f 100644 --- a/src/mod_muc_mnesia.erl +++ b/src/mod_muc_mnesia.erl @@ -39,6 +39,7 @@ %% gen_server callbacks -export([start_link/2, init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2, code_change/3]). +-export([need_transform/1, transform/1]). -include("mod_muc.hrl"). -include("logger.hrl"). @@ -306,14 +307,12 @@ init([Host, Opts]) -> [{disc_copies, [node()]}, {attributes, record_info(fields, muc_registered)}, - {index, [nick]}]), - update_tables(MyHost); + {index, [nick]}]); _ -> ok end, case gen_mod:ram_db_mod(Host, Opts, mod_muc) of ?MODULE -> - update_muc_online_table(), ejabberd_mnesia:create(?MODULE, muc_online_room, [{ram_copies, [node()]}, {type, ordered_set}, @@ -377,60 +376,21 @@ clean_table_from_bad_node(Node, Host) -> end, mnesia:async_dirty(F). -update_tables(Host) -> - update_muc_room_table(Host), - update_muc_registered_table(Host). +need_transform(#muc_room{name_host = {N, H}}) + when is_list(N) orelse is_list(H) -> + ?INFO_MSG("Mnesia table 'muc_room' will be converted to binary", []), + true; +need_transform(#muc_registered{us_host = {{U, S}, H}, nick = Nick}) + when is_list(U) orelse is_list(S) orelse is_list(H) orelse is_list(Nick) -> + ?INFO_MSG("Mnesia table 'muc_registered' will be converted to binary", []), + true; +need_transform(_) -> + false. -update_muc_room_table(_Host) -> - Fields = record_info(fields, muc_room), - case mnesia:table_info(muc_room, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - muc_room, Fields, set, - fun(#muc_room{name_host = {N, _}}) -> N end, - fun(#muc_room{name_host = {N, H}, - opts = Opts} = R) -> - R#muc_room{name_host = {iolist_to_binary(N), - iolist_to_binary(H)}, - opts = mod_muc:opts_to_binary(Opts)} - end); - _ -> - ?INFO_MSG("Recreating muc_room table", []), - mnesia:transform_table(muc_room, ignore, Fields) - end. - -update_muc_registered_table(_Host) -> - Fields = record_info(fields, muc_registered), - case mnesia:table_info(muc_registered, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - muc_registered, Fields, set, - fun(#muc_registered{us_host = {_, H}}) -> H end, - fun(#muc_registered{us_host = {{U, S}, H}, - nick = Nick} = R) -> - R#muc_registered{us_host = {{iolist_to_binary(U), - iolist_to_binary(S)}, - iolist_to_binary(H)}, - nick = iolist_to_binary(Nick)} - end); - _ -> - ?INFO_MSG("Recreating muc_registered table", []), - mnesia:transform_table(muc_registered, ignore, Fields) - end. - -update_muc_online_table() -> - try - case mnesia:table_info(muc_online_room, type) of - ordered_set -> ok; - _ -> - case mnesia:delete_table(muc_online_room) of - {atomic, ok} -> ok; - Err -> erlang:error(Err) - end - end - catch _:{aborted, {no_exists, muc_online_room}} -> ok; - _:{aborted, {no_exists, muc_online_room, type}} -> ok; - E:R -> - ?ERROR_MSG("failed to update mnesia table '~s': ~p", - [muc_online_room, {E, R, erlang:get_stacktrace()}]) - end. +transform(#muc_room{name_host = {N, H}, opts = Opts} = R) -> + R#muc_room{name_host = {iolist_to_binary(N), iolist_to_binary(H)}, + opts = mod_muc:opts_to_binary(Opts)}; +transform(#muc_registered{us_host = {{U, S}, H}, nick = Nick} = R) -> + R#muc_registered{us_host = {{iolist_to_binary(U), iolist_to_binary(S)}, + iolist_to_binary(H)}, + nick = iolist_to_binary(Nick)}. diff --git a/src/mod_offline_mnesia.erl b/src/mod_offline_mnesia.erl index f04321237..d0d0de418 100644 --- a/src/mod_offline_mnesia.erl +++ b/src/mod_offline_mnesia.erl @@ -30,6 +30,7 @@ remove_old_messages/2, remove_user/2, read_message_headers/2, read_message/3, remove_message/3, read_all_messages/2, remove_all_messages/2, count_messages/2, import/1]). +-export([need_transform/1, transform/1]). -include("xmpp.hrl"). -include("mod_offline.hrl"). @@ -42,9 +43,8 @@ %%%=================================================================== init(_Host, _Opts) -> ejabberd_mnesia:create(?MODULE, offline_msg, - [{disc_only_copies, [node()]}, {type, bag}, - {attributes, record_info(fields, offline_msg)}]), - update_table(). + [{disc_only_copies, [node()]}, {type, bag}, + {attributes, record_info(fields, offline_msg)}]). store_messages(_Host, US, Msgs, Len, MaxOfflineMsgs) -> F = fun () -> @@ -183,6 +183,19 @@ count_messages(LUser, LServer) -> import(#offline_msg{} = Msg) -> mnesia:dirty_write(Msg). +need_transform(#offline_msg{us = {U, S}}) when is_list(U) orelse is_list(S) -> + ?INFO_MSG("Mnesia table 'offline_msg' will be converted to binary", []), + true; +need_transform(_) -> + false. + +transform(#offline_msg{us = {U, S}, from = From, to = To, + packet = El} = R) -> + R#offline_msg{us = {iolist_to_binary(U), iolist_to_binary(S)}, + from = jid_to_binary(From), + to = jid_to_binary(To), + packet = fxml:to_xmlel(El)}. + %%%=================================================================== %%% Internal functions %%%=================================================================== @@ -230,25 +243,3 @@ integer_to_now(Int) -> MSec = Secs div 1000000, Sec = Secs rem 1000000, {MSec, Sec, USec}. - -update_table() -> - Fields = record_info(fields, offline_msg), - case mnesia:table_info(offline_msg, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - offline_msg, Fields, bag, - fun(#offline_msg{us = {U, _}}) -> U end, - fun(#offline_msg{us = {U, S}, - from = From, - to = To, - packet = El} = R) -> - R#offline_msg{us = {iolist_to_binary(U), - iolist_to_binary(S)}, - from = jid_to_binary(From), - to = jid_to_binary(To), - packet = fxml:to_xmlel(El)} - end); - _ -> - ?INFO_MSG("Recreating offline_msg table", []), - mnesia:transform_table(offline_msg, ignore, Fields) - end. diff --git a/src/mod_privacy_mnesia.erl b/src/mod_privacy_mnesia.erl index e3adc7948..efa4ae6c8 100644 --- a/src/mod_privacy_mnesia.erl +++ b/src/mod_privacy_mnesia.erl @@ -32,6 +32,7 @@ remove_privacy_list/3, set_privacy_list/1, set_privacy_list/4, get_user_list/2, get_user_lists/2, remove_user/2, import/1]). +-export([need_transform/1, transform/1]). -include("xmpp.hrl"). -include("mod_privacy.hrl"). @@ -42,9 +43,8 @@ %%%=================================================================== init(_Host, _Opts) -> ejabberd_mnesia:create(?MODULE, privacy, - [{disc_copies, [node()]}, - {attributes, record_info(fields, privacy)}]), - update_table(). + [{disc_copies, [node()]}, + {attributes, record_info(fields, privacy)}]). process_lists_get(LUser, LServer) -> case catch mnesia:dirty_read(privacy, {LUser, LServer}) of @@ -163,25 +163,18 @@ remove_user(LUser, LServer) -> import(#privacy{} = P) -> mnesia:dirty_write(P). -%%%=================================================================== -%%% Internal functions -%%%=================================================================== -update_table() -> - Fields = record_info(fields, privacy), - case mnesia:table_info(privacy, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - privacy, Fields, set, - fun(#privacy{us = {U, _}}) -> U end, - fun(#privacy{us = {U, S}, default = Def, lists = Lists} = R) -> - NewLists = - lists:map( - fun({Name, Ls}) -> - NewLs = - lists:map( - fun(#listitem{value = Val} = L) -> - NewVal = - case Val of +need_transform(#privacy{us = {U, S}}) when is_list(U) orelse is_list(S) -> + ?INFO_MSG("Mnesia table 'privacy' will be converted to binary", []), + true; +need_transform(_) -> + false. + +transform(#privacy{us = {U, S}, default = Def, lists = Lists} = R) -> + NewLists = lists:map( + fun({Name, Ls}) -> + NewLs = lists:map( + fun(#listitem{value = Val} = L) -> + NewVal = case Val of {LU, LS, LR} -> {iolist_to_binary(LU), iolist_to_binary(LS), @@ -192,19 +185,17 @@ update_table() -> to -> to; _ -> iolist_to_binary(Val) end, - L#listitem{value = NewVal} - end, Ls), - {iolist_to_binary(Name), NewLs} - end, Lists), - NewDef = case Def of - none -> none; - _ -> iolist_to_binary(Def) - end, - NewUS = {iolist_to_binary(U), iolist_to_binary(S)}, - R#privacy{us = NewUS, default = NewDef, - lists = NewLists} - end); - _ -> - ?INFO_MSG("Recreating privacy table", []), - mnesia:transform_table(privacy, ignore, Fields) - end. + L#listitem{value = NewVal} + end, Ls), + {iolist_to_binary(Name), NewLs} + end, Lists), + NewDef = case Def of + none -> none; + _ -> iolist_to_binary(Def) + end, + NewUS = {iolist_to_binary(U), iolist_to_binary(S)}, + R#privacy{us = NewUS, default = NewDef, lists = NewLists}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== diff --git a/src/mod_private_mnesia.erl b/src/mod_private_mnesia.erl index fd588aac2..67417bb7f 100644 --- a/src/mod_private_mnesia.erl +++ b/src/mod_private_mnesia.erl @@ -29,6 +29,7 @@ %% API -export([init/2, set_data/3, get_data/3, get_all_data/2, remove_user/2, import/3]). +-export([need_transform/1, transform/1]). -include("xmpp.hrl"). -include("mod_private.hrl"). @@ -39,10 +40,8 @@ %%%=================================================================== init(_Host, _Opts) -> ejabberd_mnesia:create(?MODULE, private_storage, - [{disc_only_copies, [node()]}, - {attributes, - record_info(fields, private_storage)}]), - update_table(). + [{disc_only_copies, [node()]}, + {attributes, record_info(fields, private_storage)}]). set_data(LUser, LServer, Data) -> F = fun () -> @@ -95,23 +94,19 @@ import(LServer, <<"private_storage">>, PS = #private_storage{usns = {LUser, LServer, XMLNS}, xml = El}, mnesia:dirty_write(PS). +need_transform(#private_storage{usns = {U, S, NS}}) + when is_list(U) orelse is_list(S) orelse is_list(NS) -> + ?INFO_MSG("Mnesia table 'private_storage' will be converted to binary", []), + true; +need_transform(_) -> + false. + +transform(#private_storage{usns = {U, S, NS}, xml = El} = R) -> + R#private_storage{usns = {iolist_to_binary(U), + iolist_to_binary(S), + iolist_to_binary(NS)}, + xml = fxml:to_xmlel(El)}. + %%%=================================================================== %%% Internal functions %%%=================================================================== -update_table() -> - Fields = record_info(fields, private_storage), - case mnesia:table_info(private_storage, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - private_storage, Fields, set, - fun(#private_storage{usns = {U, _, _}}) -> U end, - fun(#private_storage{usns = {U, S, NS}, xml = El} = R) -> - R#private_storage{usns = {iolist_to_binary(U), - iolist_to_binary(S), - iolist_to_binary(NS)}, - xml = fxml:to_xmlel(El)} - end); - _ -> - ?INFO_MSG("Recreating private_storage table", []), - mnesia:transform_table(private_storage, ignore, Fields) - end. diff --git a/src/mod_roster_mnesia.erl b/src/mod_roster_mnesia.erl index e1a1fa1b5..0207b6dc5 100644 --- a/src/mod_roster_mnesia.erl +++ b/src/mod_roster_mnesia.erl @@ -32,6 +32,7 @@ roster_subscribe/4, get_roster_by_jid_with_groups/3, remove_user/2, update_roster/4, del_roster/3, transaction/2, read_subscription_and_groups/3, import/3, create_roster/1]). +-export([need_transform/1, transform/1]). -include("mod_roster.hrl"). -include("logger.hrl"). @@ -47,8 +48,7 @@ init(_Host, _Opts) -> ejabberd_mnesia:create(?MODULE, roster_version, [{disc_copies, [node()]}, {attributes, - record_info(fields, roster_version)}]), - update_tables(). + record_info(fields, roster_version)}]). read_roster_version(LUser, LServer) -> US = {LUser, LServer}, @@ -127,63 +127,40 @@ import(LServer, <<"roster_version">>, [LUser, Ver]) -> RV = #roster_version{us = {LUser, LServer}, version = Ver}, mnesia:dirty_write(RV). +need_transform(#roster{usj = {U, S, _}}) when is_list(U) orelse is_list(S) -> + ?INFO_MSG("Mnesia table 'roster' will be converted to binary", []), + true; +need_transform(#roster_version{us = {U, S}, version = Ver}) + when is_list(U) orelse is_list(S) orelse is_list(Ver) -> + ?INFO_MSG("Mnesia table 'roster_version' will be converted to binary", []), + true; +need_transform(_) -> + false. + +transform(#roster{usj = {U, S, {LU, LS, LR}}, + us = {U1, S1}, + jid = {U2, S2, R2}, + name = Name, + groups = Gs, + askmessage = Ask, + xs = Xs} = R) -> + R#roster{usj = {iolist_to_binary(U), iolist_to_binary(S), + {iolist_to_binary(LU), + iolist_to_binary(LS), + iolist_to_binary(LR)}}, + us = {iolist_to_binary(U1), iolist_to_binary(S1)}, + jid = {iolist_to_binary(U2), + iolist_to_binary(S2), + iolist_to_binary(R2)}, + name = iolist_to_binary(Name), + groups = [iolist_to_binary(G) || G <- Gs], + askmessage = try iolist_to_binary(Ask) + catch _:_ -> <<"">> end, + xs = [fxml:to_xmlel(X) || X <- Xs]}; +transform(#roster_version{us = {U, S}, version = Ver} = R) -> + R#roster_version{us = {iolist_to_binary(U), iolist_to_binary(S)}, + version = iolist_to_binary(Ver)}. + %%%=================================================================== %%% Internal functions %%%=================================================================== -update_tables() -> - update_roster_table(), - update_roster_version_table(). - -update_roster_table() -> - Fields = record_info(fields, roster), - case mnesia:table_info(roster, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - roster, Fields, set, - fun(#roster{usj = {U, _, _}}) -> U end, - fun(#roster{usj = {U, S, {LU, LS, LR}}, - us = {U1, S1}, - jid = {U2, S2, R2}, - name = Name, - groups = Gs, - askmessage = Ask, - xs = Xs} = R) -> - R#roster{usj = {iolist_to_binary(U), - iolist_to_binary(S), - {iolist_to_binary(LU), - iolist_to_binary(LS), - iolist_to_binary(LR)}}, - us = {iolist_to_binary(U1), - iolist_to_binary(S1)}, - jid = {iolist_to_binary(U2), - iolist_to_binary(S2), - iolist_to_binary(R2)}, - name = iolist_to_binary(Name), - groups = [iolist_to_binary(G) || G <- Gs], - askmessage = try iolist_to_binary(Ask) - catch _:_ -> <<"">> end, - xs = [fxml:to_xmlel(X) || X <- Xs]} - end); - _ -> - ?INFO_MSG("Recreating roster table", []), - mnesia:transform_table(roster, ignore, Fields) - end. - -%% Convert roster table to support virtual host -%% Convert roster table: xattrs fields become -update_roster_version_table() -> - Fields = record_info(fields, roster_version), - case mnesia:table_info(roster_version, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - roster_version, Fields, set, - fun(#roster_version{us = {U, _}}) -> U end, - fun(#roster_version{us = {U, S}, version = Ver} = R) -> - R#roster_version{us = {iolist_to_binary(U), - iolist_to_binary(S)}, - version = iolist_to_binary(Ver)} - end); - _ -> - ?INFO_MSG("Recreating roster_version table", []), - mnesia:transform_table(roster_version, ignore, Fields) - end. diff --git a/src/mod_shared_roster_mnesia.erl b/src/mod_shared_roster_mnesia.erl index 1e3012794..adfbac680 100644 --- a/src/mod_shared_roster_mnesia.erl +++ b/src/mod_shared_roster_mnesia.erl @@ -32,6 +32,7 @@ get_user_groups/2, get_group_explicit_users/2, get_user_displayed_groups/3, is_user_in_group/3, add_user_to_group/3, remove_user_from_group/3, import/3]). +-export([need_transform/1, transform/1]). -include("mod_roster.hrl"). -include("mod_shared_roster.hrl"). @@ -43,13 +44,12 @@ %%%=================================================================== init(_Host, _Opts) -> ejabberd_mnesia:create(?MODULE, sr_group, - [{disc_copies, [node()]}, - {attributes, record_info(fields, sr_group)}]), + [{disc_copies, [node()]}, + {attributes, record_info(fields, sr_group)}]), ejabberd_mnesia:create(?MODULE, sr_user, - [{disc_copies, [node()]}, {type, bag}, - {attributes, record_info(fields, sr_user)}, - {index, [group_host]}]), - update_tables(). + [{disc_copies, [node()]}, {type, bag}, + {attributes, record_info(fields, sr_user)}, + {index, [group_host]}]). list_groups(Host) -> mnesia:dirty_select(sr_group, @@ -144,44 +144,24 @@ import(LServer, <<"sr_user">>, [SJID, Group, _TimeStamp]) -> User = #sr_user{us = {U, S}, group_host = {Group, LServer}}, mnesia:dirty_write(User). +need_transform(#sr_group{group_host = {G, H}}) + when is_list(G) orelse is_list(H) -> + ?INFO_MSG("Mnesia table 'sr_group' will be converted to binary", []), + true; +need_transform(#sr_user{us = {U, S}, group_host = {G, H}}) + when is_list(U) orelse is_list(S) orelse is_list(G) orelse is_list(H) -> + ?INFO_MSG("Mnesia table 'sr_user' will be converted to binary", []), + true; +need_transform(_) -> + false. + +transform(#sr_group{group_host = {G, H}, opts = Opts} = R) -> + R#sr_group{group_host = {iolist_to_binary(G), iolist_to_binary(H)}, + opts = mod_shared_roster:opts_to_binary(Opts)}; +transform(#sr_user{us = {U, S}, group_host = {G, H}} = R) -> + R#sr_user{us = {iolist_to_binary(U), iolist_to_binary(S)}, + group_host = {iolist_to_binary(G), iolist_to_binary(H)}}. + %%%=================================================================== %%% Internal functions %%%=================================================================== -update_tables() -> - update_sr_group_table(), - update_sr_user_table(). - -update_sr_group_table() -> - Fields = record_info(fields, sr_group), - case mnesia:table_info(sr_group, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - sr_group, Fields, set, - fun(#sr_group{group_host = {G, _}}) -> G end, - fun(#sr_group{group_host = {G, H}, - opts = Opts} = R) -> - R#sr_group{group_host = {iolist_to_binary(G), - iolist_to_binary(H)}, - opts = mod_shared_roster:opts_to_binary(Opts)} - end); - _ -> - ?INFO_MSG("Recreating sr_group table", []), - mnesia:transform_table(sr_group, ignore, Fields) - end. - -update_sr_user_table() -> - Fields = record_info(fields, sr_user), - case mnesia:table_info(sr_user, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - sr_user, Fields, bag, - fun(#sr_user{us = {U, _}}) -> U end, - fun(#sr_user{us = {U, S}, group_host = {G, H}} = R) -> - R#sr_user{us = {iolist_to_binary(U), iolist_to_binary(S)}, - group_host = {iolist_to_binary(G), - iolist_to_binary(H)}} - end); - _ -> - ?INFO_MSG("Recreating sr_user table", []), - mnesia:transform_table(sr_user, ignore, Fields) - end. diff --git a/src/mod_vcard_mnesia.erl b/src/mod_vcard_mnesia.erl index 5faff5261..340f8928e 100644 --- a/src/mod_vcard_mnesia.erl +++ b/src/mod_vcard_mnesia.erl @@ -30,6 +30,7 @@ -export([init/2, stop/1, import/3, get_vcard/2, set_vcard/4, search/4, search_fields/1, search_reported/1, remove_user/2]). -export([is_search_supported/1]). +-export([need_transform/1, transform/1]). -include("ejabberd.hrl"). -include("xmpp.hrl"). @@ -51,8 +52,7 @@ init(_Host, _Opts) -> lgiven, lmiddle, lnickname, lbday, lctry, llocality, lemail, lorgname, lorgunit - ]}]), - update_tables(). + ]}]). stop(_Host) -> ok. @@ -158,53 +158,31 @@ import(LServer, <<"vcard_search">>, orgname = OrgName, lorgname = LOrgName, orgunit = OrgUnit, lorgunit = LOrgUnit}). +need_transform(#vcard{us = {U, S}}) when is_list(U) orelse is_list(S) -> + ?INFO_MSG("Mnesia table 'vcard' will be converted to binary", []), + true; +need_transform(#vcard_search{us = {U, S}}) when is_list(U) orelse is_list(S) -> + ?INFO_MSG("Mnesia table 'vcard_search' will be converted to binary", []), + true; +need_transform(_) -> + false. + +transform(#vcard{us = {U, S}, vcard = El} = R) -> + R#vcard{us = {iolist_to_binary(U), iolist_to_binary(S)}, + vcard = fxml:to_xmlel(El)}; +transform(#vcard_search{} = VS) -> + [vcard_search | L] = tuple_to_list(VS), + NewL = lists:map( + fun({U, S}) -> + {iolist_to_binary(U), iolist_to_binary(S)}; + (Str) -> + iolist_to_binary(Str) + end, L), + list_to_tuple([vcard_search | NewL]). + %%%=================================================================== %%% Internal functions %%%=================================================================== -update_tables() -> - update_vcard_table(), - update_vcard_search_table(). - -update_vcard_table() -> - Fields = record_info(fields, vcard), - case mnesia:table_info(vcard, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - vcard, Fields, set, - fun(#vcard{us = {U, _}}) -> U end, - fun(#vcard{us = {U, S}, vcard = El} = R) -> - R#vcard{us = {iolist_to_binary(U), - iolist_to_binary(S)}, - vcard = fxml:to_xmlel(El)} - end); - _ -> - ?INFO_MSG("Recreating vcard table", []), - mnesia:transform_table(vcard, ignore, Fields) - end. - -update_vcard_search_table() -> - Fields = record_info(fields, vcard_search), - case mnesia:table_info(vcard_search, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - vcard_search, Fields, set, - fun(#vcard_search{us = {U, _}}) -> U end, - fun(#vcard_search{} = VS) -> - [vcard_search | L] = tuple_to_list(VS), - NewL = lists:map( - fun({U, S}) -> - {iolist_to_binary(U), - iolist_to_binary(S)}; - (Str) -> - iolist_to_binary(Str) - end, L), - list_to_tuple([vcard_search | NewL]) - end); - _ -> - ?INFO_MSG("Recreating vcard_search table", []), - mnesia:transform_table(vcard_search, ignore, Fields) - end. - make_matchspec(LServer, Data) -> GlobMatch = #vcard_search{_ = '_'}, Match = filter_fields(Data, GlobMatch, LServer), diff --git a/src/mod_vcard_xupdate_mnesia.erl b/src/mod_vcard_xupdate_mnesia.erl index 879f5b115..9b80e0672 100644 --- a/src/mod_vcard_xupdate_mnesia.erl +++ b/src/mod_vcard_xupdate_mnesia.erl @@ -28,6 +28,7 @@ %% API -export([init/2, import/3, add_xupdate/3, get_xupdate/2, remove_xupdate/2]). +-export([need_transform/1, transform/1]). -include("mod_vcard_xupdate.hrl"). -include("logger.hrl"). @@ -39,8 +40,7 @@ init(_Host, _Opts) -> ejabberd_mnesia:create(?MODULE, vcard_xupdate, [{disc_copies, [node()]}, {attributes, - record_info(fields, vcard_xupdate)}]), - update_table(). + record_info(fields, vcard_xupdate)}]). add_xupdate(LUser, LServer, Hash) -> F = fun () -> @@ -66,22 +66,17 @@ import(LServer, <<"vcard_xupdate">>, [LUser, Hash, _TimeStamp]) -> mnesia:dirty_write( #vcard_xupdate{us = {LUser, LServer}, hash = Hash}). +need_transform(#vcard_xupdate{us = {U, S}, hash = Hash}) + when is_list(U) orelse is_list(S) orelse is_list(Hash) -> + ?INFO_MSG("Mnesia table 'vcard_xupdate' will be converted to binary", []), + true; +need_transform(_) -> + false. + +transform(#vcard_xupdate{us = {U, S}, hash = Hash} = R) -> + R#vcard_xupdate{us = {iolist_to_binary(U), iolist_to_binary(S)}, + hash = iolist_to_binary(Hash)}. + %%%=================================================================== %%% Internal functions %%%=================================================================== -update_table() -> - Fields = record_info(fields, vcard_xupdate), - case mnesia:table_info(vcard_xupdate, attributes) of - Fields -> - ejabberd_config:convert_table_to_binary( - vcard_xupdate, Fields, set, - fun(#vcard_xupdate{us = {U, _}}) -> U end, - fun(#vcard_xupdate{us = {U, S}, hash = Hash} = R) -> - R#vcard_xupdate{us = {iolist_to_binary(U), - iolist_to_binary(S)}, - hash = iolist_to_binary(Hash)} - end); - _ -> - ?INFO_MSG("Recreating vcard_xupdate table", []), - mnesia:transform_table(vcard_xupdate, ignore, Fields) - end. diff --git a/src/pubsub_migrate.erl b/src/pubsub_migrate.erl index 17bdb2368..f0b324461 100644 --- a/src/pubsub_migrate.erl +++ b/src/pubsub_migrate.erl @@ -430,7 +430,7 @@ convert_list_lasts() -> convert_list_records(Tab, Fields, DetectFun, ConvertFun) -> case mnesia:table_info(Tab, attributes) of Fields -> - ejabberd_config:convert_table_to_binary( + convert_table_to_binary( Tab, Fields, set, DetectFun, ConvertFun); _ -> ?INFO_MSG("Recreating ~p table", [Tab]), @@ -445,3 +445,89 @@ binusr({U,S,R}) -> {bin(U), bin(S), bin(R)}. bin(L) -> iolist_to_binary(L). +%% The code should be updated to support new ejabberd_mnesia +%% transform functions (i.e. need_transform/1 and transform/1) +convert_table_to_binary(Tab, Fields, Type, DetectFun, ConvertFun) -> + case is_table_still_list(Tab, DetectFun) of + true -> + ?INFO_MSG("Converting '~s' table from strings to binaries.", [Tab]), + TmpTab = list_to_atom(atom_to_list(Tab) ++ "_tmp_table"), + catch mnesia:delete_table(TmpTab), + case ejabberd_mnesia:create(?MODULE, TmpTab, + [{disc_only_copies, [node()]}, + {type, Type}, + {local_content, true}, + {record_name, Tab}, + {attributes, Fields}]) of + {atomic, ok} -> + mnesia:transform_table(Tab, ignore, Fields), + case mnesia:transaction( + fun() -> + mnesia:write_lock_table(TmpTab), + mnesia:foldl( + fun(R, _) -> + NewR = ConvertFun(R), + mnesia:dirty_write(TmpTab, NewR) + end, ok, Tab) + end) of + {atomic, ok} -> + mnesia:clear_table(Tab), + case mnesia:transaction( + fun() -> + mnesia:write_lock_table(Tab), + mnesia:foldl( + fun(R, _) -> + mnesia:dirty_write(R) + end, ok, TmpTab) + end) of + {atomic, ok} -> + mnesia:delete_table(TmpTab); + Err -> + report_and_stop(Tab, Err) + end; + Err -> + report_and_stop(Tab, Err) + end; + Err -> + report_and_stop(Tab, Err) + end; + false -> + ok + end. + +is_table_still_list(Tab, DetectFun) -> + is_table_still_list(Tab, DetectFun, mnesia:dirty_first(Tab)). + +is_table_still_list(_Tab, _DetectFun, '$end_of_table') -> + false; +is_table_still_list(Tab, DetectFun, Key) -> + Rs = mnesia:dirty_read(Tab, Key), + Res = lists:foldl(fun(_, true) -> + true; + (_, false) -> + false; + (R, _) -> + case DetectFun(R) of + '$next' -> + '$next'; + El -> + is_list(El) + end + end, '$next', Rs), + case Res of + true -> + true; + false -> + false; + '$next' -> + is_table_still_list(Tab, DetectFun, mnesia:dirty_next(Tab, Key)) + end. + +report_and_stop(Tab, Err) -> + ErrTxt = lists:flatten( + io_lib:format( + "Failed to convert '~s' table to binary: ~p", + [Tab, Err])), + ?CRITICAL_MSG(ErrTxt, []), + timer:sleep(1000), + halt(string:substr(ErrTxt, 1, 199)).