Merge pull request #3982 from nosnilmot/sql-update-tests

SQL schema migration fixes and testing

.github/workflows/ci.yml

@@ -58,23 +58,34 @@ jobs:
         wget https://github.com/processone/ejabberd/raw/21.12/rebar3
         chmod +x rebar3
 
+    - name: Install MS SQL Server
+      run: |
+        docker run -d -e "ACCEPT_EULA=Y" -e "SA_PASSWORD=ejabberd_Test1" \
+                   -v $(pwd)/test/docker/db/mssql/initdb/initdb_mssql.sql:/initdb_mssql.sql:ro \
+                   -v $(pwd)/sql/mssql.sql:/mssql.sql:ro \
+                   -v $(pwd)/sql/mssql.new.sql:/mssql.new.sql:ro \
+                   -p 1433:1433 --name ejabberd-mssql "mcr.microsoft.com/mssql/server:2019-latest"
+        sleep 10
+
     - name: Prepare databases
       run: |
+        docker exec ejabberd-mssql /opt/mssql-tools/bin/sqlcmd -U SA -P ejabberd_Test1 -S localhost -i /initdb_mssql.sql
+        docker exec ejabberd-mssql /opt/mssql-tools/bin/sqlcmd -U SA -P ejabberd_Test1 -S localhost -d ejabberd_test -i /mssql.sql
         sudo systemctl start mysql.service
         sudo systemctl start postgresql.service
+        mysql -u root -proot -e "CREATE DATABASE ejabberd_test;"
         mysql -u root -proot -e "CREATE USER 'ejabberd_test'@'localhost'
                                  IDENTIFIED BY 'ejabberd_test';"
-        mysql -u root -proot -e "CREATE DATABASE ejabberd_test;"
         mysql -u root -proot -e "GRANT ALL ON ejabberd_test.*
                                  TO 'ejabberd_test'@'localhost';"
-        mysql -u root -proot ejabberd_test < sql/mysql.sql
+        mysql -u ejabberd_test -pejabberd_test ejabberd_test < sql/mysql.sql
         pg_isready
+        sudo -u postgres psql -c "CREATE DATABASE ejabberd_test;"
         sudo -u postgres psql -c "CREATE USER ejabberd_test
                                   WITH PASSWORD 'ejabberd_test';"
-        sudo -u postgres psql -c "CREATE DATABASE ejabberd_test;"
-        sudo -u postgres psql ejabberd_test -f sql/pg.sql
         sudo -u postgres psql -c "GRANT ALL PRIVILEGES
                                   ON DATABASE ejabberd_test TO ejabberd_test;"
+        PGPASSWORD="ejabberd_test" psql -h localhost -U ejabberd_test ejabberd_test -f sql/pg.sql
         sudo -u postgres psql ejabberd_test -c "GRANT ALL PRIVILEGES ON ALL
                                                 TABLES IN SCHEMA public
                                                 TO ejabberd_test;"
@@ -221,20 +232,60 @@ jobs:
         CTRUN=`ls -la _build/test/logs/last | sed 's|.*-> ||'`
         echo "::notice::View CT results: https://processone.github.io/ecil/logs/$CTRUN/"
 
+    - name: Check for changes to trigger schema upgrade test
+      uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          sql:
+            - 'sql/**'
+            - 'src/mod_admin_update_sql.erl'
+
+    - name: Prepare for schema upgrade test
+      id: prepupgradetest
+      if: ${{ steps.filter.outputs.sql == 'true' }}
+      run: |
+        [[ -d logs ]] && rm -rf logs
+        [[ -d _build/test/logs ]] && rm -rf _build/test/logs || true
+        sed -i 's|update_sql, false|update_sql, true|g' test/suite.erl
+    - name: Run DB tests on upgraded schema (mssql, mysql, pgsql)
+      run: CT_BACKENDS=mssql,mysql,pgsql make test
+      if: always() && steps.prepupgradetest.outcome != 'skipped'
+      id: ctupgradedschema
+    - name: Check results
+      if: always() && steps.ctupgradedschema.outcome != 'skipped'
+      run: |
+        [[ -d _build ]] && ln -s _build/test/logs/last/ logs || true
+        ln `find logs/ -name suite.log` logs/suite.log
+        grep 'TEST COMPLETE' logs/suite.log
+        grep -q 'TEST COMPLETE,.* 0 failed' logs/suite.log
+        test $(find logs/ -empty -name error.log)
+    - name: View logs failures
+      if: failure() && steps.ctupgradedschema.outcome != 'skipped'
+      run: |
+        cat logs/suite.log | awk \
+          'BEGIN{RS="\n=case";FS="\n"} /=result\s*failed/ {print "=case" $0}'
+        find logs/ -name error.log -exec cat '{}' ';'
+        find logs/ -name exunit.log -exec cat '{}' ';'
+
     - name: Prepare new schema
       run: |
         [[ -d logs ]] && rm -rf logs
         [[ -d _build/test/logs ]] && rm -rf _build/test/logs || true
+        docker exec ejabberd-mssql /opt/mssql-tools/bin/sqlcmd -U SA -P ejabberd_Test1 -S localhost -Q "drop database [ejabberd_test];"
+        docker exec ejabberd-mssql /opt/mssql-tools/bin/sqlcmd -U SA -P ejabberd_Test1 -S localhost -Q "drop login [ejabberd_test];"
         mysql -u root -proot -e "DROP DATABASE ejabberd_test;"
         sudo -u postgres psql -c "DROP DATABASE ejabberd_test;"
+        docker exec ejabberd-mssql /opt/mssql-tools/bin/sqlcmd -U SA -P ejabberd_Test1 -S localhost -i /initdb_mssql.sql
+        docker exec ejabberd-mssql /opt/mssql-tools/bin/sqlcmd -U SA -P ejabberd_Test1 -S localhost -d ejabberd_test -i /mssql.new.sql
         mysql -u root -proot -e "CREATE DATABASE ejabberd_test;"
         mysql -u root -proot -e "GRANT ALL ON ejabberd_test.*
                                  TO 'ejabberd_test'@'localhost';"
-        mysql -u root -proot ejabberd_test < sql/mysql.new.sql
+        mysql -u ejabberd_test -pejabberd_test ejabberd_test < sql/mysql.new.sql
         sudo -u postgres psql -c "CREATE DATABASE ejabberd_test;"
-        sudo -u postgres psql ejabberd_test -f sql/pg.new.sql
         sudo -u postgres psql -c "GRANT ALL PRIVILEGES
                                   ON DATABASE ejabberd_test TO ejabberd_test;"
+        PGPASSWORD="ejabberd_test" psql -h localhost -U ejabberd_test ejabberd_test -f sql/pg.new.sql
         sudo -u postgres psql ejabberd_test -c "GRANT ALL PRIVILEGES ON ALL
                                                 TABLES IN SCHEMA public
                                                 TO ejabberd_test;"
@@ -242,7 +293,8 @@ jobs:
                                                 SEQUENCES IN SCHEMA public
                                                 TO ejabberd_test;"
         sed -i 's|new_schema, false|new_schema, true|g' test/suite.erl
-    - run: CT_BACKENDS=mysql,pgsql make test
+    - name: Run DB tests on new schema (mssql, mysql, pgsql)
+      run: CT_BACKENDS=mssql,mysql,pgsql make test
       id: ctnewschema
     - name: Check results
       if: always() && steps.ctnewschema.outcome != 'skipped'

sql/pg.new.sql

@@ -92,7 +92,7 @@
 -- DROP INDEX i_vcard_search_lemail;
 -- DROP INDEX i_vcard_search_lorgname;
 -- DROP INDEX i_vcard_search_lorgunit;
--- ALTER TABLE vcard_search ADD PRIMARY KEY (server_host, username);
+-- ALTER TABLE vcard_search ADD PRIMARY KEY (server_host, lusername);
 -- CREATE INDEX i_vcard_search_sh_lfn       ON vcard_search(server_host, lfn);
 -- CREATE INDEX i_vcard_search_sh_lfamily   ON vcard_search(server_host, lfamily);
 -- CREATE INDEX i_vcard_search_sh_lgiven    ON vcard_search(server_host, lgiven);

src/ejabberd_config.erl

@@ -37,13 +37,13 @@
 -export([beams/1, validators/1, globals/0, may_hide_data/1]).
 -export([dump/0, dump/1, convert_to_yaml/1, convert_to_yaml/2]).
 -export([callback_modules/1]).
+-export([set_option/2]).
 
 %% Deprecated functions
--export([get_option/2, set_option/2]).
+-export([get_option/2]).
 -export([get_version/0, get_myhosts/0]).
 -export([get_mylang/0, get_lang/1]).
 -deprecated([{get_option, 2},
-	     {set_option, 2},
 	     {get_version, 0},
 	     {get_myhosts, 0},
 	     {get_mylang, 0},

src/mod_admin_update_sql.erl

@@ -34,6 +34,8 @@
 % Commands API
 -export([update_sql/0]).
 
+% For testing
+-export([update_sql/1]).
 
 -include("logger.hrl").
 -include("ejabberd_commands.hrl").
@@ -112,182 +114,313 @@ update_sql(Host) ->
             State = #state{host = LHost,
                            dbtype = DBType,
                            escape = Escape},
-            update_tables(State)
+            update_tables(State),
+            check_config()
+    end.
+
+check_config() ->
+    case ejabberd_sql:use_new_schema() of
+        true -> ok;
+        false ->
+            ejabberd_config:set_option(new_sql_schema, true),
+            io:format('~nNOTE: you must add "new_sql_schema: true" to ejabberd.yml before next restart~n~n', [])
     end.
 
 update_tables(State) ->
-    add_sh_column(State, "users"),
+    case add_sh_column(State, "users") of
-    drop_pkey(State, "users"),
+        true ->
-    add_pkey(State, "users", ["server_host", "username"]),
+            drop_pkey(State, "users"),
-    drop_sh_default(State, "users"),
+            add_pkey(State, "users", ["server_host", "username"]),
+            drop_sh_default(State, "users");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "last"),
+    case add_sh_column(State, "last") of
-    drop_pkey(State, "last"),
+        true ->
-    add_pkey(State, "last", ["server_host", "username"]),
+            drop_pkey(State, "last"),
-    drop_sh_default(State, "last"),
+            add_pkey(State, "last", ["server_host", "username"]),
+            drop_sh_default(State, "last");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "rosterusers"),
+    case add_sh_column(State, "rosterusers") of
-    drop_index(State, "rosterusers", "i_rosteru_user_jid"),
+        true ->
-    drop_index(State, "rosterusers", "i_rosteru_username"),
+            drop_index(State, "rosterusers", "i_rosteru_user_jid"),
-    drop_index(State, "rosterusers", "i_rosteru_jid"),
+            drop_index(State, "rosterusers", "i_rosteru_username"),
-    create_unique_index(State, "rosterusers", "i_rosteru_sh_user_jid", ["server_host", "username", "jid"]),
+            drop_index(State, "rosterusers", "i_rosteru_jid"),
-    create_index(State, "rosterusers", "i_rosteru_sh_jid", ["server_host", "jid"]),
+            create_unique_index(State, "rosterusers", "i_rosteru_sh_user_jid", ["server_host", "username", "jid"]),
-    drop_sh_default(State, "rosterusers"),
+            create_index(State, "rosterusers", "i_rosteru_sh_jid", ["server_host", "jid"]),
+            drop_sh_default(State, "rosterusers");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "rostergroups"),
+    case add_sh_column(State, "rostergroups") of
-    drop_index(State, "rostergroups", "pk_rosterg_user_jid"),
+        true ->
-    create_index(State, "rostergroups", "i_rosterg_sh_user_jid", ["server_host", "username", "jid"]),
+            drop_index(State, "rostergroups", "pk_rosterg_user_jid"),
-    drop_sh_default(State, "rostergroups"),
+            create_index(State, "rostergroups", "i_rosterg_sh_user_jid", ["server_host", "username", "jid"]),
+            drop_sh_default(State, "rostergroups");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "sr_group"),
+    case add_sh_column(State, "sr_group") of
-    drop_index(State, "sr_group", "i_sr_group_name"),
+        true ->
-    create_unique_index(State, "sr_group", "i_sr_group_sh_name", ["server_host", "name"]),
+            drop_index(State, "sr_group", "i_sr_group_name"),
-    drop_sh_default(State, "sr_group"),
+            create_unique_index(State, "sr_group", "i_sr_group_sh_name", ["server_host", "name"]),
+            drop_sh_default(State, "sr_group");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "sr_user"),
+    case add_sh_column(State, "sr_user") of
-    drop_index(State, "sr_user", "i_sr_user_jid_grp"),
+        true ->
-    drop_index(State, "sr_user", "i_sr_user_jid"),
+            drop_index(State, "sr_user", "i_sr_user_jid_grp"),
-    drop_index(State, "sr_user", "i_sr_user_grp"),
+            drop_index(State, "sr_user", "i_sr_user_jid"),
-    create_unique_index(State, "sr_user", "i_sr_user_sh_jid_grp", ["server_host", "jid", "grp"]),
+            drop_index(State, "sr_user", "i_sr_user_grp"),
-    create_index(State, "sr_user", "i_sr_user_sh_grp", ["server_host", "grp"]),
+            create_unique_index(State, "sr_user", "i_sr_user_sh_jid_grp", ["server_host", "jid", "grp"]),
-    drop_sh_default(State, "sr_user"),
+            create_index(State, "sr_user", "i_sr_user_sh_grp", ["server_host", "grp"]),
+            drop_sh_default(State, "sr_user");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "spool"),
+    case add_sh_column(State, "spool") of
-    drop_index(State, "spool", "i_despool"),
+        true ->
-    create_index(State, "spool", "i_spool_sh_username", ["server_host", "username"]),
+            drop_index(State, "spool", "i_despool"),
-    drop_sh_default(State, "spool"),
+            create_index(State, "spool", "i_spool_sh_username", ["server_host", "username"]),
+            drop_sh_default(State, "spool");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "archive"),
+    case add_sh_column(State, "archive") of
-    drop_index(State, "archive", "i_username"),
+        true ->
-    drop_index(State, "archive", "i_username_timestamp"),
+            drop_index(State, "archive", "i_username"),
-    drop_index(State, "archive", "i_timestamp"),
+            drop_index(State, "archive", "i_username_timestamp"),
-    drop_index(State, "archive", "i_peer"),
+            drop_index(State, "archive", "i_timestamp"),
-    drop_index(State, "archive", "i_bare_peer"),
+            drop_index(State, "archive", "i_peer"),
-    drop_index(State, "archive", "i_username_peer"),
+            drop_index(State, "archive", "i_bare_peer"),
-    drop_index(State, "archive", "i_username_bare_peer"),
+            drop_index(State, "archive", "i_username_peer"),
-    create_index(State, "archive", "i_archive_sh_username_timestamp", ["server_host", "username", "timestamp"]),
+            drop_index(State, "archive", "i_username_bare_peer"),
-    create_index(State, "archive", "i_archive_sh_timestamp", ["server_host", "timestamp"]),
+            create_index(State, "archive", "i_archive_sh_username_timestamp", ["server_host", "username", "timestamp"]),
-    create_index(State, "archive", "i_archive_sh_username_peer", ["server_host", "username", "peer"]),
+            create_index(State, "archive", "i_archive_sh_timestamp", ["server_host", "timestamp"]),
-    create_index(State, "archive", "i_archive_sh_username_bare_peer", ["server_host", "username", "bare_peer"]),
+            create_index(State, "archive", "i_archive_sh_username_peer", ["server_host", "username", "peer"]),
-    drop_sh_default(State, "archive"),
+            create_index(State, "archive", "i_archive_sh_username_bare_peer", ["server_host", "username", "bare_peer"]),
+            drop_sh_default(State, "archive");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "archive_prefs"),
+    case add_sh_column(State, "archive_prefs") of
-    drop_pkey(State, "archive_prefs"),
+        true ->
-    add_pkey(State, "archive_prefs", ["server_host", "username"]),
+            drop_pkey(State, "archive_prefs"),
-    drop_sh_default(State, "archive_prefs"),
+            add_pkey(State, "archive_prefs", ["server_host", "username"]),
+            drop_sh_default(State, "archive_prefs");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "vcard"),
+    case add_sh_column(State, "vcard") of
-    drop_pkey(State, "vcard"),
+        true ->
-    add_pkey(State, "vcard", ["server_host", "username"]),
+            drop_pkey(State, "vcard"),
-    drop_sh_default(State, "vcard"),
+            add_pkey(State, "vcard", ["server_host", "username"]),
+            drop_sh_default(State, "vcard");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "vcard_search"),
+    case add_sh_column(State, "vcard_search") of
-    drop_pkey(State, "vcard_search"),
+        true ->
-    drop_index(State, "vcard_search", "i_vcard_search_lfn"),
+            drop_pkey(State, "vcard_search"),
-    drop_index(State, "vcard_search", "i_vcard_search_lfamily"),
+            drop_index(State, "vcard_search", "i_vcard_search_lfn"),
-    drop_index(State, "vcard_search", "i_vcard_search_lgiven"),
+            drop_index(State, "vcard_search", "i_vcard_search_lfamily"),
-    drop_index(State, "vcard_search", "i_vcard_search_lmiddle"),
+            drop_index(State, "vcard_search", "i_vcard_search_lgiven"),
-    drop_index(State, "vcard_search", "i_vcard_search_lnickname"),
+            drop_index(State, "vcard_search", "i_vcard_search_lmiddle"),
-    drop_index(State, "vcard_search", "i_vcard_search_lbday"),
+            drop_index(State, "vcard_search", "i_vcard_search_lnickname"),
-    drop_index(State, "vcard_search", "i_vcard_search_lctry"),
+            drop_index(State, "vcard_search", "i_vcard_search_lbday"),
-    drop_index(State, "vcard_search", "i_vcard_search_llocality"),
+            drop_index(State, "vcard_search", "i_vcard_search_lctry"),
-    drop_index(State, "vcard_search", "i_vcard_search_lemail"),
+            drop_index(State, "vcard_search", "i_vcard_search_llocality"),
-    drop_index(State, "vcard_search", "i_vcard_search_lorgname"),
+            drop_index(State, "vcard_search", "i_vcard_search_lemail"),
-    drop_index(State, "vcard_search", "i_vcard_search_lorgunit"),
+            drop_index(State, "vcard_search", "i_vcard_search_lorgname"),
-    add_pkey(State, "vcard_search", ["server_host", "username"]),
+            drop_index(State, "vcard_search", "i_vcard_search_lorgunit"),
-    create_index(State, "vcard_search", "i_vcard_search_sh_lfn",       ["server_host", "lfn"]),
+            add_pkey(State, "vcard_search", ["server_host", "lusername"]),
-    create_index(State, "vcard_search", "i_vcard_search_sh_lfamily",   ["server_host", "lfamily"]),
+            create_index(State, "vcard_search", "i_vcard_search_sh_lfn",       ["server_host", "lfn"]),
-    create_index(State, "vcard_search", "i_vcard_search_sh_lgiven",    ["server_host", "lgiven"]),
+            create_index(State, "vcard_search", "i_vcard_search_sh_lfamily",   ["server_host", "lfamily"]),
-    create_index(State, "vcard_search", "i_vcard_search_sh_lmiddle",   ["server_host", "lmiddle"]),
+            create_index(State, "vcard_search", "i_vcard_search_sh_lgiven",    ["server_host", "lgiven"]),
-    create_index(State, "vcard_search", "i_vcard_search_sh_lnickname", ["server_host", "lnickname"]),
+            create_index(State, "vcard_search", "i_vcard_search_sh_lmiddle",   ["server_host", "lmiddle"]),
-    create_index(State, "vcard_search", "i_vcard_search_sh_lbday",     ["server_host", "lbday"]),
+            create_index(State, "vcard_search", "i_vcard_search_sh_lnickname", ["server_host", "lnickname"]),
-    create_index(State, "vcard_search", "i_vcard_search_sh_lctry",     ["server_host", "lctry"]),
+            create_index(State, "vcard_search", "i_vcard_search_sh_lbday",     ["server_host", "lbday"]),
-    create_index(State, "vcard_search", "i_vcard_search_sh_llocality", ["server_host", "llocality"]),
+            create_index(State, "vcard_search", "i_vcard_search_sh_lctry",     ["server_host", "lctry"]),
-    create_index(State, "vcard_search", "i_vcard_search_sh_lemail",    ["server_host", "lemail"]),
+            create_index(State, "vcard_search", "i_vcard_search_sh_llocality", ["server_host", "llocality"]),
-    create_index(State, "vcard_search", "i_vcard_search_sh_lorgname",  ["server_host", "lorgname"]),
+            create_index(State, "vcard_search", "i_vcard_search_sh_lemail",    ["server_host", "lemail"]),
-    create_index(State, "vcard_search", "i_vcard_search_sh_lorgunit",  ["server_host", "lorgunit"]),
+            create_index(State, "vcard_search", "i_vcard_search_sh_lorgname",  ["server_host", "lorgname"]),
-    drop_sh_default(State, "vcard_search"),
+            create_index(State, "vcard_search", "i_vcard_search_sh_lorgunit",  ["server_host", "lorgunit"]),
+            drop_sh_default(State, "vcard_search");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "privacy_default_list"),
+    case add_sh_column(State, "privacy_default_list") of
-    drop_pkey(State, "privacy_default_list"),
+        true ->
-    add_pkey(State, "privacy_default_list", ["server_host", "username"]),
+            drop_pkey(State, "privacy_default_list"),
-    drop_sh_default(State, "privacy_default_list"),
+            add_pkey(State, "privacy_default_list", ["server_host", "username"]),
+            drop_sh_default(State, "privacy_default_list");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "privacy_list"),
+    case add_sh_column(State, "privacy_list") of
-    drop_index(State, "privacy_list", "i_privacy_list_username"),
+        true ->
-    drop_index(State, "privacy_list", "i_privacy_list_username_name"),
+            drop_index(State, "privacy_list", "i_privacy_list_username"),
-    create_unique_index(State, "privacy_list", "i_privacy_list_sh_username_name", ["server_host", "username", "name"]),
+            drop_index(State, "privacy_list", "i_privacy_list_username_name"),
-    drop_sh_default(State, "privacy_list"),
+            create_unique_index(State, "privacy_list", "i_privacy_list_sh_username_name", ["server_host", "username", "name"]),
+            drop_sh_default(State, "privacy_list");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "private_storage"),
+    case add_sh_column(State, "private_storage") of
-    drop_index(State, "private_storage", "i_private_storage_username"),
+        true ->
-    drop_index(State, "private_storage", "i_private_storage_username_namespace"),
+            drop_index(State, "private_storage", "i_private_storage_username"),
-    add_pkey(State, "private_storage", ["server_host", "username", "namespace"]),
+            drop_index(State, "private_storage", "i_private_storage_username_namespace"),
-    drop_sh_default(State, "private_storage"),
+            add_pkey(State, "private_storage", ["server_host", "username", "namespace"]),
+            drop_sh_default(State, "private_storage");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "roster_version"),
+    case add_sh_column(State, "roster_version") of
-    drop_pkey(State, "roster_version"),
+        true ->
-    add_pkey(State, "roster_version", ["server_host", "username"]),
+            drop_pkey(State, "roster_version"),
-    drop_sh_default(State, "roster_version"),
+            add_pkey(State, "roster_version", ["server_host", "username"]),
+            drop_sh_default(State, "roster_version");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "muc_room"),
+    case add_sh_column(State, "muc_room") of
-    drop_sh_default(State, "muc_room"),
+        true ->
+            drop_sh_default(State, "muc_room");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "muc_registered"),
+    case add_sh_column(State, "muc_registered") of
-    drop_sh_default(State, "muc_registered"),
+        true ->
+            drop_sh_default(State, "muc_registered");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "muc_online_room"),
+    case add_sh_column(State, "muc_online_room") of
-    drop_sh_default(State, "muc_online_room"),
+        true ->
+            drop_sh_default(State, "muc_online_room");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "muc_online_users"),
+    case add_sh_column(State, "muc_online_users") of
-    drop_sh_default(State, "muc_online_users"),
+        true ->
+            drop_sh_default(State, "muc_online_users");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "motd"),
+    case add_sh_column(State, "motd") of
-    drop_pkey(State, "motd"),
+        true ->
-    add_pkey(State, "motd", ["server_host", "username"]),
+            drop_pkey(State, "motd"),
-    drop_sh_default(State, "motd"),
+            add_pkey(State, "motd", ["server_host", "username"]),
+            drop_sh_default(State, "motd");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "sm"),
+    case add_sh_column(State, "sm") of
-    drop_index(State, "sm", "i_sm_sid"),
+        true ->
-    drop_index(State, "sm", "i_sm_username"),
+            drop_index(State, "sm", "i_sm_sid"),
-    add_pkey(State, "sm", ["usec", "pid"]),
+            drop_index(State, "sm", "i_sm_username"),
-    create_index(State, "sm", "i_sm_sh_username", ["server_host", "username"]),
+            add_pkey(State, "sm", ["usec", "pid"]),
-    drop_sh_default(State, "sm"),
+            create_index(State, "sm", "i_sm_sh_username", ["server_host", "username"]),
+            drop_sh_default(State, "sm");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "push_session"),
+    case add_sh_column(State, "push_session") of
-    drop_index(State, "push_session", "i_push_usn"),
+        true ->
-    drop_index(State, "push_session", "i_push_ut"),
+            drop_index(State, "push_session", "i_push_usn"),
-    create_unique_index(State, "push_session", "i_push_session_susn", ["server_host", "username", "service", "node"]),
+            drop_index(State, "push_session", "i_push_ut"),
-    create_index(State, "push_session", "i_push_session_sh_username_timestamp", ["server_host", "username", "timestamp"]),
+            create_unique_index(State, "push_session", "i_push_session_susn", ["server_host", "username", "service", "node"]),
-    drop_sh_default(State, "push_session"),
+            create_index(State, "push_session", "i_push_session_sh_username_timestamp", ["server_host", "username", "timestamp"]),
+            drop_sh_default(State, "push_session");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "mix_pam"),
+    case add_sh_column(State, "mix_pam") of
-    drop_index(State, "mix_pam", "i_mix_pam"),
+        true ->
-    drop_index(State, "mix_pam", "i_mix_pam_u"),
+            drop_index(State, "mix_pam", "i_mix_pam"),
-    drop_index(State, "mix_pam", "i_mix_pam_us"),
+            drop_index(State, "mix_pam", "i_mix_pam_u"),
-    create_unique_index(State, "mix_pam", "i_mix_pam", ["username", "server_host", "channel", "service"]),
+            drop_index(State, "mix_pam", "i_mix_pam_us"),
-    drop_sh_default(State, "mix_pam"),
+            create_unique_index(State, "mix_pam", "i_mix_pam", ["username", "server_host", "channel", "service"]),
+            drop_sh_default(State, "mix_pam");
+        false ->
+            ok
+    end,
 
-    add_sh_column(State, "mqtt_pub"),
+    case add_sh_column(State, "mqtt_pub") of
-    drop_index(State, "mqtt_pub", "i_mqtt_topic"),
+        true ->
-    create_unique_index(State, "mqtt_pub", "i_mqtt_topic_server", ["topic", "server_host"]),
+            drop_index(State, "mqtt_pub", "i_mqtt_topic"),
-    drop_sh_default(State, "mqtt_pub"),
+            create_unique_index(State, "mqtt_pub", "i_mqtt_topic_server", ["topic", "server_host"]),
+            drop_sh_default(State, "mqtt_pub");
+        false ->
+            ok
+    end,
 
     ok.
 
-add_sh_column(#state{dbtype = pgsql} = State, Table) ->
+check_sh_column(#state{dbtype = mysql} = State, Table) ->
+    DB = ejabberd_option:sql_database(State#state.host),
+    sql_query(
+      State#state.host,
+      ["SELECT 1 FROM information_schema.columns ",
+       "WHERE table_name = '", Table, "' AND column_name = 'server_host' ",
+       "AND table_schema = '", (State#state.escape)(DB), "' ",
+       "GROUP BY table_name, column_name;"], false);
+check_sh_column(State, Table) ->
+    DB = ejabberd_option:sql_database(State#state.host),
+    sql_query(
+      State#state.host,
+      ["SELECT 1 FROM information_schema.columns ",
+       "WHERE table_name = '", Table, "' AND column_name = 'server_host' ",
+       "AND table_catalog = '", (State#state.escape)(DB), "' ",
+       "GROUP BY table_name, column_name;"], false).
+
+add_sh_column(State, Table) ->
+    case check_sh_column(State, Table) of
+        true -> false;
+        false ->
+            do_add_sh_column(State, Table),
+            true
+    end.
+
+do_add_sh_column(#state{dbtype = pgsql} = State, Table) ->
     sql_query(
       State#state.host,
       ["ALTER TABLE ", Table, " ADD COLUMN server_host text NOT NULL DEFAULT '",
        (State#state.escape)(State#state.host),
        "';"]);
-add_sh_column(#state{dbtype = mssql} = State, Table) ->
+do_add_sh_column(#state{dbtype = mssql} = State, Table) ->
     sql_query(
       State#state.host,
-      ["ALTER TABLE [", Table, "] ADD [server_host] varchar (250) NOT NULL CONSTRAINT [server_host_default] DEFAULT '",
+      ["ALTER TABLE [", Table, "] ADD [server_host] varchar (250) NOT NULL ",
+       "CONSTRAINT [server_host_default] DEFAULT '",
        (State#state.escape)(State#state.host),
        "';"]);
-add_sh_column(#state{dbtype = mysql} = State, Table) ->
+do_add_sh_column(#state{dbtype = mysql} = State, Table) ->
     sql_query(
       State#state.host,
       ["ALTER TABLE ", Table, " ADD COLUMN server_host varchar(191) NOT NULL DEFAULT '",
@@ -339,15 +472,44 @@ drop_sh_default(#state{dbtype = mysql} = State, Table) ->
       State#state.host,
       ["ALTER TABLE ", Table, " ALTER COLUMN server_host DROP DEFAULT;"]).
 
-drop_index(#state{dbtype = pgsql} = State, _Table, Index) ->
+check_index(#state{dbtype = pgsql} = State, Table, Index) ->
+    sql_query(
+      State#state.host,
+      ["SELECT 1 FROM pg_indexes WHERE tablename = '", Table,
+       "' AND indexname = '", Index, "';"], false);
+check_index(#state{dbtype = mssql} = State, Table, Index) ->
+    sql_query(
+      State#state.host,
+      ["SELECT 1 FROM sys.tables t ",
+       "INNER JOIN sys.indexes i ON i.object_id = t.object_id ",
+       "WHERE i.index_id > 0 ",
+       "AND i.name = '", Index, "' ",
+       "AND t.name = '", Table, "';"], false);
+check_index(#state{dbtype = mysql} = State, Table, Index) ->
+    DB = ejabberd_option:sql_database(State#state.host),
+    sql_query(
+      State#state.host,
+      ["SELECT 1 FROM information_schema.statistics ",
+       "WHERE table_name = '", Table, "' AND index_name = '", Index, "' ",
+       "AND table_schema = '", (State#state.escape)(DB), "' ",
+       "GROUP BY table_name, index_name;"], false).
+
+drop_index(State, Table, Index) ->
+    OldIndex = old_index_name(State#state.dbtype, Index),
+    case check_index(State, Table, OldIndex) of
+        true -> do_drop_index(State, Table, OldIndex);
+        false -> ok
+    end.
+
+do_drop_index(#state{dbtype = pgsql} = State, _Table, Index) ->
     sql_query(
       State#state.host,
       ["DROP INDEX ", Index, ";"]);
-drop_index(#state{dbtype = mssql} = State, Table, Index) ->
+do_drop_index(#state{dbtype = mssql} = State, Table, Index) ->
     sql_query(
       State#state.host,
-      ["DROP INDEX [", mssql_old_index_name(Index), "] ON [", Table, "];"]);
+      ["DROP INDEX [", Index, "] ON [", Table, "];"]);
-drop_index(#state{dbtype = mysql} = State, Table, Index) ->
+do_drop_index(#state{dbtype = mysql} = State, Table, Index) ->
     sql_query(
       State#state.host,
       ["ALTER TABLE ", Table, " DROP INDEX ", Index, ";"]).
@@ -364,7 +526,7 @@ create_unique_index(#state{dbtype = mssql} = State, Table, Index, Cols) ->
     SCols = string:join(Cols, ", "),
     sql_query(
       State#state.host,
-      ["CREATE UNIQUE ", mssql_clustered(Index), "INDEX [", mssql_new_index_name(Index), "] ",
+      ["CREATE UNIQUE ", mssql_clustered(Index), "INDEX [", new_index_name(State#state.dbtype, Index), "] ",
        "ON [", Table, "] (", SCols, ") ",
        "WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);"]);
 create_unique_index(#state{dbtype = mysql} = State, Table, Index, Cols) ->
@@ -376,47 +538,52 @@ create_unique_index(#state{dbtype = mysql} = State, Table, Index, Cols) ->
        SCols, ");"]).
 
 create_index(#state{dbtype = pgsql} = State, Table, Index, Cols) ->
+    NewIndex = new_index_name(State#state.dbtype, Index),
     SCols = string:join(Cols, ", "),
     sql_query(
       State#state.host,
-      ["CREATE INDEX ", Index, " ON ", Table, " USING btree (",
+      ["CREATE INDEX ", NewIndex, " ON ", Table, " USING btree (",
        SCols, ");"]);
 create_index(#state{dbtype = mssql} = State, Table, Index, Cols) ->
+    NewIndex = new_index_name(State#state.dbtype, Index),
     SCols = string:join(Cols, ", "),
     sql_query(
       State#state.host,
-      ["CREATE INDEX [", mssql_new_index_name(Index), "] ON [", Table, "] (", SCols, ") ",
+      ["CREATE INDEX [", NewIndex, "] ON [", Table, "] (", SCols, ") ",
        "WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);"]);
 create_index(#state{dbtype = mysql} = State, Table, Index, Cols) ->
-    Cols2 = [C ++ mysql_keylen(Index, C) || C <- Cols],
+    NewIndex = new_index_name(State#state.dbtype, Index),
+    Cols2 = [C ++ mysql_keylen(NewIndex, C) || C <- Cols],
     SCols = string:join(Cols2, ", "),
     sql_query(
       State#state.host,
-      ["CREATE INDEX ", Index, " ON ", Table, "(",
+      ["CREATE INDEX ", NewIndex, " ON ", Table, "(",
        SCols, ");"]).
 
-mssql_old_index_name("i_bare_peer") -> "archive_bare_peer";
+old_index_name(mssql, "i_bare_peer") -> "archive_bare_peer";
-mssql_old_index_name("i_peer") -> "archive_peer";
+old_index_name(mssql, "i_peer") -> "archive_peer";
-mssql_old_index_name("i_timestamp") -> "archive_timestamp";
+old_index_name(mssql, "i_timestamp") -> "archive_timestamp";
-mssql_old_index_name("i_username") -> "archive_username";
+old_index_name(mssql, "i_username") -> "archive_username";
-mssql_old_index_name("i_username_bare_peer") -> "archive_username_bare_peer";
+old_index_name(mssql, "i_username_bare_peer") -> "archive_username_bare_peer";
-mssql_old_index_name("i_username_peer") -> "archive_username_peer";
+old_index_name(mssql, "i_username_peer") -> "archive_username_peer";
-mssql_old_index_name("i_username_timestamp") -> "archive_username_timestamp";
+old_index_name(mssql, "i_username_timestamp") -> "archive_username_timestamp";
-mssql_old_index_name("i_push_usn") -> "i_push_usn";
+old_index_name(mssql, "i_push_usn") -> "i_push_usn";
-mssql_old_index_name("i_push_ut") -> "i_push_ut";
+old_index_name(mssql, "i_push_ut") -> "i_push_ut";
-mssql_old_index_name("pk_rosterg_user_jid") -> "rostergroups_username_jid";
+old_index_name(mssql, "pk_rosterg_user_jid") -> "rostergroups_username_jid";
-mssql_old_index_name("i_rosteru_jid") -> "rosterusers_jid";
+old_index_name(mssql, "i_rosteru_jid") -> "rosterusers_jid";
-mssql_old_index_name("i_rosteru_username") -> "rosterusers_username";
+old_index_name(mssql, "i_rosteru_username") -> "rosterusers_username";
-mssql_old_index_name("i_rosteru_user_jid") -> "rosterusers_username_jid";
+old_index_name(mssql, "i_rosteru_user_jid") -> "rosterusers_username_jid";
-mssql_old_index_name("i_despool") -> "spool_username";
+old_index_name(mssql, "i_despool") -> "spool_username";
-mssql_old_index_name("i_sr_user_jid_grp") -> "sr_user_jid_group";
+old_index_name(mssql, "i_sr_user_jid_grp") -> "sr_user_jid_group";
-mssql_old_index_name(Index) -> string:substr(Index, 3).
+old_index_name(mssql, Index) -> string:substr(Index, 3);
+old_index_name(_Type, Index) -> Index.
 
-mssql_new_index_name("i_rosterg_sh_user_jid") -> "rostergroups_sh_username_jid";
+new_index_name(mssql, "i_rosterg_sh_user_jid") -> "rostergroups_sh_username_jid";
-mssql_new_index_name("i_rosteru_sh_jid") -> "rosterusers_sh_jid";
+new_index_name(mssql, "i_rosteru_sh_jid") -> "rosterusers_sh_jid";
-mssql_new_index_name("i_rosteru_sh_user_jid") -> "rosterusers_sh_username_jid";
+new_index_name(mssql, "i_rosteru_sh_user_jid") -> "rosterusers_sh_username_jid";
-mssql_new_index_name("i_sr_user_sh_jid_grp") -> "sr_user_sh_jid_group";
+new_index_name(mssql, "i_sr_user_sh_jid_grp") -> "sr_user_sh_jid_group";
-mssql_new_index_name(Index) -> string:substr(Index, 3).
+new_index_name(mssql, Index) -> string:substr(Index, 3);
+new_index_name(_Type, Index) -> Index.
 
 mssql_clustered("i_mix_pam") -> "";
 mssql_clustered("i_push_session_susn") -> "";
@@ -440,11 +607,21 @@ mysql_keylen(_, "username") -> "(191)";
 mysql_keylen(_, _) -> "".
 
 sql_query(Host, Query) ->
-    io:format("executing \"~ts\" on ~ts~n", [Query, Host]),
+    sql_query(Host, Query, true).
+
+sql_query(Host, Query, Log) ->
+    case Log of
+        true -> io:format("executing \"~ts\" on ~ts~n", [Query, Host]);
+        false -> ok
+    end,
     case ejabberd_sql:sql_query(Host, Query) of
+        {selected, _Cols, []} ->
+            false;
+        {selected, _Cols, [_Rows]} ->
+            true;
         {error, Error} ->
             io:format("error: ~p~n", [Error]),
-            ok;
+            false;
         _ ->
             ok
     end.

test/ejabberd_SUITE.erl

@@ -99,7 +99,8 @@ do_init_per_group(mysql, Config) ->
     case catch ejabberd_sql:sql_query(?MYSQL_VHOST, [<<"select 1;">>]) of
         {selected, _, _} ->
             mod_muc:shutdown_rooms(?MYSQL_VHOST),
-            clear_sql_tables(mysql, ?config(base_dir, Config)),
+            clear_sql_tables(mysql, Config),
+            update_sql(?MYSQL_VHOST, Config),
             set_opt(server, ?MYSQL_VHOST, Config);
         Err ->
             {skip, {mysql_not_available, Err}}
@@ -108,7 +109,8 @@ do_init_per_group(mssql, Config) ->
     case catch ejabberd_sql:sql_query(?MSSQL_VHOST, [<<"select 1;">>]) of
         {selected, _, _} ->
             mod_muc:shutdown_rooms(?MSSQL_VHOST),
-            clear_sql_tables(mssql, ?config(base_dir, Config)),
+            clear_sql_tables(mssql, Config),
+            update_sql(?MSSQL_VHOST, Config),
             set_opt(server, ?MSSQL_VHOST, Config);
         Err ->
             {skip, {mssql_not_available, Err}}
@@ -117,7 +119,8 @@ do_init_per_group(pgsql, Config) ->
     case catch ejabberd_sql:sql_query(?PGSQL_VHOST, [<<"select 1;">>]) of
         {selected, _, _} ->
             mod_muc:shutdown_rooms(?PGSQL_VHOST),
-            clear_sql_tables(pgsql, ?config(base_dir, Config)),
+            clear_sql_tables(pgsql, Config),
+            update_sql(?PGSQL_VHOST, Config),
             set_opt(server, ?PGSQL_VHOST, Config);
         Err ->
             {skip, {pgsql_not_available, Err}}
@@ -1011,37 +1014,35 @@ bookmark_conference() ->
 '$handle_undefined_function'(_, _) ->
     erlang:error(undef).
 
+
 %%%===================================================================
 %%% SQL stuff
 %%%===================================================================
-clear_sql_tables(sqlite, _BaseDir) ->
+update_sql(Host, Config) ->
+    case ?config(update_sql, Config) of
+        true ->
+            mod_admin_update_sql:update_sql(Host);
+        false -> ok
+    end.
+
+schema_suffix(Config) ->
+    case ejabberd_sql:use_new_schema() of
+        true ->
+            case ?config(update_sql, Config) of
+                true ->  ".sql";
+                _ -> ".new.sql"
+            end;
+        _ -> ".sql"
+    end.
+
+clear_sql_tables(sqlite, _Config) ->
     ok;
-clear_sql_tables(Type, BaseDir) ->
+clear_sql_tables(Type, Config) ->
+    BaseDir = ?config(base_dir, Config),
     {VHost, File} = case Type of
-                        mysql ->
+                        mysql -> {?MYSQL_VHOST, "mysql" ++ schema_suffix(Config)};
-                            Path = case ejabberd_sql:use_new_schema() of
+                        mssql -> {?MSSQL_VHOST, "mssql" ++ schema_suffix(Config)};
-                                true ->
+                        pgsql -> {?PGSQL_VHOST, "pg" ++ schema_suffix(Config)}
-                                    "mysql.new.sql";
-                                false ->
-                                    "mysql.sql"
-                            end,
-                            {?MYSQL_VHOST, Path};
-                        mssql ->
-                            Path = case ejabberd_sql:use_new_schema() of
-                                true ->
-                                    "mssql.new.sql";
-                                false ->
-                                    "mssql.sql"
-                            end,
-                            {?MSSQL_VHOST, Path};
-                        pgsql ->
-                            Path = case ejabberd_sql:use_new_schema() of
-                                true ->
-                                    "pg.new.sql";
-                                false ->
-                                    "pg.sql"
-                            end,
-                            {?PGSQL_VHOST, Path}
                     end,
     SQLFile = filename:join([BaseDir, "sql", File]),
     CreationQueries = read_sql_queries(SQLFile),

test/ejabberd_SUITE_data/ejabberd.yml

@@ -108,6 +108,7 @@ max_fsm_queue: 1000
 queue_type: file
 modules:
   mod_adhoc: []
+  mod_admin_update_sql: []
   mod_announce: []
   mod_configure: []
   mod_disco: []

test/suite.erl

@@ -131,6 +131,7 @@ init_config(Config) ->
      {resource, <<"resource!@#$%^&*()'\"`~<>+-/;:_=[]{}|\\">>},
      {master_resource, <<"master_resource!@#$%^&*()'\"`~<>+-/;:_=[]{}|\\">>},
      {slave_resource, <<"slave_resource!@#$%^&*()'\"`~<>+-/;:_=[]{}|\\">>},
+     {update_sql, false},
      {password, Password},
      {backends, Backends}
      |Config].

test/vcard_tests.erl

@@ -31,6 +31,7 @@
 		recv_presence/1, recv/1]).
 
 -include("suite.hrl").
+-include_lib("stdlib/include/assert.hrl").
 
 %%%===================================================================
 %%% API
@@ -83,9 +84,9 @@ get_set(Config) ->
 			     "personal website: http://www.saint-andre.com/">>},
     #iq{type = result, sub_els = []} =
         send_recv(Config, #iq{type = set, sub_els = [VCard]}),
-    %% TODO: check if VCard == VCard1.
+    #iq{type = result, sub_els = [VCard1]} =
-    #iq{type = result, sub_els = [_VCard1]} =
         send_recv(Config, #iq{type = get, sub_els = [#vcard_temp{}]}),
+    ?assertEqual(VCard, VCard1),
     disconnect(Config).
 
 service_vcard(Config) ->