Merge pull request #4142 from esl/fix-mnesia-mods-starting-dynamically

Fix mnesia mods starting dynamically (start CETS/RDBMS modules for specific presets)

big_tests/tests/bosh_SUITE.erl

@@ -154,8 +154,9 @@ end_per_testcase(CaseName, Config) ->
 required_modules(without_bosh) ->
     [{mod_bosh, stopped}];
 required_modules(GroupName) ->
-    [{mod_bosh, maps:merge(config_parser_helper:default_mod_config(mod_bosh),
-                           required_bosh_opts(GroupName))}].
+    Backend = ct_helper:get_internal_database(),
+    ModOpts = config_parser_helper:mod_config(mod_bosh, #{backend => Backend}),
+    [{mod_bosh, maps:merge(ModOpts, required_bosh_opts(GroupName))}].
 
 required_bosh_opts(time) ->
     #{max_wait => ?MAX_WAIT, inactivity => ?INACTIVITY};
@@ -941,7 +942,7 @@ wait_until_user_has_no_stanzas(User) ->
 
 wait_for_zero_bosh_sessions() ->
     mongoose_helper:wait_until(fun() ->
-                                       length(get_bosh_sessions())
+                                       get_bosh_sessions()
                                end,
-                               0,
+                               [],
                                #{name => get_bosh_sessions}).

big_tests/tests/domain_isolation_SUITE.erl

@@ -42,8 +42,9 @@ end_per_suite(Config) ->
 
 modules() ->
     MucHost = subhost_pattern(muc_helper:muc_host_pattern()),
+    Backend = mongoose_helper:mnesia_or_rdbms_backend(),
     [{mod_domain_isolation, []},
-     {mod_muc_light, mod_config(mod_muc_light, #{host => MucHost})}].
+     {mod_muc_light, mod_config(mod_muc_light, #{host => MucHost, backend => Backend})}].
 
 init_per_group(two_domains, Config) ->
     Config2 = dynamic_modules:save_modules(host_types(), Config),

big_tests/tests/domain_removal_SUITE.erl

@@ -133,7 +133,8 @@ group_to_modules(muc_light_removal) ->
     [{mod_muc_light, mod_config(mod_muc_light, #{backend => rdbms})}];
 group_to_modules(muc_removal) ->
     MucHost = subhost_pattern(muc_helper:muc_host_pattern()),
-    Opts = #{backend => rdbms, host => MucHost},
+    OnlineBackend = ct_helper:get_internal_database(),
+    Opts = #{backend => rdbms, online_backend => OnlineBackend, host => MucHost},
     [{mod_muc, muc_helper:make_opts(Opts)}];
 group_to_modules(inbox_removal) ->
     [{mod_inbox, inbox_helper:inbox_opts()}];

big_tests/tests/gdpr_SUITE.erl

@@ -344,36 +344,46 @@ groupchat_module(muclight) ->
                                 #{backend => mongoose_helper:mnesia_or_rdbms_backend(),
                                   rooms_in_rosters => true})}].
 
-mam_required_modules(CN, Backend) when CN =:= remove_mam_pm;
-                                       CN =:= retrieve_mam_pm ->
+mam_required_modules(CN, Backend)
+        when CN =:= remove_mam_pm;
+             CN =:= retrieve_mam_pm ->
     [{mod_mam, mam_helper:config_opts(#{backend => Backend, pm => #{}})}];
-mam_required_modules(CN, Backend) when CN =:= retrieve_mam_pm_and_muc_light_dont_interfere;
-                                       CN =:= retrieve_mam_muc_light ->
+mam_required_modules(CN, Backend)
+        when CN =:= retrieve_mam_pm_and_muc_light_dont_interfere;
+             CN =:= retrieve_mam_muc_light ->
     HostPattern = subhost_pattern(muc_light_helper:muc_host_pattern()),
+    MucLightOpts = #{backend => mongoose_helper:mnesia_or_rdbms_backend()},
     [{mod_mam, mam_helper:config_opts(#{backend => Backend,
                                         pm => #{},
                                         muc => #{host => HostPattern}})},
-     {mod_muc_light, default_mod_config(mod_muc_light)}];
+     {mod_muc_light, mod_config(mod_muc_light, MucLightOpts)}];
 mam_required_modules(retrieve_mam_pm_and_muc_light_interfere, Backend) ->
     HostPattern = subhost_pattern(muc_light_helper:muc_host_pattern()),
+    MucLightOpts = #{backend => mongoose_helper:mnesia_or_rdbms_backend()},
     [{mod_mam, mam_helper:config_opts(#{backend => Backend,
                                         db_message_format => mam_message_xml,
                                         pm => #{archive_groupchats => true},
                                         muc => #{host => HostPattern}})},
-     {mod_muc_light, default_mod_config(mod_muc_light)}];
+     {mod_muc_light, mod_config(mod_muc_light, MucLightOpts)}];
 mam_required_modules(CN, Backend) when CN =:= retrieve_mam_muc_private_msg;
-                                       CN =:= retrieve_mam_muc ->
+                                               CN =:= retrieve_mam_muc ->
     HostPattern = subhost_pattern(muc_helper:muc_host_pattern()),
+    MucOpts = #{host => HostPattern,
+                online_backend => ct_helper:get_internal_database(),
+                backend => mongoose_helper:mnesia_or_rdbms_backend()},
     [{mod_mam, mam_helper:config_opts(#{backend => Backend,
                                         pm => #{},
                                         muc => #{host => HostPattern}})},
-     {mod_muc, muc_helper:make_opts(#{host => HostPattern})}];
+     {mod_muc, muc_helper:make_opts(MucOpts)}];
 mam_required_modules(retrieve_mam_muc_store_pm, Backend) ->
     HostPattern = subhost_pattern(muc_helper:muc_host_pattern()),
+    MucOpts = #{host => HostPattern,
+                online_backend => ct_helper:get_internal_database(),
+                backend => mongoose_helper:mnesia_or_rdbms_backend()},
     [{mod_mam, mam_helper:config_opts(#{backend => Backend,
                                         pm => #{archive_groupchats => true},
                                         muc => #{host => HostPattern}})},
-     {mod_muc, muc_helper:make_opts(#{host => HostPattern})}].
+     {mod_muc, muc_helper:make_opts(MucOpts)}].
 
 pick_enabled_backend() ->
     BackendsList = [

big_tests/tests/graphql_muc_SUITE.erl

@@ -289,8 +289,9 @@ ensure_muc_stopped() ->
     muc_helper:unload_muc(SecondaryHostType).
 
 ensure_muc_light_started(Config) ->
+    Backend = mongoose_helper:mnesia_or_rdbms_backend(),
     MucLightOpts = config_parser_helper:mod_config(mod_muc_light,
-        #{rooms_in_rosters => true, config_schema => custom_schema()}),
+        #{backend => Backend, rooms_in_rosters => true, config_schema => custom_schema()}),
     HostType = domain_helper:host_type(),
     dynamic_modules:ensure_modules(HostType, [{mod_muc_light, MucLightOpts}]),
     [{muc_light_host, muc_light_helper:muc_host()} | Config].

big_tests/tests/graphql_muc_light_SUITE.erl

@@ -209,7 +209,9 @@ end_per_suite(Config) ->
     escalus:end_per_suite(Config).
 
 required_modules(_) ->
-    MucLightOpts = mod_config(mod_muc_light, #{rooms_in_rosters => true,
+    Backend = mongoose_helper:mnesia_or_rdbms_backend(),
+    MucLightOpts = mod_config(mod_muc_light, #{backend => Backend,
+                                               rooms_in_rosters => true,
                                                config_schema => custom_schema()}),
     [{mod_muc_light, MucLightOpts}].
 
@@ -518,7 +520,7 @@ user_delete_room_story(Config, Alice, Bob) ->
     AliceBin = escalus_client:short_jid(Alice),
     BobBin = escalus_client:short_jid(Bob),
     Name = <<"first room">>,
-    {ok, #{jid := #jid{luser = RoomID} = RoomJID}} =
+    {ok, #{jid := #jid{luser = RoomID} = RoomJID} = RoomInfo} =
         create_room(MUCServer, Name, <<"subject">>, AliceBin),
     {ok, _} = invite_user(RoomJID, AliceBin, BobBin),
     % Member cannot delete room
@@ -529,7 +531,7 @@ user_delete_room_story(Config, Alice, Bob) ->
     Res2 = user_delete_room(Alice, jid:to_binary(RoomJID), Config),
     ?assertNotEqual(nomatch, binary:match(get_ok_value(?DELETE_ROOM_PATH, Res2),
                                           <<"successfully">>)),
-    ?assertEqual({error, not_exists}, get_room_info(jid:from_binary(RoomJID))),
+    ?assertEqual({error, not_exists}, get_room_info(RoomJID), RoomInfo),
     % Try with a non-existent domain
     Res3 = user_delete_room(Alice, make_bare_jid(RoomID, ?UNKNOWN_DOMAIN), Config),
     ?assertNotEqual(nomatch, binary:match(get_err_msg(Res3), <<"not found">>)),
@@ -1290,11 +1292,11 @@ admin_delete_room_story(Config, Alice) ->
     AliceBin = escalus_client:short_jid(Alice),
     MUCServer = ?config(muc_light_host, Config),
     Name = <<"first room">>,
-    {ok, #{jid := RoomJID}} = create_room(MUCServer, Name, <<"subject">>, AliceBin),
+    {ok, #{jid := RoomJID} = RoomInfo} = create_room(MUCServer, Name, <<"subject">>, AliceBin),
     Res = delete_room(jid:to_binary(RoomJID), Config),
     ?assertNotEqual(nomatch, binary:match(get_ok_value(?DELETE_ROOM_PATH, Res),
                                           <<"successfully">>)),
-    ?assertEqual({error, not_exists}, get_room_info(jid:from_binary(RoomJID))),
+    ?assertEqual({error, not_exists}, get_room_info(RoomJID), RoomInfo),
     % Try with a non-existent room
     Res2 = delete_room(make_bare_jid(?UNKNOWN, MUCServer), Config),
     ?assertNotEqual(nomatch, binary:match(get_err_msg(Res2), <<"not found">>)).
@@ -1721,7 +1723,9 @@ invite_user(RoomJID, SenderBin, RecipientBin) ->
     RecipientJID = jid:from_binary(RecipientBin),
     rpc(mim(), mod_muc_light_api, invite_to_room, [RoomJID, SenderJID, RecipientJID]).
 
-get_room_info(JID) ->
+get_room_info(error) ->
+    arg_is_not_jid;
+get_room_info(#jid{} = JID) ->
     HostType = domain_helper:host_type(),
     RoomUS = jid:to_lus(JID),
     rpc(mim(), mod_muc_light_db_backend, get_info, [HostType, RoomUS]).

big_tests/tests/mam_SUITE.erl

@@ -221,8 +221,6 @@
 
 -import(domain_helper, [domain/0]).
 
--import(config_parser_helper, [default_mod_config/1]).
-
 -include("mam_helper.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("exml/include/exml_stream.hrl").
@@ -244,15 +242,15 @@ configurations() ->
 %% Called by test-runner for autocompletion
 all_configurations() ->
     cassandra_configs(true)
-    ++ rdbms_configs(true)
+    ++ rdbms_configs(true, mnesia)
     ++ elasticsearch_configs(true).
 
 configurations_for_running_ct() ->
     cassandra_configs(is_cassandra_enabled(host_type()))
-    ++ rdbms_configs(mongoose_helper:is_rdbms_enabled(host_type()))
+    ++ rdbms_configs(mongoose_helper:is_rdbms_enabled(host_type()), ct_helper:get_internal_database())
     ++ elasticsearch_configs(is_elasticsearch_enabled(host_type())).
 
-rdbms_configs(true) ->
+rdbms_configs(true, mnesia) ->
     [rdbms,
      rdbms_easy,
      rdbms_async_pool,
@@ -261,7 +259,14 @@ rdbms_configs(true) ->
      rdbms_cache,
      rdbms_mnesia_cache
     ];
-rdbms_configs(_) ->
+rdbms_configs(true, cets) ->
+    [rdbms,
+     rdbms_easy,
+     rdbms_async_pool,
+     rdbms_async_cache,
+     rdbms_cache
+    ];
+rdbms_configs(_, _) ->
     [].
 
 cassandra_configs(true) ->
@@ -661,7 +666,8 @@ required_modules_for_group(C, muc_light, Config) ->
     MUCHost = subhost_pattern(muc_light_helper:muc_host_pattern()),
     Opts = config_opts(Extra#{pm => #{}, muc => #{host => MUCHost}}),
     Config1 = maybe_set_wait(C, [muc, pm], [{mam_meta_opts, Opts} | Config]),
-    {[{mod_muc_light, default_mod_config(mod_muc_light)},
+    Backend = mongoose_helper:mnesia_or_rdbms_backend(),
+    {[{mod_muc_light, config_parser_helper:mod_config(mod_muc_light, #{backend => Backend})},
       {mod_mam, Opts}], Config1};
 required_modules_for_group(C, BG, Config) when BG =:= muc_all;
                                                BG =:= muc_disabled_retraction ->
@@ -883,7 +889,6 @@ init_per_testcase(C=muc_text_search_request, Config) ->
     skip_if_cassandra(Config, Init);
 init_per_testcase(C = muc_light_stored_in_pm_if_allowed_to, Config) ->
     dynamic_modules:ensure_modules(host_type(), required_modules(C, Config)),
-    clean_archives(Config),
     escalus:init_per_testcase(C, Config);
 init_per_testcase(C, Config) when C =:= muc_light_easy;
                                   C =:= muc_light_shouldnt_modify_pm_archive;
@@ -901,7 +906,8 @@ init_per_testcase(CaseName, Config) ->
     escalus:init_per_testcase(CaseName, Config).
 
 skip_if_retraction_not_supported(Config, Init) ->
-    case lists:member(?config(configuration, Config), rdbms_configs(true)) of
+    ConfList = rdbms_configs(true, ct_helper:get_internal_database()),
+    case lists:member(?config(configuration, Config), ConfList) of
         false ->
             {skip, "message retraction not supported"};
         true ->
@@ -1558,7 +1564,7 @@ muc_querying_for_all_messages_with_jid(Config) ->
 
 muc_light_easy(Config) ->
     escalus:story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
-            Room = <<"testroom">>,
+            Room = muc_helper:fresh_room_name(),
             given_muc_light_room(Room, Alice, []),
 
             M1 = when_muc_light_message_is_sent(Alice, Room,
@@ -1572,7 +1578,7 @@ muc_light_easy(Config) ->
             Aff = when_muc_light_affiliations_are_set(Alice, Room, [{Bob, member}]),
             then_muc_light_affiliations_are_received_by([Alice, Bob], Aff),
 
-            maybe_wait_for_archive(Config),
+            mam_helper:wait_for_room_archive_size(muc_light_host(), Room, 4),
             when_archive_query_is_sent(Bob, muc_light_helper:room_bin_jid(Room), Config),
             ExpectedResponse = [{create, [{Alice, owner}]},
                                 {muc_message, Room, Alice, <<"Msg 1">>},
@@ -1583,7 +1589,7 @@ muc_light_easy(Config) ->
 
 muc_light_shouldnt_modify_pm_archive(Config) ->
     escalus:story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
-            Room = <<"testroom2">>,
+            Room = muc_helper:fresh_room_name(),
             given_muc_light_room(Room, Alice, [{Bob, member}]),
 
             when_pm_message_is_sent(Alice, Bob, <<"private hi!">>),
@@ -1611,8 +1617,8 @@ muc_light_shouldnt_modify_pm_archive(Config) ->
         end).
 
 muc_light_stored_in_pm_if_allowed_to(Config) ->
-    escalus:story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
-            Room = <<"testroom_pm">>,
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
+            Room = muc_helper:fresh_room_name(),
             given_muc_light_room(Room, Alice, [{Bob, member}]),
 
             maybe_wait_for_archive(Config),
@@ -1636,7 +1642,7 @@ muc_light_stored_in_pm_if_allowed_to(Config) ->
 
 muc_light_chat_markers_are_archived_if_enabled(Config) ->
     escalus:story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
-            Room = <<"testroom_markers">>,
+            Room = muc_helper:fresh_room_name(),
             given_muc_light_room(Room, Alice, [{Bob, member}]),
 
             %% Alice sends 3 chat markers
@@ -1664,7 +1670,7 @@ muc_light_chat_markers_are_archived_if_enabled(Config) ->
 
 muc_light_chat_markers_are_not_archived_if_disabled(Config) ->
     escalus:story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
-            Room = <<"testroom_no_markers">>,
+            Room = muc_helper:fresh_room_name(),
             given_muc_light_room(Room, Alice, [{Bob, member}]),
 
             %% Alice sends 3 chat markers

big_tests/tests/mod_global_distrib_SUITE.erl

@@ -198,7 +198,9 @@ init_modules_per_node({NodeName, LocalHost, ReceiverPort}, Config0) ->
                      mod_stream_management],
     [dynamic_modules:ensure_stopped(Node, VirtHost, ModulesToStop) || VirtHost <- VirtHosts],
 
-    SMOpts = config_parser_helper:mod_config(mod_stream_management, #{resume_timeout => 1}),
+    SMBackend = ct_helper:get_internal_database(),
+    SMOpts = config_parser_helper:mod_config(mod_stream_management,
+                                             #{resume_timeout => 1, backend => SMBackend}),
     dynamic_modules:ensure_modules(Node, domain(), [{mod_global_distrib, Opts},
                                                     {mod_stream_management, SMOpts}]),
     Config1.

big_tests/tests/mongooseimctl_SUITE.erl

@@ -26,7 +26,7 @@
                              require_rpc_nodes/1,
                              rpc/4]).
 -import(domain_helper, [host_type/0, domain/0]).
--import(config_parser_helper, [config/2, default_mod_config/1]).
+-import(config_parser_helper, [config/2, mod_config/2]).
 
 -define(HTTP_UPLOAD_FILENAME, "tmp.txt").
 -define(HTTP_UPLOAD_FILESIZE, "5").
@@ -211,10 +211,10 @@ init_per_suite(Config) ->
     Config1 = ejabberd_node_utils:init(Node, Config),
     Config2 = escalus:init_per_suite([{ctl_auth_mods, AuthMods},
                                       {roster_template, TemplatePath} | Config1]),
-    dynamic_modules:ensure_modules(domain_helper:host_type(), [{mod_last,
-        default_mod_config(mod_last)}]),
-    dynamic_modules:ensure_modules(domain_helper:secondary_host_type(),
-        [{mod_last, default_mod_config(mod_last)}]),
+    Backend = mongoose_helper:mnesia_or_rdbms_backend(),
+    LastOpts = mod_config(mod_last, #{backend => Backend}),
+    dynamic_modules:ensure_modules(domain_helper:host_type(), [{mod_last, LastOpts}]),
+    dynamic_modules:ensure_modules(domain_helper:secondary_host_type(), [{mod_last, LastOpts}]),
     prepare_roster_template(TemplatePath, domain()),
     %% dump_and_load requires at least one mnesia table
     %% ensure, that passwd table is available
@@ -236,13 +236,15 @@ end_per_suite(Config) ->
     escalus:end_per_suite(Config1).
 
 init_per_group(basic, Config) ->
+    Backend = mongoose_helper:mnesia_or_rdbms_backend(),
+    OfflineOpts = mod_config(mod_offline, #{backend => Backend}),
     dynamic_modules:ensure_modules(domain_helper:host_type(),
-        [{mod_offline, default_mod_config(mod_offline)}]),
+                                   [{mod_offline, OfflineOpts}]),
     Config;
 init_per_group(private, Config) ->
-    dynamic_modules:ensure_modules(domain_helper:host_type(),
-                                   [{mod_private, #{iqdisc => one_queue}}]
-                                  ),
+    Backend = mongoose_helper:mnesia_or_rdbms_backend(),
+    PrivateOpts = mod_config(mod_private, #{backend => Backend, iqdisc => one_queue}),
+    dynamic_modules:ensure_modules(domain_helper:host_type(), [{mod_private, PrivateOpts}]),
     Config;
 init_per_group(vcard, Config) ->
     case rpc(mim(), gen_mod, get_module_opt, [host_type(), mod_vcard, backend, mnesia]) of

big_tests/tests/push_integration_SUITE.erl

@@ -1020,20 +1020,25 @@ required_modules_for_group(pm_notifications_with_inbox, API, PubSubHost) ->
     [{mod_inbox, inbox_opts()},
      {mod_offline, config_parser_helper:mod_config(mod_offline, #{backend => Backend})} |
      required_modules(API, PubSubHost)];
-required_modules_for_group(groupchat_notifications_with_inbox, API, PubSubHost)->
+required_modules_for_group(groupchat_notifications_with_inbox, API, PubSubHost) ->
     [{mod_inbox, inbox_opts()}, {mod_muc_light, muc_light_opts()}
      | required_modules(API, PubSubHost)];
 required_modules_for_group(muclight_msg_notifications, API, PubSubHost) ->
     [{mod_muc_light, muc_light_opts()} | required_modules(API, PubSubHost)];
 required_modules_for_group(integration_with_sm_and_offline_storage, API, PubSubHost) ->
     Backend = mongoose_helper:mnesia_or_rdbms_backend(),
+    MemBackend = ct_helper:get_internal_database(),
     [{mod_muc_light, muc_light_opts()},
      {mod_stream_management, config_parser_helper:mod_config(mod_stream_management,
-                                                             #{ack_freq => never, resume_timeout => 1})},
+                                                             #{ack_freq => never, resume_timeout => 1,
+                                                               backend => MemBackend})},
      {mod_offline, config_parser_helper:mod_config(mod_offline, #{backend => Backend})} |
      required_modules(API, PubSubHost)];
 required_modules_for_group(enhanced_integration_with_sm, API, PubSubHost) ->
-    [{mod_stream_management, config_parser_helper:mod_config(mod_stream_management, #{ack_freq => never})} |
+    MemBackend = ct_helper:get_internal_database(),
+    [{mod_stream_management,
+      config_parser_helper:mod_config(mod_stream_management,
+                                      #{ack_freq => never, backend => MemBackend})} |
      required_modules(API, PubSubHost, enhanced_plugin_module_opts())];
 required_modules_for_group(_, API, PubSubHost) ->
     required_modules(API, PubSubHost).