Make CI: Add mixed version testing

This is enabled on main and for pull requests. Bazel remains
used in previous branches.

.github/workflows/test-make-target.yaml

@@ -11,6 +11,10 @@ on:
       metadata_store:
         required: true
         type: string
+      mixed_clusters:
+        required: false
+        default: false
+        type: boolean
       make_target:
         required: true
         type: string
@@ -41,6 +45,31 @@ jobs:
         # restricted to the build jobs to avoid duplication in output.
         disable_problem_matchers: true
 
+    - name: MIXED CLUSTERS - FETCH SIGNING KEYS
+      uses: dsaltares/fetch-gh-release-asset@master
+      if: inputs.mixed_clusters
+      with:
+        repo: rabbitmq/signing-keys
+        file: rabbitmq-release-signing-key.asc
+
+    - name: MIXED CLUSTERS - FETCH PREVIOUS VERSION
+      id: fetch_secondary_dist
+      uses: dsaltares/fetch-gh-release-asset@master
+      if: inputs.mixed_clusters
+      with:
+        regex: true
+        file: "rabbitmq-server-generic-unix-[\\d.]*\\.tar.xz"
+        target: ./
+
+    - name: MIXED CLUSTERS - SETUP SECONDARY_DIST
+      if: inputs.mixed_clusters
+      run: |
+        gpg --import rabbitmq-release-signing-key.asc
+        gpg --verify rabbitmq-server-generic-unix-*.asc rabbitmq-server-generic-unix-*.tar.xz
+        tar xf rabbitmq-server-generic-unix-*.tar.xz
+
+        echo "SECONDARY_DIST=${GITHUB_WORKSPACE}/rabbitmq_server-`echo -n ${{ steps.fetch_secondary_dist.outputs.version }} | sed s/v//`" >> $GITHUB_ENV
+
     - name: SETUP DOTNET (rabbit)
       uses: actions/setup-dotnet@v4
       if: inputs.plugin == 'rabbit'
@@ -74,7 +103,7 @@ jobs:
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: CT logs (${{ inputs.plugin }} ${{ inputs.make_target }} OTP-${{ inputs.erlang_version }} ${{ inputs.metadata_store }})
+        name: CT logs (${{ inputs.plugin }} ${{ inputs.make_target }} OTP-${{ inputs.erlang_version }} ${{ inputs.metadata_store }}${{ inputs.mixed_clusters && ' mixed' || '' }})
         path: |
           logs/
 #          !logs/**/log_private

.github/workflows/test-make-tests.yaml

@@ -11,6 +11,9 @@ on:
       metadata_store:
         required: true
         type: string
+      mixed_clusters:
+        required: true
+        type: boolean
 jobs:
   test-rabbit:
     name: Test rabbit
@@ -33,6 +36,7 @@ jobs:
       erlang_version: ${{ inputs.erlang_version }}
       elixir_version: ${{ inputs.elixir_version }}
       metadata_store: ${{ inputs.metadata_store }}
+      mixed_clusters: ${{ inputs.mixed_clusters }}
       make_target: ${{ matrix.make_target }}
       plugin: rabbit
 
@@ -43,6 +47,7 @@ jobs:
       erlang_version: ${{ inputs.erlang_version }}
       elixir_version: ${{ inputs.elixir_version }}
       metadata_store: ${{ inputs.metadata_store }}
+      mixed_clusters: ${{ inputs.mixed_clusters }}
       make_target: parallel-ct-set-1
       plugin: rabbitmq_mqtt
 
@@ -55,6 +60,7 @@ jobs:
       erlang_version: ${{ inputs.erlang_version }}
       elixir_version: ${{ inputs.elixir_version }}
       metadata_store: ${{ inputs.metadata_store }}
+      mixed_clusters: ${{ inputs.mixed_clusters }}
       make_target: ct-config_schema ct-unit
       plugin: rabbitmq_peer_discovery_aws
 
@@ -110,5 +116,6 @@ jobs:
       erlang_version: ${{ inputs.erlang_version }}
       elixir_version: ${{ inputs.elixir_version }}
       metadata_store: ${{ inputs.metadata_store }}
+      mixed_clusters: ${{ inputs.mixed_clusters }}
       make_target: tests
       plugin: ${{ matrix.plugin }}

.github/workflows/test-make.yaml

@@ -74,6 +74,27 @@ jobs:
       erlang_version: ${{ matrix.erlang_version }}
       elixir_version: ${{ matrix.elixir_version }}
       metadata_store: ${{ matrix.metadata_store }}
+      mixed_clusters: false
+
+  test-mixed-clusters:
+    name: Test mixed clusters
+    strategy:
+      fail-fast: false
+      matrix:
+        erlang_version:
+        - '26'
+#        - '27'
+        elixir_version:
+        - '1.17'
+        metadata_store:
+          - mnesia
+#          - khepri
+    uses: ./.github/workflows/test-make-tests.yaml
+    with:
+      erlang_version: ${{ matrix.erlang_version }}
+      elixir_version: ${{ matrix.elixir_version }}
+      metadata_store: ${{ matrix.metadata_store }}
+      mixed_clusters: true
 
   type-check:
     name: Type check

.github/workflows/test-mixed-versions.yaml

@@ -2,7 +2,6 @@ name: Test Mixed Version Clusters
 on:
   push:
     branches:
-    - main
     - v4.0.x
     - v3.13.x
     - bump-otp-*
@@ -21,7 +20,6 @@ on:
     - '*.bzl'
     - '*.bazel'
     - .github/workflows/test-mixed-versions.yaml
-  pull_request:
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true

deps/rabbit/test/cluster_upgrade_SUITE.erl

@@ -55,7 +55,7 @@ init_per_testcase(Testcase, Config) ->
     Config1 = rabbit_ct_helpers:set_config(Config, [
         {rmq_nodename_suffix, Testcase},
         {rmq_nodes_count, 3},
-        {force_secondary_umbrella, true}
+        {force_secondary, true}
       ]),
     Config2 = rabbit_ct_helpers:run_steps(Config1,
                 rabbit_ct_broker_helpers:setup_steps() ++
@@ -139,7 +139,7 @@ upgrade_cluster(Config) ->
      || N <- Cluster],
     ct:pal(?LOW_IMPORTANCE, "Restarting cluster ~p", [Cluster]),
     Config1 = rabbit_ct_helpers:set_config(
-                Config, {force_secondary_umbrella, false}),
+                Config, {force_secondary, false}),
     [ok = rabbit_ct_broker_helpers:async_start_node(Config1, N)
      || N <- Cluster],
     [ok = rabbit_ct_broker_helpers:wait_for_async_start_node(N)

deps/rabbit/test/feature_flags_SUITE.erl

@@ -126,6 +126,7 @@ init_per_group(registry, Config) ->
     logger:set_primary_config(level, debug),
     rabbit_ct_helpers:run_steps(Config, []);
 init_per_group(feature_flags_v2, Config) ->
+    %% @todo Remove this entirely as that FF became required in 3.12.
     %% `feature_flags_v2' is now required and won't work in mixed-version
     %% clusters if the other version doesn't support it.
     case rabbit_ct_helpers:is_mixed_versions() of
@@ -267,6 +268,7 @@ init_per_testcase(Testcase, Config) ->
             Config2 = rabbit_ct_helpers:set_config(
                         Config1,
                         [{rmq_nodename_suffix, Testcase},
+                         {secondary_enabled_plugins, "my_plugin"},
                          {tcp_ports_base, {skip_n_nodes,
                                            TestNumber * ClusterSize}},
                          {net_ticktime, 5}

deps/rabbit_common/mk/rabbitmq-dist.mk

@@ -1,8 +1,8 @@
 .PHONY: dist test-dist do-dist cli-scripts cli-escripts clean-dist
 
-DIST_DIR = plugins
-CLI_SCRIPTS_DIR = sbin
-CLI_ESCRIPTS_DIR = escript
+DIST_DIR ?= $(CURDIR)/plugins
+CLI_SCRIPTS_DIR ?= $(CURDIR)/sbin
+CLI_ESCRIPTS_DIR ?= $(CURDIR)/escript
 MIX = echo y | mix
 
 # Set $(DIST_AS_EZS) to a non-empty value to enable the packaging of

deps/rabbit_common/mk/rabbitmq-run.mk

@@ -19,7 +19,7 @@ TEST_TMPDIR ?= $(TMPDIR)/rabbitmq-test-instances
 endif
 
 # Location of the scripts controlling the broker.
-RABBITMQ_SCRIPTS_DIR ?= $(CURDIR)/sbin
+RABBITMQ_SCRIPTS_DIR ?= $(CLI_SCRIPTS_DIR)
 
 ifeq ($(PLATFORM),msys2)
 RABBITMQ_PLUGINS ?= $(RABBITMQ_SCRIPTS_DIR)/rabbitmq-plugins.bat
@@ -39,7 +39,7 @@ export RABBITMQ_SCRIPTS_DIR RABBITMQCTL RABBITMQ_PLUGINS RABBITMQ_SERVER RABBITM
 export MAKE
 
 # We need to pass the location of codegen to the Java client ant
-# process.
+# process. @todo Delete?
 CODEGEN_DIR = $(DEPS_DIR)/rabbitmq_codegen
 PYTHONPATH = $(CODEGEN_DIR)
 export PYTHONPATH
@@ -90,7 +90,7 @@ ifdef PLUGINS_FROM_DEPS_DIR
 RMQ_PLUGINS_DIR = $(DEPS_DIR)
 DIST_ERL_LIBS = $(ERL_LIBS)
 else
-RMQ_PLUGINS_DIR = $(CURDIR)/$(DIST_DIR)
+RMQ_PLUGINS_DIR = $(DIST_DIR)
 # We do not want to add apps/ or deps/ to ERL_LIBS
 # when running the release from dist. The `plugins`
 # directory is added to ERL_LIBS by rabbitmq-env.

deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl

@@ -433,6 +433,7 @@ start_rabbitmq_node(Master, Config, NodeConfig, I) ->
             %% It's unlikely we'll ever succeed to start RabbitMQ.
             Master ! {self(), Error},
             unlink(Master);
+        %% @todo This might not work right now in at least some cases...
         {skip, _} ->
             %% Try again with another TCP port numbers base.
             NodeConfig4 = move_nonworking_nodedir_away(NodeConfig3),
@@ -506,6 +507,7 @@ tcp_port_base_for_broker0(Config, I, PortsCount) ->
 tcp_port_base_for_broker1(Base, I, PortsCount) ->
     Base + I * PortsCount * ?NODE_START_ATTEMPTS.
 
+%% @todo Refactor to simplify this...
 update_tcp_ports_in_rmq_config(NodeConfig, [tcp_port_amqp = Key | Rest]) ->
     NodeConfig1 = rabbit_ct_helpers:merge_app_env(NodeConfig,
       {rabbit, [{tcp_listeners, [?config(Key, NodeConfig)]}]}),
@@ -626,21 +628,52 @@ write_config_file(Config, NodeConfig, _I) ->
              ConfigFile ++ "\": " ++ file:format_error(Reason)}
     end.
 
+-define(REQUIRED_FEATURE_FLAGS, [
+    %% Required in 3.11:
+    "virtual_host_metadata,"
+    "quorum_queue,"
+    "implicit_default_bindings,"
+    "maintenance_mode_status,"
+    "user_limits,"
+    %% Required in 3.12:
+    "stream_queue,"
+    "classic_queue_type_delivery_support,"
+    "tracking_records_in_ets,"
+    "stream_single_active_consumer,"
+    "listener_records_in_ets,"
+    "feature_flags_v2,"
+    "direct_exchange_routing_v2,"
+    "classic_mirrored_queue_version," %% @todo Missing in FF docs!!
+    %% Required in 3.12 in rabbitmq_management_agent:
+%    "drop_unroutable_metric,"
+%    "empty_basic_get_metric,"
+    %% Required in 4.0:
+    "stream_sac_coordinator_unblock_group,"
+    "restart_streams,"
+    "stream_update_config_command,"
+    "stream_filtering,"
+    "message_containers" %% @todo Update FF docs!! It *is* required.
+]).
+
 do_start_rabbitmq_node(Config, NodeConfig, I) ->
     WithPlugins0 = rabbit_ct_helpers:get_config(Config,
-      broker_with_plugins),
+      broker_with_plugins), %% @todo This is probably not used.
     WithPlugins = case is_list(WithPlugins0) of
         true  -> lists:nth(I + 1, WithPlugins0);
         false -> WithPlugins0
     end,
     ForceUseSecondary = rabbit_ct_helpers:get_config(
-                          Config, force_secondary_umbrella, undefined),
+                          Config, force_secondary, undefined),
     CanUseSecondary = case ForceUseSecondary of
                           undefined ->
                               (I + 1) rem 2 =:= 0;
                           Override when is_boolean(Override) ->
                               Override
                       end,
+    UseSecondaryDist = case ?config(secondary_dist, Config) of
+                               false -> false;
+                               _     -> CanUseSecondary
+                           end,
     UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of
                                false -> false;
                                _     -> CanUseSecondary
@@ -686,8 +719,10 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
     StartWithPluginsDisabled = rabbit_ct_helpers:get_config(
                                  Config, start_rmq_with_plugins_disabled),
     ExtraArgs2 = case StartWithPluginsDisabled of
-                     true -> ["LEAVE_PLUGINS_DISABLED=yes" | ExtraArgs1];
-                     _    -> ExtraArgs1
+                     true ->
+                        ["LEAVE_PLUGINS_DISABLED=1" | ExtraArgs1];
+                     _ ->
+                        ExtraArgs1
                  end,
     KeepPidFile = rabbit_ct_helpers:get_config(
                     Config, keep_pid_file_on_exit),
@@ -731,7 +766,30 @@ do_start_rabbitmq_node(Config, NodeConfig, I) ->
                          {"RABBITMQ_PLUGINS=~ts/rabbitmq-plugins", [SecScriptsDir]}
                          | ExtraArgs4];
                     false ->
-                        ExtraArgs4
+                        case UseSecondaryDist of
+                            true ->
+                                SecondaryDist = ?config(secondary_dist, Config),
+                                SecondaryEnabledPlugins = case {
+                                    StartWithPluginsDisabled,
+                                    ?config(secondary_enabled_plugins, Config),
+                                    filename:basename(SrcDir)
+                                } of
+                                    {true, _, _} -> "";
+                                    {_, undefined, "rabbit"} -> "";
+                                    {_, undefined, SrcPlugin} -> SrcPlugin;
+                                    {_, SecondaryEnabledPlugins0, _} -> SecondaryEnabledPlugins0
+                                end,
+                                [{"DIST_DIR=~ts/plugins", [SecondaryDist]},
+                                 {"CLI_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]},
+                                 {"CLI_ESCRIPTS_DIR=~ts/escript", [SecondaryDist]},
+                                 {"RABBITMQ_SCRIPTS_DIR=~ts/sbin", [SecondaryDist]},
+                                 {"RABBITMQ_SERVER=~ts/sbin/rabbitmq-server", [SecondaryDist]},
+                                 {"RABBITMQ_ENABLED_PLUGINS=~ts", [SecondaryEnabledPlugins]},
+                                 {"RABBITMQ_FEATURE_FLAGS=~ts", [?REQUIRED_FEATURE_FLAGS]}
+                                | ExtraArgs4];
+                            false ->
+                                ExtraArgs4
+                        end
                 end,
     MakeVars = [
       {"RABBITMQ_NODENAME=~ts", [Nodename]},
@@ -1285,6 +1343,10 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
     CanUseSecondary = (I + 1) rem 2 =:= 0,
     BazelRunSecCmd = rabbit_ct_helpers:get_config(
                        Config, rabbitmq_run_secondary_cmd),
+    UseSecondaryDist = case ?config(secondary_dist, Config) of
+                               false -> false;
+                               _     -> CanUseSecondary
+                           end,
     UseSecondaryUmbrella = case ?config(secondary_umbrella, Config) of
                                false ->
                                    case BazelRunSecCmd of
@@ -1327,7 +1389,14 @@ rabbitmqctl(Config, Node, Args, Timeout) ->
                                      "rabbitmqctl"])
                           end;
                       false ->
-                          ?config(rabbitmqctl_cmd, Config)
+                          case UseSecondaryDist of
+                              true ->
+                                  SecondaryDist = ?config(secondary_dist, Config),
+                                  rabbit_misc:format(
+                                    "~ts/sbin/rabbitmqctl", [SecondaryDist]);
+                              false ->
+                                  ?config(rabbitmqctl_cmd, Config)
+                          end
                   end,
 
     NodeConfig = get_node_config(Config, Node),

deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl

@@ -78,6 +78,7 @@ run_setup_steps(Config, ExtraSteps) ->
             [
                 fun init_skip_as_error_flag/1,
                 fun guess_tested_erlang_app_name/1,
+                fun ensure_secondary_dist/1,
                 fun ensure_secondary_umbrella/1,
                 fun ensure_current_srcdir/1,
                 fun ensure_rabbitmq_ct_helpers_srcdir/1,
@@ -201,6 +202,18 @@ guess_tested_erlang_app_name(Config) ->
             set_config(Config, {tested_erlang_app, list_to_atom(AppName)})
     end.
 
+ensure_secondary_dist(Config) ->
+    Path = case get_config(Config, secondary_dist) of
+               undefined -> os:getenv("SECONDARY_DIST");
+               P         -> P
+           end,
+    %% Hard fail if the path is invalid.
+    case Path =:= false orelse filelib:is_dir(Path) of
+        true -> ok;
+        false -> error(secondary_dist_path_invalid)
+    end,
+    set_config(Config, {secondary_dist, Path}).
+
 ensure_secondary_umbrella(Config) ->
     Path = case get_config(Config, secondary_umbrella) of
                undefined -> os:getenv("SECONDARY_UMBRELLA");
@@ -1060,11 +1073,13 @@ convert_to_unicode_binary(Arg) when is_binary(Arg) ->
     Arg.
 
 is_mixed_versions() ->
-    os:getenv("SECONDARY_UMBRELLA") =/= false
+    os:getenv("SECONDARY_DIST") =/= false
+        orelse os:getenv("SECONDARY_UMBRELLA") =/= false
         orelse os:getenv("RABBITMQ_RUN_SECONDARY") =/= false.
 
 is_mixed_versions(Config) ->
-    get_config(Config, secondary_umbrella, false) =/= false
+    get_config(Config, secondary_dist, false) =/= false
+        orelse get_config(Config, secondary_umbrella, false) =/= false
         orelse get_config(Config, rabbitmq_run_secondary_cmd, false) =/= false.
 
 %% -------------------------------------------------------------------