From 7eb3c3c752f7dfce1d07781161d6f0bef52852cb Mon Sep 17 00:00:00 2001 From: Santiago Bartesaghi Date: Tue, 18 Jan 2022 17:09:35 -0300 Subject: [PATCH] Make order be random by default --- lib/rspec/core/ordering.rb | 11 +++++------ .../project_initializer/spec/spec_helper.rb | 7 +++---- spec/integration/bisect_runners_spec.rb | 2 +- spec/integration/suite_hooks_errors_spec.rb | 13 +++++++++++++ spec/rspec/core/example_group_spec.rb | 18 ++++++++++++++++-- spec/rspec/core/example_spec.rb | 2 +- .../formatters/documentation_formatter_spec.rb | 4 ++++ .../core/formatters/json_formatter_spec.rb | 2 ++ spec/rspec/core/hooks_filtering_spec.rb | 8 ++++++++ spec/rspec/core/reporter_spec.rb | 2 ++ spec/rspec/core/shared_example_group_spec.rb | 2 ++ 11 files changed, 57 insertions(+), 14 deletions(-) diff --git a/lib/rspec/core/ordering.rb b/lib/rspec/core/ordering.rb index d852324dc5..4db693165f 100644 --- a/lib/rspec/core/ordering.rb +++ b/lib/rspec/core/ordering.rb @@ -85,14 +85,13 @@ def initialize(configuration) @configuration = configuration @strategies = {} - register(:random, Random.new(configuration)) + random = Random.new(configuration) + register(:random, random) register(:recently_modified, RecentlyModified.new) + register(:defined, Identity.new) - identity = Identity.new - register(:defined, identity) - - # The default global ordering is --defined. - register(:global, identity) + # The default global ordering is --random. + register(:global, random) end def fetch(name, &fallback) diff --git a/lib/rspec/core/project_initializer/spec/spec_helper.rb b/lib/rspec/core/project_initializer/spec/spec_helper.rb index 110669d188..5040d5d495 100644 --- a/lib/rspec/core/project_initializer/spec/spec_helper.rb +++ b/lib/rspec/core/project_initializer/spec/spec_helper.rb @@ -66,10 +66,9 @@ # particularly slow. config.profile_examples = 10 - # Run specs in random order to surface order dependencies. If you find an - # order dependency and want to debug it, you can fix the order by providing - # the seed, which is printed after each run. - # --seed 1234 + # RSpec runs specs in a random order by default to surface order dependencies. + # We recommend this setting but it can be changed to others, such as `:defined` + # to run specs in the defined order. See the documentation for more details. config.order = :random # Seed global randomization in this process using the `--seed` CLI option. diff --git a/spec/integration/bisect_runners_spec.rb b/spec/integration/bisect_runners_spec.rb index 19e31c4484..b3de601471 100644 --- a/spec/integration/bisect_runners_spec.rb +++ b/spec/integration/bisect_runners_spec.rb @@ -39,7 +39,7 @@ def with_runner(&block) with_runner do |runner| expect(runner.original_results).to have_attributes( - :all_example_ids => %w[ ./spec/a_spec.rb[1:1] ./spec/a_spec.rb[1:2] ], + :all_example_ids => match_array(%w[ ./spec/a_spec.rb[1:1] ./spec/a_spec.rb[1:2] ]), :failed_example_ids => %w[ ./spec/a_spec.rb[1:2] ] ) diff --git a/spec/integration/suite_hooks_errors_spec.rb b/spec/integration/suite_hooks_errors_spec.rb index 19854f0cf0..5858a00e04 100644 --- a/spec/integration/suite_hooks_errors_spec.rb +++ b/spec/integration/suite_hooks_errors_spec.rb @@ -25,6 +25,7 @@ c.backtrace_exclusion_patterns << %r{/rspec-core/spec/} << %r{rspec_with_simplecov} c.failure_exit_code = failure_exit_code c.error_exit_code = error_exit_code + c.seed = 123 end end @@ -50,6 +51,8 @@ def run_spec_expecting_non_zero(before_or_after) output = run_spec_expecting_non_zero(:before) expect(output).to eq unindent(<<-EOS) + Randomized with seed 123 + An error occurred in a `before(:suite)` hook. Failure/Error: raise 'boom' @@ -61,12 +64,16 @@ def run_spec_expecting_non_zero(before_or_after) Finished in n.nnnn seconds (files took n.nnnn seconds to load) 0 examples, 0 failures, 1 error occurred outside of examples + Randomized with seed 123 + EOS end it 'nicely formats errors in `after(:suite)` hooks and exits with non-zero' do output = run_spec_expecting_non_zero(:after) expect(output).to eq unindent(<<-EOS) + + Randomized with seed 123 . An error occurred in an `after(:suite)` hook. Failure/Error: raise 'boom' @@ -79,6 +86,8 @@ def run_spec_expecting_non_zero(before_or_after) Finished in n.nnnn seconds (files took n.nnnn seconds to load) 1 example, 0 failures, 1 error occurred outside of examples + Randomized with seed 123 + EOS end @@ -115,6 +124,8 @@ def run_spec_expecting_non_zero(before_or_after) expect(output).to eq unindent(<<-EOS) + Randomized with seed 123 + An error occurred in a `before(:suite)` hook. Failure/Error: c.before(:suite) { raise 'before 1' } @@ -140,6 +151,8 @@ def run_spec_expecting_non_zero(before_or_after) Finished in n.nnnn seconds (files took n.nnnn seconds to load) 0 examples, 0 failures, 3 errors occurred outside of examples + Randomized with seed 123 + EOS end end diff --git a/spec/rspec/core/example_group_spec.rb b/spec/rspec/core/example_group_spec.rb index 2d61d434c3..cb6a992932 100644 --- a/spec/rspec/core/example_group_spec.rb +++ b/spec/rspec/core/example_group_spec.rb @@ -823,6 +823,8 @@ def define_and_run_group(define_outer_example = false) end it "runs before_all_defined_in_config, before all, before each, example, after each, after all, after_all_defined_in_config in that order" do + RSpec.configuration.order = :defined + order = [] RSpec.configure do |c| @@ -1442,7 +1444,11 @@ def extract_execution_results(group) end context "with fail_fast enabled" do - before { RSpec.configuration.fail_fast = true } + before do + RSpec.configuration.fail_fast = true + RSpec.configuration.order = :defined + end + let(:group) { RSpec.describe } let(:reporter) { Reporter.new(RSpec.configuration) } @@ -1466,7 +1472,11 @@ def extract_execution_results(group) end context "with fail_fast set to 3" do - before { RSpec.configuration.fail_fast = 3 } + before do + RSpec.configuration.fail_fast = 3 + RSpec.configuration.order = :defined + end + let(:group) { RSpec.describe } let(:reporter) { Reporter.new(RSpec.configuration) } @@ -1628,6 +1638,8 @@ def extract_execution_results(group) end it "applies new metadata-based config items based on the update" do + RSpec.configuration.order = :defined + extension = Module.new do def extension_method; 17; end end @@ -1674,6 +1686,8 @@ def extension_method; 17; end end it "does not cause duplicate hooks to be added when re-configuring the group" do + RSpec.configuration.order = :defined + sequence = [] RSpec.configure do |c| c.before(:example, :foo => true) { sequence << :global_before_hook } diff --git a/spec/rspec/core/example_spec.rb b/spec/rspec/core/example_spec.rb index 76076ce182..dffc33dbae 100644 --- a/spec/rspec/core/example_spec.rb +++ b/spec/rspec/core/example_spec.rb @@ -913,7 +913,7 @@ def expect_pending_result(example) example2 = group.example("example 2") { current_examples << RSpec.current_example } group.run - expect(current_examples).to eq([example1, example2]) + expect(current_examples).to match_array([example1, example2]) end end diff --git a/spec/rspec/core/formatters/documentation_formatter_spec.rb b/spec/rspec/core/formatters/documentation_formatter_spec.rb index e0348115c1..c386dd28a7 100644 --- a/spec/rspec/core/formatters/documentation_formatter_spec.rb +++ b/spec/rspec/core/formatters/documentation_formatter_spec.rb @@ -52,6 +52,8 @@ def execution_result(values) end it "represents nested group using hierarchy tree" do + RSpec.configuration.order = :defined + group = RSpec.describe("root") context1 = group.describe("context 1") context1.example("nested example 1.1"){} @@ -112,6 +114,8 @@ def execution_result(values) end it "strips whitespace for each row" do + RSpec.configuration.order = :defined + group = RSpec.describe(" root ") context1 = group.describe(" nested ") context1.example(" example 1 ") {} diff --git a/spec/rspec/core/formatters/json_formatter_spec.rb b/spec/rspec/core/formatters/json_formatter_spec.rb index ce8a7f6a48..9e1ecdd69f 100644 --- a/spec/rspec/core/formatters/json_formatter_spec.rb +++ b/spec/rspec/core/formatters/json_formatter_spec.rb @@ -20,6 +20,8 @@ end it "outputs expected json (brittle high level functional test)" do + RSpec.configuration.order = :defined + its = [] group = RSpec.describe("one apiece") do its.push it("succeeds") { expect(1).to eq 1 } diff --git a/spec/rspec/core/hooks_filtering_spec.rb b/spec/rspec/core/hooks_filtering_spec.rb index 2a86b11cb6..75e0f15966 100644 --- a/spec/rspec/core/hooks_filtering_spec.rb +++ b/spec/rspec/core/hooks_filtering_spec.rb @@ -1,6 +1,10 @@ module RSpec::Core RSpec.describe "config block hook filtering" do context "when hooks are defined after a group has been defined" do + before do + RSpec.configuration.order = :defined + end + it "still applies" do sequence = [] @@ -444,6 +448,8 @@ def filters end it "does not run if some hook filters don't match the group's filters" do + RSpec.configuration.order = :defined + sequence = [] RSpec.configure do |c| @@ -471,6 +477,8 @@ def filters end it "does not run for examples that do not match, even if their group matches" do + RSpec.configuration.order = :defined + filters = [] RSpec.configure do |c| diff --git a/spec/rspec/core/reporter_spec.rb b/spec/rspec/core/reporter_spec.rb index 16e4822681..1380bf1388 100644 --- a/spec/rspec/core/reporter_spec.rb +++ b/spec/rspec/core/reporter_spec.rb @@ -84,6 +84,8 @@ module RSpec::Core end it "passes messages to the formatter in the correct order" do + RSpec.configuration.order = :defined + order = [] formatter = double("formatter") diff --git a/spec/rspec/core/shared_example_group_spec.rb b/spec/rspec/core/shared_example_group_spec.rb index 6d38eaee22..cb795f8c8b 100644 --- a/spec/rspec/core/shared_example_group_spec.rb +++ b/spec/rspec/core/shared_example_group_spec.rb @@ -298,6 +298,8 @@ def self.bar; 'bar'; end describe "hooks for individual examples that have matching metadata" do it 'runs them' do + RSpec.configuration.order = :defined + sequence = [] define_top_level_shared_group("name") do