diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 687a6729e2..04797e16a5 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,6 +1,6 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2025-03-14 09:41:32 UTC using RuboCop version 1.69.2. +# on 2025-12-15 17:20:38 UTC using RuboCop version 1.71.2. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new @@ -11,6 +11,7 @@ # TODO - [LH] -> Jul '24 - 369 files inspected, 661 offenses detected, 98 offenses autocorrectable # TODO - [LH] -> Jan '25 (Updated deps and v10 prep) - 369 files inspected, 704 offenses detected, 112 offenses autocorrectable # TODO - [LH] -> Mar '25 (v10 prep) - 370 files inspected, 721 offenses detected, 116 offenses autocorrectable +# TODO - [LH] -> Dec '25 - 375 files inspected, 713 offenses detected, 109 offenses autocorrectable # Offense count: 1 # This cop supports safe autocorrection (--autocorrect). @@ -46,7 +47,7 @@ Lint/UselessMethodDefinition: Exclude: - 'lib/cucumber/glue/proto_world.rb' -# Offense count: 61 +# Offense count: 62 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes. Metrics/AbcSize: Max: 127 @@ -55,29 +56,29 @@ Metrics/AbcSize: # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns, inherit_mode. # AllowedMethods: refine Metrics/BlockLength: - Max: 52 + Max: 53 # Offense count: 13 # Configuration parameters: CountComments, CountAsOne. Metrics/ClassLength: - Max: 515 + Max: 516 -# Offense count: 8 +# Offense count: 7 # Configuration parameters: AllowedMethods, AllowedPatterns. Metrics/CyclomaticComplexity: Max: 12 -# Offense count: 76 +# Offense count: 78 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. Metrics/MethodLength: - Max: 64 + Max: 65 # Offense count: 15 # Configuration parameters: CountComments, CountAsOne. Metrics/ModuleLength: Max: 804 -# Offense count: 8 +# Offense count: 7 # Configuration parameters: AllowedMethods, AllowedPatterns. Metrics/PerceivedComplexity: Max: 13 @@ -146,7 +147,7 @@ RSpec/EmptyExampleGroup: - 'spec/cucumber/filters/activate_steps_spec.rb' - 'spec/cucumber/running_test_case_spec.rb' -# Offense count: 74 +# Offense count: 70 # Configuration parameters: CountAsOne. RSpec/ExampleLength: Max: 58 @@ -178,12 +179,12 @@ RSpec/ExpectInHook: - 'spec/cucumber/multiline_argument/data_table_spec.rb' - 'spec/cucumber/runtime/meta_message_builder_spec.rb' -# Offense count: 13 +# Offense count: 14 RSpec/ExpectOutput: Exclude: - 'spec/cucumber/formatter/interceptor_spec.rb' -# Offense count: 65 +# Offense count: 63 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: EnforcedStyle. # SupportedStyles: implicit, each, example @@ -197,7 +198,7 @@ RSpec/IndexedLet: - 'spec/cucumber/filters/retry_spec.rb' - 'spec/cucumber/glue/registry_and_more_spec.rb' -# Offense count: 36 +# Offense count: 37 # Configuration parameters: AssignmentOnly. RSpec/InstanceVariable: Exclude: @@ -211,24 +212,14 @@ RSpec/MatchArray: Exclude: - 'spec/cucumber/cli/configuration_spec.rb' -# Offense count: 5 +# Offense count: 4 # Configuration parameters: EnforcedStyle. # SupportedStyles: have_received, receive RSpec/MessageSpies: Exclude: - - 'spec/cucumber/deprecate_spec.rb' - 'spec/cucumber/formatter/io_http_buffer_spec.rb' - 'spec/cucumber/runtime/hooks_examples.rb' -# Offense count: 2 -# This cop supports safe autocorrection (--autocorrect). -# Configuration parameters: EnforcedStyle. -# SupportedStyles: hash, symbol -RSpec/MetadataStyle: - Exclude: - - 'compatibility/cck_spec.rb' - - 'spec/cucumber/project_initializer_spec.rb' - # Offense count: 15 RSpec/MissingExampleGroupArgument: Exclude: @@ -236,11 +227,11 @@ RSpec/MissingExampleGroupArgument: - 'spec/cucumber/formatter/fail_fast_spec.rb' - 'spec/cucumber/formatter/rerun_spec.rb' -# Offense count: 58 +# Offense count: 60 RSpec/MultipleExpectations: Max: 3 -# Offense count: 38 +# Offense count: 39 # Configuration parameters: AllowSubject. RSpec/MultipleMemoizedHelpers: Max: 10 @@ -300,11 +291,6 @@ RSpec/RepeatedExample: - 'spec/cucumber/formatter/rerun_spec.rb' - 'spec/cucumber/world/pending_spec.rb' -# Offense count: 2 -RSpec/RepeatedExampleGroupDescription: - Exclude: - - 'spec/cucumber/glue/proto_world_spec.rb' - # Offense count: 3 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: AutoCorrect. @@ -312,12 +298,6 @@ RSpec/ScatteredLet: Exclude: - 'spec/cucumber/runtime/support_code_spec.rb' -# Offense count: 1 -# This cop supports safe autocorrection (--autocorrect). -RSpec/SortMetadata: - Exclude: - - 'compatibility/cck_spec.rb' - # Offense count: 1 # Configuration parameters: Include, CustomTransform, IgnoreMethods, IgnoreMetadata. # Include: **/*_spec.rb @@ -357,12 +337,6 @@ RSpec/VerifiedDoubles: - 'spec/cucumber/runtime/support_code_spec.rb' - 'spec/cucumber/world/pending_spec.rb' -# Offense count: 1 -# This cop supports unsafe autocorrection (--autocorrect-all). -Security/YAMLLoad: - Exclude: - - 'lib/cucumber/cli/profile_loader.rb' - # Offense count: 3 Style/ClassVars: Exclude: @@ -400,13 +374,12 @@ Style/RedundantFreeze: - 'lib/cucumber/runtime.rb' - 'lib/cucumber/term/ansicolor.rb' -# Offense count: 6 +# Offense count: 5 # This cop supports safe autocorrection (--autocorrect). Style/StderrPuts: Exclude: - 'examples/i18n/Rakefile' - 'lib/cucumber/cli/main.rb' - - 'lib/cucumber/deprecate.rb' - 'lib/cucumber/formatter/unicode.rb' - 'lib/cucumber/rake/task.rb' - 'spec/cucumber/formatter/interceptor_spec.rb' diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f07d81097..e010bd2ace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ This document is formatted according to the principles of [Keep A CHANGELOG](htt Please visit [cucumber/CONTRIBUTING.md](https://github.com/cucumber/cucumber/blob/master/CONTRIBUTING.md) for more info on how to contribute to Cucumber. ## [Unreleased] +### Added +- Added a new option for running order `--reverse` which will run the scenarios in reverse order ([#1807](https://github.com/cucumber/cucumber-ruby/pull/1807) [luke-hill](https://github.com/luke-hill)) ## [10.2.0] - 2025-12-10 ### Changed diff --git a/features/docs/cli/ordering.feature b/features/docs/cli/ordering.feature new file mode 100644 index 0000000000..1c49a98cbd --- /dev/null +++ b/features/docs/cli/ordering.feature @@ -0,0 +1,244 @@ +Feature: Ordering + + Cucumber can run scenarios in different orders. By default, scenarios are run in the order they + appear in the feature files. Use the `--order random` switch to run scenarios in random order. + + You can also run cucumber in a reverse order using `--order reverse`. + + Using different ordering can help you detect situations where you have state + leaking between scenarios, which can cause flickering or fragile tests. + + If you do find a random run that exposes dependencies between your tests, + you can reproduce that run by using the seed that's printed at the end of + the test run. + + For a given seed, the order of scenarios is constant, i.e. if step A runs + before step B, it will always run before step B even if other steps are + skipped. + + Background: + Given a file named "features/bad_practice_part_1.feature" with: + """ + Feature: Bad practice, part 1 + + Scenario: Set state + Given I set some state + + Scenario: Depend on state from a preceding scenario + When I depend on the state + """ + And a file named "features/bad_practice_part_2.feature" with: + """ + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature + When I depend on the state + """ + And a file named "features/unrelated.feature" with: + """ + Feature: Unrelated + + @skipme + Scenario: Do something unrelated + When I do something + """ + And a file named "features/step_definitions/steps.rb" with: + """ + Given('I set some state') do + $global_state = 'set' + end + + Given('I depend on the state') do + raise 'I expect the state to be set!' unless $global_state == 'set' + end + + Given('I do something') do + end + """ + + Scenario: Run scenarios in order + When I run `cucumber` + Then it should pass + And the stdout should contain exactly: + """ + Feature: Bad practice, part 1 + + Scenario: Set state # features/bad_practice_part_1.feature:3 + Given I set some state # features/step_definitions/steps.rb:1 + + Scenario: Depend on state from a preceding scenario # features/bad_practice_part_1.feature:6 + When I depend on the state # features/step_definitions/steps.rb:5 + + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature # features/bad_practice_part_2.feature:3 + When I depend on the state # features/step_definitions/steps.rb:5 + + Feature: Unrelated + + @skipme + Scenario: Do something unrelated # features/unrelated.feature:4 + When I do something # features/step_definitions/steps.rb:9 + + 4 scenarios (4 passed) + 4 steps (4 passed) + 0m0.012s + """ + + @global_state + Scenario: Run scenarios randomized + When I run `cucumber --order random:41544 -q` + Then it should fail + And the stdout should contain exactly: + """ + Feature: Bad practice, part 1 + + Scenario: Depend on state from a preceding scenario + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_1.feature:7:in `I depend on the state' + + Feature: Unrelated + + @skipme + Scenario: Do something unrelated + When I do something + + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_2.feature:4:in `I depend on the state' + + Feature: Bad practice, part 1 + + Scenario: Set state + Given I set some state + + Failing Scenarios: + cucumber features/bad_practice_part_1.feature:6 + cucumber features/bad_practice_part_2.feature:3 + + 4 scenarios (2 failed, 2 passed) + 4 steps (2 failed, 2 passed) + + Randomized with seed 41544 + """ + + @force_legacy_loader + Scenario: Rerun scenarios randomized + When I run `cucumber --order random --format summary` + And I rerun the previous command with the same seed + Then the output of both commands should be the same + + @global_state + Scenario: Run scenarios randomized with some skipped + When I run `cucumber --tags "not @skipme" --order random:41544 -q` + Then it should fail with exactly: + """ + Feature: Bad practice, part 1 + + Scenario: Depend on state from a preceding scenario + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_1.feature:7:in `I depend on the state' + + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_2.feature:4:in `I depend on the state' + + Feature: Bad practice, part 1 + + Scenario: Set state + Given I set some state + + Failing Scenarios: + cucumber features/bad_practice_part_1.feature:6 + cucumber features/bad_practice_part_2.feature:3 + + 3 scenarios (2 failed, 1 passed) + 3 steps (2 failed, 1 passed) + + Randomized with seed 41544 + + """ + + @global_state + Scenario: Run scenarios in reverse order + When I run `cucumber --order reverse -q` + Then it should fail + And the stdout should contain exactly: + """ + Feature: Unrelated + + @skipme + Scenario: Do something unrelated + When I do something + + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_2.feature:4:in `I depend on the state' + + Feature: Bad practice, part 1 + + Scenario: Depend on state from a preceding scenario + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_1.feature:7:in `I depend on the state' + + Scenario: Set state + Given I set some state + + Failing Scenarios: + cucumber features/bad_practice_part_2.feature:3 + cucumber features/bad_practice_part_1.feature:6 + + 4 scenarios (2 failed, 2 passed) + 4 steps (2 failed, 2 passed) + """ + + @global_state + Scenario: Run scenarios in reverse order with some skipped + When I run `cucumber --tags "not @skipme" --order reverse -q` + Then it should fail + And the stdout should contain exactly: + """ + Feature: Bad practice, part 2 + + Scenario: Depend on state from a preceding feature + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_2.feature:4:in `I depend on the state' + + Feature: Bad practice, part 1 + + Scenario: Depend on state from a preceding scenario + When I depend on the state + I expect the state to be set! (RuntimeError) + ./features/step_definitions/steps.rb:6:in `"I depend on the state"' + features/bad_practice_part_1.feature:7:in `I depend on the state' + + Scenario: Set state + Given I set some state + + Failing Scenarios: + cucumber features/bad_practice_part_2.feature:3 + cucumber features/bad_practice_part_1.feature:6 + + 3 scenarios (2 failed, 1 passed) + 3 steps (2 failed, 1 passed) + """ diff --git a/features/docs/cli/randomize.feature b/features/docs/cli/randomize.feature deleted file mode 100644 index f810a774e6..0000000000 --- a/features/docs/cli/randomize.feature +++ /dev/null @@ -1,144 +0,0 @@ -Feature: Randomize - - Use the `--order random` switch to run scenarios in random order. - - This is especially helpful for detecting situations where you have state - leaking between scenarios, which can cause flickering or fragile tests. - - If you do find a random run that exposes dependencies between your tests, - you can reproduce that run by using the seed that's printed at the end of - the test run. - - For a given seed, the order of scenarios is constant, i.e. if step A runs - before step B, it will always run before step B even if other steps are - skipped. - - Background: - Given a file named "features/bad_practice_part_1.feature" with: - """ - Feature: Bad practice, part 1 - - Scenario: Set state - Given I set some state - - Scenario: Depend on state from a preceding scenario - When I depend on the state - """ - And a file named "features/bad_practice_part_2.feature" with: - """ - Feature: Bad practice, part 2 - - Scenario: Depend on state from a preceding feature - When I depend on the state - """ - And a file named "features/unrelated.feature" with: - """ - Feature: Unrelated - - @skipme - Scenario: Do something unrelated - When I do something - """ - And a file named "features/step_definitions/steps.rb" with: - """ - Given(/^I set some state$/) do - $global_state = "set" - end - - Given(/^I depend on the state$/) do - raise "I expect the state to be set!" unless $global_state == "set" - end - - Given(/^I do something$/) do - end - """ - - Scenario: Run scenarios in order - When I run `cucumber` - Then it should pass - - @global_state - Scenario: Run scenarios randomized - When I run `cucumber --order random:41544 -q` - Then it should fail - And the stdout should contain exactly: - """ - Feature: Bad practice, part 1 - - Scenario: Depend on state from a preceding scenario - When I depend on the state - I expect the state to be set! (RuntimeError) - ./features/step_definitions/steps.rb:6:in `/^I depend on the state$/' - features/bad_practice_part_1.feature:7:in `I depend on the state' - - Feature: Unrelated - - @skipme - Scenario: Do something unrelated - When I do something - - Feature: Bad practice, part 2 - - Scenario: Depend on state from a preceding feature - When I depend on the state - I expect the state to be set! (RuntimeError) - ./features/step_definitions/steps.rb:6:in `/^I depend on the state$/' - features/bad_practice_part_2.feature:4:in `I depend on the state' - - Feature: Bad practice, part 1 - - Scenario: Set state - Given I set some state - - Failing Scenarios: - cucumber features/bad_practice_part_1.feature:6 - cucumber features/bad_practice_part_2.feature:3 - - 4 scenarios (2 failed, 2 passed) - 4 steps (2 failed, 2 passed) - - Randomized with seed 41544 - """ - - @force_legacy_loader - Scenario: Rerun scenarios randomized - When I run `cucumber --order random --format summary` - And I rerun the previous command with the same seed - Then the output of both commands should be the same - - @global_state - Scenario: Run scenarios randomized with some skipped - When I run `cucumber --tags "not @skipme" --order random:41544 -q` - Then it should fail with exactly: - """ - Feature: Bad practice, part 1 - - Scenario: Depend on state from a preceding scenario - When I depend on the state - I expect the state to be set! (RuntimeError) - ./features/step_definitions/steps.rb:6:in `/^I depend on the state$/' - features/bad_practice_part_1.feature:7:in `I depend on the state' - - Feature: Bad practice, part 2 - - Scenario: Depend on state from a preceding feature - When I depend on the state - I expect the state to be set! (RuntimeError) - ./features/step_definitions/steps.rb:6:in `/^I depend on the state$/' - features/bad_practice_part_2.feature:4:in `I depend on the state' - - Feature: Bad practice, part 1 - - Scenario: Set state - Given I set some state - - Failing Scenarios: - cucumber features/bad_practice_part_1.feature:6 - cucumber features/bad_practice_part_2.feature:3 - - 3 scenarios (2 failed, 1 passed) - 3 steps (2 failed, 1 passed) - - Randomized with seed 41544 - - """ diff --git a/lib/cucumber/cli/options.rb b/lib/cucumber/cli/options.rb index 3e9104567b..218112e54d 100644 --- a/lib/cucumber/cli/options.rb +++ b/lib/cucumber/cli/options.rb @@ -63,7 +63,7 @@ class Options PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, RETRY_FLAG, RETRY_TOTAL_FLAG, '-l', '--lines', '--port', '-I', '--snippet-type' ].freeze - ORDER_TYPES = %w[defined random].freeze + ORDER_TYPES = %w[defined random reverse].freeze TAG_LIMIT_MATCHER = /(?@\w+):(?\d+)/x.freeze def self.parse(args, out_stream, error_stream, options = {}) @@ -146,6 +146,7 @@ def parse!(args) *<<~TEXT.split("\n")) do |order| [defined] Run scenarios in the order they were defined (default). [random] Shuffle scenarios before running. + [reverse] Run scenarios in the opposite order to which they were defined. Specify SEED to reproduce the shuffling from a previous run. e.g. --order random:5738 TEXT diff --git a/lib/cucumber/configuration.rb b/lib/cucumber/configuration.rb index 7ce2c73a03..db394caec1 100644 --- a/lib/cucumber/configuration.rb +++ b/lib/cucumber/configuration.rb @@ -54,6 +54,10 @@ def randomize? @options[:order] == 'random' end + def reverse_order? + @options[:order] == 'reverse' + end + def seed @options[:seed] end diff --git a/lib/cucumber/filters.rb b/lib/cucumber/filters.rb index 6ca9c5d023..d14c38bdd4 100644 --- a/lib/cucumber/filters.rb +++ b/lib/cucumber/filters.rb @@ -9,6 +9,7 @@ require 'cucumber/filters/prepare_world' require 'cucumber/filters/quit' require 'cucumber/filters/randomizer' +require 'cucumber/filters/reverser' require 'cucumber/filters/retry' require 'cucumber/filters/tag_limits' require 'cucumber/filters/broadcast_test_case_ready_event' diff --git a/lib/cucumber/filters/randomizer.rb b/lib/cucumber/filters/randomizer.rb index 92cf6d5884..f2fd517c73 100644 --- a/lib/cucumber/filters/randomizer.rb +++ b/lib/cucumber/filters/randomizer.rb @@ -6,6 +6,9 @@ module Cucumber module Filters # Batches up all test cases, randomizes them, and then sends them on class Randomizer + attr_reader :seed + private :seed + def initialize(seed, receiver = nil) @receiver = receiver @test_cases = [] @@ -26,7 +29,7 @@ def done end def with_receiver(receiver) - self.class.new(@seed, receiver) + self.class.new(seed, receiver) end private @@ -34,12 +37,9 @@ def with_receiver(receiver) def shuffled_test_cases digester = Digest::SHA2.new(256) @test_cases.map.with_index - .sort_by { |_, index| digester.digest((@seed + index).to_s) } + .sort_by { |_, index| digester.digest((seed + index).to_s) } .map { |test_case, _| test_case } end - - attr_reader :seed - private :seed end end end diff --git a/lib/cucumber/filters/reverser.rb b/lib/cucumber/filters/reverser.rb new file mode 100644 index 0000000000..6bc21c3f47 --- /dev/null +++ b/lib/cucumber/filters/reverser.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'digest/sha2' + +module Cucumber + module Filters + # Reverses the order of test cases + class Reverser + attr_reader :seed + private :seed + + def initialize(receiver = nil) + @receiver = receiver + @test_cases = [] + end + + def test_case(test_case) + @test_cases << test_case + self + end + + def done + reversed_test_cases.each do |test_case| + test_case.describe_to(@receiver) + end + @receiver.done + self + end + + def with_receiver(receiver) + self.class.new(receiver) + end + + private + + def reversed_test_cases + @test_cases.reverse + end + end + end +end diff --git a/lib/cucumber/runtime.rb b/lib/cucumber/runtime.rb index da05e6f211..3315fd171d 100644 --- a/lib/cucumber/runtime.rb +++ b/lib/cucumber/runtime.rb @@ -244,6 +244,7 @@ def filters filters << Cucumber::Core::Test::NameFilter.new(name_regexps) filters << Cucumber::Core::Test::LocationsFilter.new(filespecs.locations) filters << Filters::Randomizer.new(@configuration.seed) if @configuration.randomize? + filters << Filters::Reverser.new if @configuration.reverse_order? # TODO: can we just use Glue::RegistryAndMore's step definitions directly? step_match_search = StepMatchSearch.new(@support_code.registry.method(:step_matches), @configuration) filters << Filters::ActivateSteps.new(step_match_search, @configuration)