From 41a5bfe24ab8a676a7bf92a513ff1cb1a635c07d Mon Sep 17 00:00:00 2001 From: Aaron Patterson Date: Thu, 19 Dec 2024 15:10:26 -0800 Subject: [PATCH] Fix `it` lexing Given the following code: ```ruby i=2; 42.tap { it /1/i } ``` Before this commit, the tokenizer would treat `it /1/i` as a method call with a regular expression parameter. `it` is to be treated as a local variable, so this needs to be tokenized as division rather than a regular expression. In other words it should be tokenized as `it / 1 / i` This commit fixes tokenization. [Bug #20970] --- src/prism.c | 3 ++- test/prism/lex_test.rb | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/prism.c b/src/prism.c index ab8de96975..d67b0d760e 100644 --- a/src/prism.c +++ b/src/prism.c @@ -11874,7 +11874,8 @@ parser_lex(pm_parser_t *parser) { !(last_state & (PM_LEX_STATE_DOT | PM_LEX_STATE_FNAME)) && (type == PM_TOKEN_IDENTIFIER) && ((pm_parser_local_depth(parser, &parser->current) != -1) || - pm_token_is_numbered_parameter(parser->current.start, parser->current.end)) + pm_token_is_numbered_parameter(parser->current.start, parser->current.end) || + pm_token_is_it(parser->current.start, parser->current.end)) ) { lex_state_set(parser, PM_LEX_STATE_END | PM_LEX_STATE_LABEL); } diff --git a/test/prism/lex_test.rb b/test/prism/lex_test.rb index 7eac677ef7..eaeb425566 100644 --- a/test/prism/lex_test.rb +++ b/test/prism/lex_test.rb @@ -74,6 +74,13 @@ def test_parse_lex_file end end + def test_lex_it_in_block + # `it` should be treated as a local so we have an "ambiguous regex" + # situation below + code = "i=2; 42.tap { it /1/i }" + assert_equal 2, Prism.lex(code).value.map(&:first).map(&:type).count(:SLASH) + end + private def assert_lex(fixture)