From 9cd5ff342dba0953673b8c0cc490612c23822026 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 30 Nov 2025 21:24:32 -0500 Subject: [PATCH 01/22] Introduce using slices instead of locations In the C API, we want to use slices instead of locations in the AST. In this case a "slice" is effectively the same thing as the location, expect it is represented using a 32-bit offset and a 32-bit length. This will cut down on half of the space of all of the locations in the AST. I am introducing this as a new field type to ease the migration path, so that this can be merged on its own and then fields can be moved one at a time. Note that from the Ruby/Java/JavaScript side, this is effectively an invisible change. This only impacts the C/Rust side. --- config.yml | 2 +- rbi/prism/reflection.rbi | 3 ++ rust/ruby-prism/build.rs | 11 ++++- rust/ruby-prism/src/lib.rs | 49 ++++++++++++++++++++- sig/prism/reflection.rbs | 3 ++ src/prism.c | 4 +- templates/ext/prism/api_node.c.erb | 19 ++++++++ templates/include/prism/ast.h.erb | 17 +++++++ templates/java/org/prism/Loader.java.erb | 2 +- templates/java/org/prism/Nodes.java.erb | 2 +- templates/javascript/src/deserialize.js.erb | 2 +- templates/javascript/src/nodes.js.erb | 2 +- templates/lib/prism/dot_visitor.rb.erb | 2 +- templates/lib/prism/dsl.rb.erb | 2 +- templates/lib/prism/inspect_visitor.rb.erb | 2 +- templates/lib/prism/node.rb.erb | 8 ++-- templates/lib/prism/reflection.rb.erb | 7 +++ templates/lib/prism/serialize.rb.erb | 4 +- templates/rbi/prism/dsl.rbi.erb | 2 +- templates/rbi/prism/node.rbi.erb | 2 +- templates/sig/prism/node.rbs.erb | 4 +- templates/src/node.c.erb | 11 ++++- templates/src/prettyprint.c.erb | 14 ++++++ templates/src/serialize.c.erb | 10 +++++ templates/template.rb | 22 +++++++++ 25 files changed, 183 insertions(+), 23 deletions(-) diff --git a/config.yml b/config.yml index 5e29d6fa18..a28d461c52 100644 --- a/config.yml +++ b/config.yml @@ -819,7 +819,7 @@ nodes: alias $foo $bar ^^^^ - name: keyword_loc - type: location + type: slice comment: | The location of the `alias` keyword. diff --git a/rbi/prism/reflection.rbi b/rbi/prism/reflection.rbi index b21e9b5def..29679cd519 100644 --- a/rbi/prism/reflection.rbi +++ b/rbi/prism/reflection.rbi @@ -35,6 +35,9 @@ end class Prism::Reflection::LocationField < Prism::Reflection::Field end +class Prism::Reflection::SliceField < Prism::Reflection::Field +end + class Prism::Reflection::OptionalLocationField < Prism::Reflection::Field end diff --git a/rust/ruby-prism/build.rs b/rust/ruby-prism/build.rs index 05cbee87d0..d4d9449c1c 100644 --- a/rust/ruby-prism/build.rs +++ b/rust/ruby-prism/build.rs @@ -31,6 +31,9 @@ enum NodeFieldType { #[serde(rename = "location")] Location, + #[serde(rename = "slice")] + Slice, + #[serde(rename = "location?")] OptionalLocation, @@ -369,6 +372,12 @@ fn write_node(file: &mut File, flags: &[Flags], node: &Node) -> Result<(), Box { + writeln!(file, " pub fn {}(&self) -> Slice<'pr> {{", field.name)?; + writeln!(file, " let pointer: *mut pm_slice_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; + writeln!(file, " Slice::new(self.parser, unsafe {{ &(*pointer) }})")?; + writeln!(file, " }}")?; + }, NodeFieldType::OptionalLocation => { writeln!(file, " pub fn {}(&self) -> Option> {{", field.name)?; writeln!(file, " let pointer: *mut pm_location_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; @@ -558,7 +567,7 @@ use std::ptr::NonNull; #[allow(clippy::wildcard_imports)] use ruby_prism_sys::*; -use crate::{{ConstantId, ConstantList, Integer, Location, NodeList}}; +use crate::{{ConstantId, ConstantList, Integer, Location, Slice, NodeList}}; " )?; diff --git a/rust/ruby-prism/src/lib.rs b/rust/ruby-prism/src/lib.rs index 34e1517be2..f5a6ed749c 100644 --- a/rust/ruby-prism/src/lib.rs +++ b/rust/ruby-prism/src/lib.rs @@ -19,7 +19,7 @@ use std::mem::MaybeUninit; use std::ptr::NonNull; pub use self::bindings::*; -use ruby_prism_sys::{pm_comment_t, pm_comment_type_t, pm_constant_id_list_t, pm_constant_id_t, pm_diagnostic_t, pm_integer_t, pm_location_t, pm_magic_comment_t, pm_node_destroy, pm_node_list, pm_node_t, pm_parse, pm_parser_free, pm_parser_init, pm_parser_t}; +use ruby_prism_sys::{pm_comment_t, pm_comment_type_t, pm_constant_id_list_t, pm_constant_id_t, pm_diagnostic_t, pm_integer_t, pm_location_t, pm_magic_comment_t, pm_node_destroy, pm_node_list, pm_node_t, pm_parse, pm_parser_free, pm_parser_init, pm_parser_t, pm_slice_t}; /// A range in the source file. pub struct Location<'pr> { @@ -109,6 +109,53 @@ impl std::fmt::Debug for Location<'_> { } } +/// A range in the source file, represented as a start offset and length. +pub struct Slice<'pr> { + parser: NonNull, + pub(crate) start: u32, + pub(crate) length: u32, + marker: PhantomData<&'pr [u8]>, +} + +impl<'pr> Slice<'pr> { + /// Returns a byte slice for the range. + #[must_use] + pub fn as_slice(&self) -> &'pr [u8] { + unsafe { + let parser_start = (*self.parser.as_ptr()).start; + std::slice::from_raw_parts(parser_start.add(self.start as usize), self.length as usize) + } + } + + /// Return a Slice from the given `pm_slice_t`. + #[must_use] + pub(crate) const fn new(parser: NonNull, slice: &'pr pm_slice_t) -> Self { + Slice { + parser, + start: slice.start, + length: slice.length, + marker: PhantomData, + } + } +} + +impl std::fmt::Debug for Slice<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let slice: &[u8] = self.as_slice(); + + let mut visible = String::new(); + visible.push('"'); + + for &byte in slice { + let part: Vec = std::ascii::escape_default(byte).collect(); + visible.push_str(std::str::from_utf8(&part).unwrap()); + } + + visible.push('"'); + write!(f, "{visible}") + } +} + /// An iterator over the nodes in a list. pub struct NodeListIter<'pr> { parser: NonNull, diff --git a/sig/prism/reflection.rbs b/sig/prism/reflection.rbs index 047ea32a50..e8c09d7f10 100644 --- a/sig/prism/reflection.rbs +++ b/sig/prism/reflection.rbs @@ -30,6 +30,9 @@ module Prism class LocationField < Field end + class SliceField < Field + end + class OptionalLocationField < Field end diff --git a/src/prism.c b/src/prism.c index cd4d166a12..66759993fa 100644 --- a/src/prism.c +++ b/src/prism.c @@ -1971,6 +1971,8 @@ pm_missing_node_create(pm_parser_t *parser, const uint8_t *start, const uint8_t return node; } +#define TOKEN2SLICE(parser_, token_) ((pm_slice_t) { .start = (uint32_t) ((token_)->start - (parser_)->start), .length = (uint32_t) ((token_)->end - (token_)->start) }) + /** * Allocate and initialize a new AliasGlobalVariableNode node. */ @@ -1983,7 +1985,7 @@ pm_alias_global_variable_node_create(pm_parser_t *parser, const pm_token_t *keyw .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_ALIAS_GLOBAL_VARIABLE_NODE, 0, keyword, old_name), .new_name = new_name, .old_name = old_name, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword) + .keyword_loc = TOKEN2SLICE(parser, keyword) }; return node; diff --git a/templates/ext/prism/api_node.c.erb b/templates/ext/prism/api_node.c.erb index 23af8886a7..28def84f1d 100644 --- a/templates/ext/prism/api_node.c.erb +++ b/templates/ext/prism/api_node.c.erb @@ -27,6 +27,22 @@ pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t * } } +static VALUE +pm_slice_new(const uint32_t start, const uint32_t length, VALUE source, bool freeze) { + if (freeze) { + VALUE slice_argv[] = { + source, + LONG2FIX(start), + LONG2FIX(length) + }; + + return rb_obj_freeze(rb_class_new_instance(3, slice_argv, rb_cPrismLocation)); + } else { + uint64_t value = ((((uint64_t) start) << 32) | ((uint64_t) length)); + return ULL2NUM(value); + } +} + VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze) { ID type = rb_intern(pm_token_type_name(token->type)); @@ -241,6 +257,9 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi <%- when Prism::Template::OptionalLocationField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze); + <%- when Prism::Template::SliceField -%> +#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" + argv[<%= index %>] = pm_slice_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::UInt8Field -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" argv[<%= index %>] = UINT2NUM(cast-><%= field.name %>); diff --git a/templates/include/prism/ast.h.erb b/templates/include/prism/ast.h.erb index 790cf9ebb8..3804dc833e 100644 --- a/templates/include/prism/ast.h.erb +++ b/templates/include/prism/ast.h.erb @@ -57,6 +57,22 @@ typedef struct { const uint8_t *end; } pm_location_t; +/** + * This struct represents a slice in the source code, defined by an offset and + * a length. Note that we have confirmation that we can represent all slices + * within Ruby source files using 32-bit integers per: + * + * https://bugs.ruby-lang.org/issues/20488#note-1 + * + */ +typedef struct { + /** The offset of the slice from the start of the source. */ + uint32_t start; + + /** The length of the slice. */ + uint32_t length; +} pm_slice_t; + struct pm_node; /** @@ -201,6 +217,7 @@ typedef struct pm_<%= node.human %> { when Prism::Template::ConstantListField then "pm_constant_id_list_t #{field.name}" when Prism::Template::StringField then "pm_string_t #{field.name}" when Prism::Template::LocationField, Prism::Template::OptionalLocationField then "pm_location_t #{field.name}" + when Prism::Template::SliceField then "pm_slice_t #{field.name}" when Prism::Template::UInt8Field then "uint8_t #{field.name}" when Prism::Template::UInt32Field then "uint32_t #{field.name}" when Prism::Template::IntegerField then "pm_integer_t #{field.name}" diff --git a/templates/java/org/prism/Loader.java.erb b/templates/java/org/prism/Loader.java.erb index 6c52d36feb..5a4777591e 100644 --- a/templates/java/org/prism/Loader.java.erb +++ b/templates/java/org/prism/Loader.java.erb @@ -386,7 +386,7 @@ public class Loader { when Prism::Template::ConstantField then "loadConstant()" when Prism::Template::OptionalConstantField then "loadOptionalConstant()" when Prism::Template::ConstantListField then "loadConstants()" - when Prism::Template::LocationField then "loadLocation()" + when Prism::Template::LocationField, Prism::Template::SliceField then "loadLocation()" when Prism::Template::OptionalLocationField then "loadOptionalLocation()" when Prism::Template::UInt8Field then "buffer.get()" when Prism::Template::UInt32Field then "loadVarUInt()" diff --git a/templates/java/org/prism/Nodes.java.erb b/templates/java/org/prism/Nodes.java.erb index 8b0a9ce20d..49977fdee9 100644 --- a/templates/java/org/prism/Nodes.java.erb +++ b/templates/java/org/prism/Nodes.java.erb @@ -342,7 +342,7 @@ public abstract class Nodes { <%- if node.fields.any?(Prism::Template::NodeListField) or node.fields.any?(Prism::Template::ConstantListField) -%> String nextNextIndent = nextIndent + " "; <%- end -%> - <%- [*node.flags, *node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::OptionalLocationField)].each do |field| -%> + <%- [*node.flags, *node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::SliceField).grep_v(Prism::Template::OptionalLocationField)].each do |field| -%> builder.append(nextIndent); builder.append("<%= field.name %>: "); <%- case field -%> diff --git a/templates/javascript/src/deserialize.js.erb b/templates/javascript/src/deserialize.js.erb index a058d6cfeb..549cf53d23 100644 --- a/templates/javascript/src/deserialize.js.erb +++ b/templates/javascript/src/deserialize.js.erb @@ -379,7 +379,7 @@ export function deserialize(source, array) { when Prism::Template::ConstantField then "readRequiredConstant()" when Prism::Template::OptionalConstantField then "readOptionalConstant()" when Prism::Template::ConstantListField then "Array.from({ length: buffer.readVarInt() }, readRequiredConstant)" - when Prism::Template::LocationField then "buffer.readLocation()" + when Prism::Template::LocationField, Prism::Template::SliceField then "buffer.readLocation()" when Prism::Template::OptionalLocationField then "buffer.readOptionalLocation()" when Prism::Template::UInt8Field then "buffer.readByte()" when Prism::Template::UInt32Field then "buffer.readVarInt()" diff --git a/templates/javascript/src/nodes.js.erb b/templates/javascript/src/nodes.js.erb index d955d61303..492a42296d 100644 --- a/templates/javascript/src/nodes.js.erb +++ b/templates/javascript/src/nodes.js.erb @@ -17,7 +17,7 @@ def jstype(field) when Prism::Template::ConstantField then "string" when Prism::Template::OptionalConstantField then "string | null" when Prism::Template::ConstantListField then "string[]" - when Prism::Template::LocationField then "Location" + when Prism::Template::LocationField, Prism::Template::SliceField then "Location" when Prism::Template::OptionalLocationField then "Location | null" when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField, Prism::Template::DoubleField then "number" else raise diff --git a/templates/lib/prism/dot_visitor.rb.erb b/templates/lib/prism/dot_visitor.rb.erb index cd2998fe61..3228816f51 100644 --- a/templates/lib/prism/dot_visitor.rb.erb +++ b/templates/lib/prism/dot_visitor.rb.erb @@ -141,7 +141,7 @@ module Prism end <%- when Prism::Template::StringField, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantListField, Prism::Template::IntegerField, Prism::Template::DoubleField -%> table.field("<%= field.name %>", node.<%= field.name %>.inspect) - <%- when Prism::Template::LocationField -%> + <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> table.field("<%= field.name %>", location_inspect(node.<%= field.name %>)) <%- when Prism::Template::OptionalLocationField -%> unless (<%= field.name %> = node.<%= field.name %>).nil? diff --git a/templates/lib/prism/dsl.rb.erb b/templates/lib/prism/dsl.rb.erb index e16ebb7110..3e4c5fa450 100644 --- a/templates/lib/prism/dsl.rb.erb +++ b/templates/lib/prism/dsl.rb.erb @@ -84,7 +84,7 @@ module Prism "#{field.name}: []" when Prism::Template::StringField "#{field.name}: \"\"" - when Prism::Template::LocationField + when Prism::Template::LocationField, Prism::Template::SliceField "#{field.name}: location" when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField "#{field.name}: 0" diff --git a/templates/lib/prism/inspect_visitor.rb.erb b/templates/lib/prism/inspect_visitor.rb.erb index 3cfe615d85..af0a435082 100644 --- a/templates/lib/prism/inspect_visitor.rb.erb +++ b/templates/lib/prism/inspect_visitor.rb.erb @@ -104,7 +104,7 @@ module Prism else commands << ["<%= pointer %><%= field.name %>: #{<%= field.name %>.inspect}\n", indent] end - <%- when Prism::Template::LocationField, Prism::Template::OptionalLocationField -%> + <%- when Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField -%> commands << ["<%= pointer %><%= field.name %>: #{inspect_location(node.<%= field.name %>)}\n", indent] <%- end -%> <%- end -%> diff --git a/templates/lib/prism/node.rb.erb b/templates/lib/prism/node.rb.erb index ceee2b0ffe..bc84199120 100644 --- a/templates/lib/prism/node.rb.erb +++ b/templates/lib/prism/node.rb.erb @@ -364,7 +364,7 @@ module Prism def comment_targets [<%= node.fields.map { |field| case field - when Prism::Template::NodeField, Prism::Template::LocationField then field.name + when Prism::Template::NodeField, Prism::Template::LocationField, Prism::Template::SliceField then field.name when Prism::Template::OptionalNodeField, Prism::Template::NodeListField, Prism::Template::OptionalLocationField then "*#{field.name}" end }.compact.join(", ") %>] #: Array[Prism::node | Location] @@ -401,7 +401,7 @@ module Prism <%- end -%> <%- end -%> <%- case field -%> - <%- when Prism::Template::LocationField -%> + <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> def <%= field.name %> location = @<%= field.name %> return location if location.is_a?(Location) @@ -437,7 +437,7 @@ module Prism <%- end -%> <%- node.fields.each do |field| -%> <%- case field -%> - <%- when Prism::Template::LocationField -%> + <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> @@ -476,7 +476,7 @@ module Prism def ===(other) other.is_a?(<%= node.name %>)<%= " &&" if (fields = [*node.flags, *node.fields]).any? %> <%- fields.each_with_index do |field, index| -%> - <%- if field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::OptionalLocationField) -%> + <%- if field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::SliceField) || field.is_a?(Prism::Template::OptionalLocationField) -%> (<%= field.name %>.nil? == other.<%= field.name %>.nil?)<%= " &&" if index != fields.length - 1 %> <%- elsif field.is_a?(Prism::Template::NodeListField) || field.is_a?(Prism::Template::ConstantListField) -%> (<%= field.name %>.length == other.<%= field.name %>.length) && diff --git a/templates/lib/prism/reflection.rb.erb b/templates/lib/prism/reflection.rb.erb index 6c8b2f4d25..62449a47a2 100644 --- a/templates/lib/prism/reflection.rb.erb +++ b/templates/lib/prism/reflection.rb.erb @@ -65,6 +65,11 @@ module Prism class OptionalLocationField < Field end + # A slice field represents a slice of the source code. It resolves to a + # Prism::Location in Ruby. + class SliceField < Field + end + # An integer field represents an integer value. It is used to represent the # value of an integer literal, the depth of local variables, and the number # of a numbered reference. It resolves to an Integer in Ruby. @@ -115,6 +120,8 @@ module Prism "StringField.new(:#{field.name})" when Prism::Template::LocationField "LocationField.new(:#{field.name})" + when Prism::Template::SliceField + "SliceField.new(:#{field.name})" when Prism::Template::OptionalLocationField "OptionalLocationField.new(:#{field.name})" when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField diff --git a/templates/lib/prism/serialize.rb.erb b/templates/lib/prism/serialize.rb.erb index 526be67431..4dad1260a9 100644 --- a/templates/lib/prism/serialize.rb.erb +++ b/templates/lib/prism/serialize.rb.erb @@ -529,7 +529,7 @@ module Prism when Prism::Template::ConstantField then "load_constant(constant_pool, encoding)" when Prism::Template::OptionalConstantField then "load_optional_constant(constant_pool, encoding)" when Prism::Template::ConstantListField then "Array.new(load_varuint) { load_constant(constant_pool, encoding) }.tap { |constants| constants.freeze if freeze }" - when Prism::Template::LocationField then "load_location(freeze)" + when Prism::Template::LocationField, Prism::Template::SliceField then "load_location(freeze)" when Prism::Template::OptionalLocationField then "load_optional_location(freeze)" when Prism::Template::UInt8Field then "io.getbyte" when Prism::Template::UInt32Field then "load_varuint" @@ -568,7 +568,7 @@ module Prism when Prism::Template::ConstantField then "load_constant(constant_pool, encoding)" when Prism::Template::OptionalConstantField then "load_optional_constant(constant_pool, encoding)" when Prism::Template::ConstantListField then "Array.new(load_varuint) { load_constant(constant_pool, encoding) }" - when Prism::Template::LocationField then "load_location(freeze)" + when Prism::Template::LocationField, Prism::Template::SliceField then "load_location(freeze)" when Prism::Template::OptionalLocationField then "load_optional_location(freeze)" when Prism::Template::UInt8Field then "io.getbyte" when Prism::Template::UInt32Field then "load_varuint" diff --git a/templates/rbi/prism/dsl.rbi.erb b/templates/rbi/prism/dsl.rbi.erb index 76a39c5051..4b577482a3 100644 --- a/templates/rbi/prism/dsl.rbi.erb +++ b/templates/rbi/prism/dsl.rbi.erb @@ -32,7 +32,7 @@ module Prism::DSL [field.name, "[]", field.rbi_class] when Prism::Template::StringField [field.name, "\"\"", field.rbi_class] - when Prism::Template::LocationField + when Prism::Template::LocationField, Prism::Template::SliceField [field.name, "location", field.rbi_class] when Prism::Template::OptionalLocationField [field.name, "nil", field.rbi_class] diff --git a/templates/rbi/prism/node.rbi.erb b/templates/rbi/prism/node.rbi.erb index c0474bd208..d408929629 100644 --- a/templates/rbi/prism/node.rbi.erb +++ b/templates/rbi/prism/node.rbi.erb @@ -116,7 +116,7 @@ class Prism::<%= node.name -%> < Prism::Node def deconstruct_keys(keys); end <%- node.fields.each do |field| -%> <%- case field -%> - <%- when Prism::Template::LocationField -%> + <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> diff --git a/templates/sig/prism/node.rbs.erb b/templates/sig/prism/node.rbs.erb index 6d6fe67336..b3b2a36d15 100644 --- a/templates/sig/prism/node.rbs.erb +++ b/templates/sig/prism/node.rbs.erb @@ -86,7 +86,7 @@ module Prism <%- node.fields.each do |field| -%> attr_reader <%= field.name %>: <%= field.rbs_class %> <%- end -%> - <%- if (locations = node.fields.select { |field| field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::OptionalLocationField) }) -%> + <%- if (locations = node.fields.select { |field| field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::SliceField) || field.is_a?(Prism::Template::OptionalLocationField) }) -%> <%- locations.each do |field| -%> def save_<%= field.name %>: (_Repository repository) -> void @@ -98,7 +98,7 @@ module Prism def deconstruct_keys: (Array[Symbol] keys) -> { <%= (["node_id: Integer", "location: Location"] + node.fields.map { |field| "#{field.name}: #{field.rbs_class}" }).join(", ") %> } <%- node.fields.each do |field| -%> <%- case field -%> - <%- when Prism::Template::LocationField -%> + <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> def <%= field.name.delete_suffix("_loc") %>: () -> String diff --git a/templates/src/node.c.erb b/templates/src/node.c.erb index 2357e55200..bb097dd7d6 100644 --- a/templates/src/node.c.erb +++ b/templates/src/node.c.erb @@ -108,12 +108,12 @@ pm_node_destroy(pm_parser_t *parser, pm_node_t *node) { <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "prism/templates/src/<%= File.basename(__FILE__) %>" case <%= node.type %>: { - <%- if node.fields.any? { |field| ![Prism::Template::LocationField, Prism::Template::OptionalLocationField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField].include?(field.class) } -%> + <%- if node.fields.any? { |field| ![Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField].include?(field.class) } -%> pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node; <%- end -%> <%- node.fields.each do |field| -%> <%- case field -%> - <%- when Prism::Template::LocationField, Prism::Template::OptionalLocationField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField -%> + <%- when Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField -%> <%- when Prism::Template::NodeField -%> pm_node_destroy(parser, (pm_node_t *)cast-><%= field.name %>); <%- when Prism::Template::OptionalNodeField -%> @@ -232,6 +232,11 @@ pm_dump_json_location(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_l pm_buffer_append_format(buffer, "{\"start\":%" PRIu32 ",\"end\":%" PRIu32 "}", start, end); } +static void +pm_dump_json_slice(pm_buffer_t *buffer, const pm_slice_t *slice) { + pm_buffer_append_format(buffer, "{\"start\":%" PRIu32 ",\"length\":%" PRIu32 "}", slice->start, slice->length); +} + /** * Dump JSON to the given buffer. */ @@ -291,6 +296,8 @@ pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *no pm_buffer_append_byte(buffer, ']'); <%- when Prism::Template::LocationField -%> pm_dump_json_location(buffer, parser, &cast-><%= field.name %>); + <%- when Prism::Template::SliceField -%> + pm_dump_json_slice(buffer, &cast-><%= field.name %>); <%- when Prism::Template::OptionalLocationField -%> if (cast-><%= field.name %>.start != NULL) { pm_dump_json_location(buffer, parser, &cast-><%= field.name %>); diff --git a/templates/src/prettyprint.c.erb b/templates/src/prettyprint.c.erb index 639c2fecf3..136c6d25b9 100644 --- a/templates/src/prettyprint.c.erb +++ b/templates/src/prettyprint.c.erb @@ -17,6 +17,13 @@ prettyprint_location(pm_buffer_t *output_buffer, const pm_parser_t *parser, cons pm_buffer_append_format(output_buffer, "(%" PRIi32 ",%" PRIu32 ")-(%" PRIi32 ",%" PRIu32 ")", start.line, start.column, end.line, end.column); } +static inline void +prettyprint_slice(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_slice_t *slice) { + pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, parser->start + slice->start, parser->start_line); + pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, parser->start + slice->start + slice->length, parser->start_line); + pm_buffer_append_format(output_buffer, "(%" PRIi32 ",%" PRIu32 ")-(%" PRIi32 ",%" PRIu32 ")", start.line, start.column, end.line, end.column); +} + static inline void prettyprint_constant(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_constant_id_t constant_id) { pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, constant_id); @@ -108,6 +115,13 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm pm_buffer_append_string(output_buffer, " = \"", 4); pm_buffer_append_source(output_buffer, location->start, (size_t) (location->end - location->start), PM_BUFFER_ESCAPING_RUBY); pm_buffer_append_string(output_buffer, "\"\n", 2); + <%- when Prism::Template::SliceField -%> + pm_slice_t *slice = &cast-><%= field.name %>; + pm_buffer_append_byte(output_buffer, ' '); + prettyprint_slice(output_buffer, parser, slice); + pm_buffer_append_string(output_buffer, " = \"", 4); + pm_buffer_append_source(output_buffer, parser->start + slice->start, (size_t) slice->length, PM_BUFFER_ESCAPING_RUBY); + pm_buffer_append_string(output_buffer, "\"\n", 2); <%- when Prism::Template::OptionalLocationField -%> pm_location_t *location = &cast-><%= field.name %>; if (location->start == NULL) { diff --git a/templates/src/serialize.c.erb b/templates/src/serialize.c.erb index 3e15a11039..aa0b8837dc 100644 --- a/templates/src/serialize.c.erb +++ b/templates/src/serialize.c.erb @@ -29,6 +29,12 @@ pm_serialize_location(const pm_parser_t *parser, const pm_location_t *location, pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(location->end - location->start)); } +PRISM_ATTRIBUTE_UNUSED static void +pm_serialize_slice(const pm_slice_t *slice, pm_buffer_t *buffer) { + pm_buffer_append_varuint(buffer, slice->start); + pm_buffer_append_varuint(buffer, slice->length); +} + static void pm_serialize_string(const pm_parser_t *parser, const pm_string_t *string, pm_buffer_t *buffer) { switch (string->type) { @@ -125,6 +131,10 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { <%- if field.should_be_serialized? -%> pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); <%- end -%> + <%- when Prism::Template::SliceField -%> + <%- if field.should_be_serialized? -%> + pm_serialize_slice(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + <%- end -%> <%- when Prism::Template::OptionalLocationField -%> <%- if field.should_be_serialized? -%> if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) { diff --git a/templates/template.rb b/templates/template.rb index 6c3efd7e6c..fcaf0ca4a9 100755 --- a/templates/template.rb +++ b/templates/template.rb @@ -398,6 +398,27 @@ def java_type end end + # This represents the same information as a location field, except it is + # represented in the source using two 32-bit integers, as opposed to two + # pointers. + class SliceField < Field + def semantic_field? + false + end + + def rbs_class + "Location" + end + + def rbi_class + "Prism::Location" + end + + def java_type + "Location" + end + end + # This class represents a node in the tree, configured by the config.yml file # in YAML format. It contains information about the name of the node and the # various child nodes it contains. @@ -493,6 +514,7 @@ def field_type_for(name) when "uint32" then UInt32Field when "integer" then IntegerField when "double" then DoubleField + when "slice" then SliceField else raise("Unknown field type: #{name.inspect}") end end From 1af57b22ede44f8d7f7057cfaf8b64732cafa3ec Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 10:45:17 -0500 Subject: [PATCH 02/22] Support optional slices --- config.yml | 4 +- rbi/prism/reflection.rbi | 3 + rust/ruby-prism/build.rs | 14 +++++ sig/prism/reflection.rbs | 3 + src/prism.c | 43 ++++++++------ templates/ext/prism/api_node.c.erb | 7 ++- templates/include/prism/ast.h.erb | 2 +- templates/java/org/prism/Loader.java.erb | 2 +- templates/java/org/prism/Nodes.java.erb | 2 +- templates/javascript/src/deserialize.js.erb | 2 +- templates/javascript/src/nodes.js.erb | 2 +- templates/lib/prism/dot_visitor.rb.erb | 2 +- templates/lib/prism/dsl.rb.erb | 2 +- templates/lib/prism/inspect_visitor.rb.erb | 2 +- templates/lib/prism/node.rb.erb | 8 +-- templates/lib/prism/reflection.rb.erb | 14 ++++- templates/lib/prism/serialize.rb.erb | 4 +- templates/rbi/prism/dsl.rbi.erb | 2 +- templates/rbi/prism/node.rbi.erb | 2 +- templates/sig/prism/node.rbs.erb | 4 +- templates/src/node.c.erb | 10 +++- templates/src/prettyprint.c.erb | 11 ++++ templates/src/serialize.c.erb | 9 +++ templates/template.rb | 62 ++++++++++++++------- 24 files changed, 150 insertions(+), 66 deletions(-) diff --git a/config.yml b/config.yml index a28d461c52..6b052c64b3 100644 --- a/config.yml +++ b/config.yml @@ -968,7 +968,7 @@ nodes: kind: non-void expression comment: Represent the list of zero or more [non-void expressions](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression) within the array. - name: opening_loc - type: location? + type: slice? comment: | Represents the optional source location for the opening token. @@ -977,7 +977,7 @@ nodes: %I(apple orange banana) # "%I(" foo = 1, 2, 3 # nil - name: closing_loc - type: location? + type: slice? comment: | Represents the optional source location for the closing token. diff --git a/rbi/prism/reflection.rbi b/rbi/prism/reflection.rbi index 29679cd519..0cbce6c9d3 100644 --- a/rbi/prism/reflection.rbi +++ b/rbi/prism/reflection.rbi @@ -41,6 +41,9 @@ end class Prism::Reflection::OptionalLocationField < Prism::Reflection::Field end +class Prism::Reflection::OptionalSliceField < Prism::Reflection::Field +end + class Prism::Reflection::IntegerField < Prism::Reflection::Field end diff --git a/rust/ruby-prism/build.rs b/rust/ruby-prism/build.rs index d4d9449c1c..4f6c468fea 100644 --- a/rust/ruby-prism/build.rs +++ b/rust/ruby-prism/build.rs @@ -37,6 +37,9 @@ enum NodeFieldType { #[serde(rename = "location?")] OptionalLocation, + #[serde(rename = "slice?")] + OptionalSlice, + #[serde(rename = "uint8")] UInt8, @@ -389,6 +392,17 @@ fn write_node(file: &mut File, flags: &[Flags], node: &Node) -> Result<(), Box { + writeln!(file, " pub fn {}(&self) -> Option> {{", field.name)?; + writeln!(file, " let pointer: *mut pm_slice_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; + writeln!(file, " let length = unsafe {{ (*pointer).length }};")?; + writeln!(file, " if length == 0 {{")?; + writeln!(file, " None")?; + writeln!(file, " }} else {{")?; + writeln!(file, " Some(Slice::new(self.parser, unsafe {{ &(*pointer) }}))")?; + writeln!(file, " }}")?; + writeln!(file, " }}")?; + }, NodeFieldType::UInt8 => { writeln!(file, " pub fn {}(&self) -> u8 {{", field.name)?; writeln!(file, " unsafe {{ (*self.pointer).{} }}", field.name)?; diff --git a/sig/prism/reflection.rbs b/sig/prism/reflection.rbs index e8c09d7f10..7cd9123154 100644 --- a/sig/prism/reflection.rbs +++ b/sig/prism/reflection.rbs @@ -36,6 +36,9 @@ module Prism class OptionalLocationField < Field end + class OptionalSliceField < Field + end + class IntegerField < Field end diff --git a/src/prism.c b/src/prism.c index 66759993fa..bb5a1b3a4d 100644 --- a/src/prism.c +++ b/src/prism.c @@ -2096,12 +2096,21 @@ static pm_array_node_t * pm_array_node_create(pm_parser_t *parser, const pm_token_t *opening) { pm_array_node_t *node = PM_NODE_ALLOC(parser, pm_array_node_t); - *node = (pm_array_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_ARRAY_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .elements = { 0 } - }; + if (opening == NULL) { + *node = (pm_array_node_t) { + .base = PM_NODE_INIT_BASE(parser, PM_ARRAY_NODE, PM_NODE_FLAG_STATIC_LITERAL), + .opening_loc = { 0 }, + .closing_loc = { 0 }, + .elements = { 0 } + }; + } else { + *node = (pm_array_node_t) { + .base = PM_NODE_INIT_TOKEN(parser, PM_ARRAY_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, opening), + .elements = { 0 } + }; + } return node; } @@ -2111,7 +2120,7 @@ pm_array_node_create(pm_parser_t *parser, const pm_token_t *opening) { */ static inline void pm_array_node_elements_append(pm_array_node_t *node, pm_node_t *element) { - if (!node->elements.size && !node->opening_loc.start) { + if (!node->elements.size && !node->opening_loc.length) { node->base.location.start = element->location.start; } @@ -2133,10 +2142,10 @@ pm_array_node_elements_append(pm_array_node_t *node, pm_node_t *element) { * Set the closing token and end location of an array node. */ static void -pm_array_node_close_set(pm_array_node_t *node, const pm_token_t *closing) { +pm_array_node_close_set(const pm_parser_t *parser, pm_array_node_t *node, const pm_token_t *closing) { assert(closing->type == PM_TOKEN_BRACKET_RIGHT || closing->type == PM_TOKEN_STRING_END || closing->type == PM_TOKEN_MISSING || closing->type == PM_TOKEN_NOT_PROVIDED); node->base.location.end = closing->end; - node->closing_loc = PM_LOCATION_TOKEN_VALUE(closing); + node->closing_loc = TOKEN2SLICE(parser, closing); } /** @@ -3331,7 +3340,7 @@ pm_class_variable_read_node_create(pm_parser_t *parser, const pm_token_t *token) */ static inline pm_node_flags_t pm_implicit_array_write_flags(const pm_node_t *node, pm_node_flags_t flags) { - if (PM_NODE_TYPE_P(node, PM_ARRAY_NODE) && ((const pm_array_node_t *) node)->opening_loc.start == NULL) { + if (PM_NODE_TYPE_P(node, PM_ARRAY_NODE) && ((const pm_array_node_t *) node)->opening_loc.length == 0) { return flags; } return 0; @@ -17465,7 +17474,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser->previous.type = PM_TOKEN_MISSING; } - pm_array_node_close_set(array, &parser->previous); + pm_array_node_close_set(parser, array, &parser->previous); pm_accepts_block_stack_pop(parser); return UP(array); @@ -19322,7 +19331,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_I_LOWER_TERM); } - pm_array_node_close_set(array, &closing); + pm_array_node_close_set(parser, array, &closing); return UP(array); } @@ -19483,7 +19492,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_I_UPPER_TERM); } - pm_array_node_close_set(array, &closing); + pm_array_node_close_set(parser, array, &closing); return UP(array); } @@ -19518,7 +19527,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_W_LOWER_TERM); } - pm_array_node_close_set(array, &closing); + pm_array_node_close_set(parser, array, &closing); return UP(array); } case PM_TOKEN_PERCENT_UPPER_W: { @@ -19665,7 +19674,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_W_UPPER_TERM); } - pm_array_node_close_set(array, &closing); + pm_array_node_close_set(parser, array, &closing); return UP(array); } case PM_TOKEN_REGEXP_BEGIN: { @@ -20159,9 +20168,7 @@ parse_assignment_values(pm_parser_t *parser, pm_binding_power_t previous_binding if (previous_binding_power == PM_BINDING_POWER_STATEMENT && (PM_NODE_TYPE_P(value, PM_SPLAT_NODE) || match1(parser, PM_TOKEN_COMMA))) { single_value = false; - pm_token_t opening = not_provided(parser); - pm_array_node_t *array = pm_array_node_create(parser, &opening); - + pm_array_node_t *array = pm_array_node_create(parser, NULL); pm_array_node_elements_append(array, value); value = UP(array); diff --git a/templates/ext/prism/api_node.c.erb b/templates/ext/prism/api_node.c.erb index 28def84f1d..f56d16fbed 100644 --- a/templates/ext/prism/api_node.c.erb +++ b/templates/ext/prism/api_node.c.erb @@ -254,12 +254,15 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi <%- when Prism::Template::LocationField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" argv[<%= index %>] = pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze); + <%- when Prism::Template::SliceField -%> +#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" + argv[<%= index %>] = pm_slice_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::OptionalLocationField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze); - <%- when Prism::Template::SliceField -%> + <%- when Prism::Template::OptionalSliceField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" - argv[<%= index %>] = pm_slice_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); + argv[<%= index %>] = cast-><%= field.name %>.length == 0 ? Qnil : pm_slice_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::UInt8Field -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" argv[<%= index %>] = UINT2NUM(cast-><%= field.name %>); diff --git a/templates/include/prism/ast.h.erb b/templates/include/prism/ast.h.erb index 3804dc833e..47faf7cfd8 100644 --- a/templates/include/prism/ast.h.erb +++ b/templates/include/prism/ast.h.erb @@ -217,7 +217,7 @@ typedef struct pm_<%= node.human %> { when Prism::Template::ConstantListField then "pm_constant_id_list_t #{field.name}" when Prism::Template::StringField then "pm_string_t #{field.name}" when Prism::Template::LocationField, Prism::Template::OptionalLocationField then "pm_location_t #{field.name}" - when Prism::Template::SliceField then "pm_slice_t #{field.name}" + when Prism::Template::SliceField, Prism::Template::OptionalSliceField then "pm_slice_t #{field.name}" when Prism::Template::UInt8Field then "uint8_t #{field.name}" when Prism::Template::UInt32Field then "uint32_t #{field.name}" when Prism::Template::IntegerField then "pm_integer_t #{field.name}" diff --git a/templates/java/org/prism/Loader.java.erb b/templates/java/org/prism/Loader.java.erb index 5a4777591e..63ff51a557 100644 --- a/templates/java/org/prism/Loader.java.erb +++ b/templates/java/org/prism/Loader.java.erb @@ -387,7 +387,7 @@ public class Loader { when Prism::Template::OptionalConstantField then "loadOptionalConstant()" when Prism::Template::ConstantListField then "loadConstants()" when Prism::Template::LocationField, Prism::Template::SliceField then "loadLocation()" - when Prism::Template::OptionalLocationField then "loadOptionalLocation()" + when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "loadOptionalLocation()" when Prism::Template::UInt8Field then "buffer.get()" when Prism::Template::UInt32Field then "loadVarUInt()" when Prism::Template::IntegerField then "loadInteger()" diff --git a/templates/java/org/prism/Nodes.java.erb b/templates/java/org/prism/Nodes.java.erb index 49977fdee9..edf264318d 100644 --- a/templates/java/org/prism/Nodes.java.erb +++ b/templates/java/org/prism/Nodes.java.erb @@ -342,7 +342,7 @@ public abstract class Nodes { <%- if node.fields.any?(Prism::Template::NodeListField) or node.fields.any?(Prism::Template::ConstantListField) -%> String nextNextIndent = nextIndent + " "; <%- end -%> - <%- [*node.flags, *node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::SliceField).grep_v(Prism::Template::OptionalLocationField)].each do |field| -%> + <%- [*node.flags, *node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::SliceField).grep_v(Prism::Template::OptionalLocationField).grep_v(Prism::Template::OptionalSliceField)].each do |field| -%> builder.append(nextIndent); builder.append("<%= field.name %>: "); <%- case field -%> diff --git a/templates/javascript/src/deserialize.js.erb b/templates/javascript/src/deserialize.js.erb index 549cf53d23..054312a0a7 100644 --- a/templates/javascript/src/deserialize.js.erb +++ b/templates/javascript/src/deserialize.js.erb @@ -380,7 +380,7 @@ export function deserialize(source, array) { when Prism::Template::OptionalConstantField then "readOptionalConstant()" when Prism::Template::ConstantListField then "Array.from({ length: buffer.readVarInt() }, readRequiredConstant)" when Prism::Template::LocationField, Prism::Template::SliceField then "buffer.readLocation()" - when Prism::Template::OptionalLocationField then "buffer.readOptionalLocation()" + when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "buffer.readOptionalLocation()" when Prism::Template::UInt8Field then "buffer.readByte()" when Prism::Template::UInt32Field then "buffer.readVarInt()" when Prism::Template::IntegerField then "readInteger()" diff --git a/templates/javascript/src/nodes.js.erb b/templates/javascript/src/nodes.js.erb index 492a42296d..092d78bdc5 100644 --- a/templates/javascript/src/nodes.js.erb +++ b/templates/javascript/src/nodes.js.erb @@ -18,7 +18,7 @@ def jstype(field) when Prism::Template::OptionalConstantField then "string | null" when Prism::Template::ConstantListField then "string[]" when Prism::Template::LocationField, Prism::Template::SliceField then "Location" - when Prism::Template::OptionalLocationField then "Location | null" + when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "Location | null" when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField, Prism::Template::DoubleField then "number" else raise end diff --git a/templates/lib/prism/dot_visitor.rb.erb b/templates/lib/prism/dot_visitor.rb.erb index 3228816f51..8a1b6f5b71 100644 --- a/templates/lib/prism/dot_visitor.rb.erb +++ b/templates/lib/prism/dot_visitor.rb.erb @@ -143,7 +143,7 @@ module Prism table.field("<%= field.name %>", node.<%= field.name %>.inspect) <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> table.field("<%= field.name %>", location_inspect(node.<%= field.name %>)) - <%- when Prism::Template::OptionalLocationField -%> + <%- when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> unless (<%= field.name %> = node.<%= field.name %>).nil? table.field("<%= field.name %>", location_inspect(<%= field.name %>)) end diff --git a/templates/lib/prism/dsl.rb.erb b/templates/lib/prism/dsl.rb.erb index 3e4c5fa450..cd9b401989 100644 --- a/templates/lib/prism/dsl.rb.erb +++ b/templates/lib/prism/dsl.rb.erb @@ -78,7 +78,7 @@ module Prism end when Prism::Template::ConstantField "#{field.name}: :\"\"" - when Prism::Template::OptionalNodeField, Prism::Template::OptionalConstantField, Prism::Template::OptionalLocationField + when Prism::Template::OptionalNodeField, Prism::Template::OptionalConstantField, Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField "#{field.name}: nil" when Prism::Template::NodeListField, Prism::Template::ConstantListField "#{field.name}: []" diff --git a/templates/lib/prism/inspect_visitor.rb.erb b/templates/lib/prism/inspect_visitor.rb.erb index af0a435082..226c28b148 100644 --- a/templates/lib/prism/inspect_visitor.rb.erb +++ b/templates/lib/prism/inspect_visitor.rb.erb @@ -104,7 +104,7 @@ module Prism else commands << ["<%= pointer %><%= field.name %>: #{<%= field.name %>.inspect}\n", indent] end - <%- when Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField -%> + <%- when Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> commands << ["<%= pointer %><%= field.name %>: #{inspect_location(node.<%= field.name %>)}\n", indent] <%- end -%> <%- end -%> diff --git a/templates/lib/prism/node.rb.erb b/templates/lib/prism/node.rb.erb index bc84199120..1b14335b57 100644 --- a/templates/lib/prism/node.rb.erb +++ b/templates/lib/prism/node.rb.erb @@ -365,7 +365,7 @@ module Prism [<%= node.fields.map { |field| case field when Prism::Template::NodeField, Prism::Template::LocationField, Prism::Template::SliceField then field.name - when Prism::Template::OptionalNodeField, Prism::Template::NodeListField, Prism::Template::OptionalLocationField then "*#{field.name}" + when Prism::Template::OptionalNodeField, Prism::Template::NodeListField, Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "*#{field.name}" end }.compact.join(", ") %>] #: Array[Prism::node | Location] end @@ -413,7 +413,7 @@ module Prism def save_<%= field.name %>(repository) repository.enter(node_id, :<%= field.name %>) end - <%- when Prism::Template::OptionalLocationField -%> + <%- when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> def <%= field.name %> location = @<%= field.name %> case location @@ -445,7 +445,7 @@ module Prism def <%= field.name.delete_suffix("_loc") %> <%= field.name %>.slice end - <%- when Prism::Template::OptionalLocationField -%> + <%- when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> @@ -476,7 +476,7 @@ module Prism def ===(other) other.is_a?(<%= node.name %>)<%= " &&" if (fields = [*node.flags, *node.fields]).any? %> <%- fields.each_with_index do |field, index| -%> - <%- if field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::SliceField) || field.is_a?(Prism::Template::OptionalLocationField) -%> + <%- if field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::SliceField) || field.is_a?(Prism::Template::OptionalLocationField) || field.is_a?(Prism::Template::OptionalSliceField) -%> (<%= field.name %>.nil? == other.<%= field.name %>.nil?)<%= " &&" if index != fields.length - 1 %> <%- elsif field.is_a?(Prism::Template::NodeListField) || field.is_a?(Prism::Template::ConstantListField) -%> (<%= field.name %>.length == other.<%= field.name %>.length) && diff --git a/templates/lib/prism/reflection.rb.erb b/templates/lib/prism/reflection.rb.erb index 62449a47a2..89c0bc6ce2 100644 --- a/templates/lib/prism/reflection.rb.erb +++ b/templates/lib/prism/reflection.rb.erb @@ -59,15 +59,21 @@ module Prism class LocationField < Field end + # A slice field represents a slice of the source code. It resolves to a + # Prism::Location in Ruby. + class SliceField < Field + end + # An optional location field represents the location of some part of the # node in the source code that may or may not be present. It resolves to # either a Prism::Location or nil in Ruby. class OptionalLocationField < Field end - # A slice field represents a slice of the source code. It resolves to a - # Prism::Location in Ruby. - class SliceField < Field + # An optional slice field represents a slice of the source code that may or + # may not be present. It resolves to either a Prism::Location or nil in + # Ruby. + class OptionalSliceField < Field end # An integer field represents an integer value. It is used to represent the @@ -124,6 +130,8 @@ module Prism "SliceField.new(:#{field.name})" when Prism::Template::OptionalLocationField "OptionalLocationField.new(:#{field.name})" + when Prism::Template::OptionalSliceField + "OptionalSliceField.new(:#{field.name})" when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField "IntegerField.new(:#{field.name})" when Prism::Template::DoubleField diff --git a/templates/lib/prism/serialize.rb.erb b/templates/lib/prism/serialize.rb.erb index 4dad1260a9..b9003d276c 100644 --- a/templates/lib/prism/serialize.rb.erb +++ b/templates/lib/prism/serialize.rb.erb @@ -530,7 +530,7 @@ module Prism when Prism::Template::OptionalConstantField then "load_optional_constant(constant_pool, encoding)" when Prism::Template::ConstantListField then "Array.new(load_varuint) { load_constant(constant_pool, encoding) }.tap { |constants| constants.freeze if freeze }" when Prism::Template::LocationField, Prism::Template::SliceField then "load_location(freeze)" - when Prism::Template::OptionalLocationField then "load_optional_location(freeze)" + when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "load_optional_location(freeze)" when Prism::Template::UInt8Field then "io.getbyte" when Prism::Template::UInt32Field then "load_varuint" when Prism::Template::IntegerField then "load_integer" @@ -569,7 +569,7 @@ module Prism when Prism::Template::OptionalConstantField then "load_optional_constant(constant_pool, encoding)" when Prism::Template::ConstantListField then "Array.new(load_varuint) { load_constant(constant_pool, encoding) }" when Prism::Template::LocationField, Prism::Template::SliceField then "load_location(freeze)" - when Prism::Template::OptionalLocationField then "load_optional_location(freeze)" + when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "load_optional_location(freeze)" when Prism::Template::UInt8Field then "io.getbyte" when Prism::Template::UInt32Field then "load_varuint" when Prism::Template::IntegerField then "load_integer" diff --git a/templates/rbi/prism/dsl.rbi.erb b/templates/rbi/prism/dsl.rbi.erb index 4b577482a3..7fcae1fada 100644 --- a/templates/rbi/prism/dsl.rbi.erb +++ b/templates/rbi/prism/dsl.rbi.erb @@ -34,7 +34,7 @@ module Prism::DSL [field.name, "\"\"", field.rbi_class] when Prism::Template::LocationField, Prism::Template::SliceField [field.name, "location", field.rbi_class] - when Prism::Template::OptionalLocationField + when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField [field.name, "nil", field.rbi_class] when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField [field.name, "0", field.rbi_class] diff --git a/templates/rbi/prism/node.rbi.erb b/templates/rbi/prism/node.rbi.erb index d408929629..3cc7436900 100644 --- a/templates/rbi/prism/node.rbi.erb +++ b/templates/rbi/prism/node.rbi.erb @@ -122,7 +122,7 @@ class Prism::<%= node.name -%> < Prism::Node sig { returns(String) } def <%= field.name.delete_suffix("_loc") %>; end - <%- when Prism::Template::OptionalLocationField -%> + <%- when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> diff --git a/templates/sig/prism/node.rbs.erb b/templates/sig/prism/node.rbs.erb index b3b2a36d15..693428bd0d 100644 --- a/templates/sig/prism/node.rbs.erb +++ b/templates/sig/prism/node.rbs.erb @@ -86,7 +86,7 @@ module Prism <%- node.fields.each do |field| -%> attr_reader <%= field.name %>: <%= field.rbs_class %> <%- end -%> - <%- if (locations = node.fields.select { |field| field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::SliceField) || field.is_a?(Prism::Template::OptionalLocationField) }) -%> + <%- if (locations = node.fields.select { |field| field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::SliceField) || field.is_a?(Prism::Template::OptionalLocationField) || field.is_a?(Prism::Template::OptionalSliceField) }) -%> <%- locations.each do |field| -%> def save_<%= field.name %>: (_Repository repository) -> void @@ -102,7 +102,7 @@ module Prism <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> def <%= field.name.delete_suffix("_loc") %>: () -> String - <%- when Prism::Template::OptionalLocationField -%> + <%- when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> def <%= field.name.delete_suffix("_loc") %>: () -> String? diff --git a/templates/src/node.c.erb b/templates/src/node.c.erb index bb097dd7d6..fba7a11d7c 100644 --- a/templates/src/node.c.erb +++ b/templates/src/node.c.erb @@ -108,12 +108,12 @@ pm_node_destroy(pm_parser_t *parser, pm_node_t *node) { <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "prism/templates/src/<%= File.basename(__FILE__) %>" case <%= node.type %>: { - <%- if node.fields.any? { |field| ![Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField].include?(field.class) } -%> + <%- if node.fields.any? { |field| ![Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField].include?(field.class) } -%> pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node; <%- end -%> <%- node.fields.each do |field| -%> <%- case field -%> - <%- when Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField -%> + <%- when Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField -%> <%- when Prism::Template::NodeField -%> pm_node_destroy(parser, (pm_node_t *)cast-><%= field.name %>); <%- when Prism::Template::OptionalNodeField -%> @@ -304,6 +304,12 @@ pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *no } else { pm_buffer_append_string(buffer, "null", 4); } + <%- when Prism::Template::OptionalSliceField -%> + if (cast-><%= field.name %>.length != 0) { + pm_dump_json_slice(buffer, &cast-><%= field.name %>); + } else { + pm_buffer_append_string(buffer, "null", 4); + } <%- when Prism::Template::UInt8Field -%> pm_buffer_append_format(buffer, "%" PRIu8, cast-><%= field.name %>); <%- when Prism::Template::UInt32Field -%> diff --git a/templates/src/prettyprint.c.erb b/templates/src/prettyprint.c.erb index 136c6d25b9..2d4e11f9f9 100644 --- a/templates/src/prettyprint.c.erb +++ b/templates/src/prettyprint.c.erb @@ -133,6 +133,17 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm pm_buffer_append_source(output_buffer, location->start, (size_t) (location->end - location->start), PM_BUFFER_ESCAPING_RUBY); pm_buffer_append_string(output_buffer, "\"\n", 2); } + <%- when Prism::Template::OptionalSliceField -%> + pm_slice_t *slice = &cast-><%= field.name %>; + if (slice->length == 0) { + pm_buffer_append_string(output_buffer, " nil\n", 5); + } else { + pm_buffer_append_byte(output_buffer, ' '); + prettyprint_slice(output_buffer, parser, slice); + pm_buffer_append_string(output_buffer, " = \"", 4); + pm_buffer_append_source(output_buffer, parser->start + slice->start, (size_t) slice->length, PM_BUFFER_ESCAPING_RUBY); + pm_buffer_append_string(output_buffer, "\"\n", 2); + } <%- when Prism::Template::UInt8Field -%> pm_buffer_append_format(output_buffer, " %" PRIu8 "\n", cast-><%= field.name %>); <%- when Prism::Template::UInt32Field -%> diff --git a/templates/src/serialize.c.erb b/templates/src/serialize.c.erb index aa0b8837dc..82751d3821 100644 --- a/templates/src/serialize.c.erb +++ b/templates/src/serialize.c.erb @@ -144,6 +144,15 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); } <%- end -%> + <%- when Prism::Template::OptionalSliceField -%> + <%- if field.should_be_serialized? -%> + if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.length == 0) { + pm_buffer_append_byte(buffer, 0); + } else { + pm_buffer_append_byte(buffer, 1); + pm_serialize_slice(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + } + <%- end -%> <%- when Prism::Template::UInt8Field -%> pm_buffer_append_byte(buffer, ((pm_<%= node.human %>_t *)node)-><%= field.name %>); <%- when Prism::Template::UInt32Field -%> diff --git a/templates/template.rb b/templates/template.rb index fcaf0ca4a9..1ea2f3d3d5 100755 --- a/templates/template.rb +++ b/templates/template.rb @@ -317,6 +317,27 @@ def java_type end end + # This represents the same information as a location field, except it is + # represented in the source using two 32-bit integers, as opposed to two + # pointers. + class SliceField < Field + def semantic_field? + false + end + + def rbs_class + "Location" + end + + def rbi_class + "Prism::Location" + end + + def java_type + "Location" + end + end + # This represents a field on a node that is a location that is optional. class OptionalLocationField < Field def semantic_field? @@ -336,6 +357,25 @@ def java_type end end + # This represents a field on a node that is a slice that is optional. + class OptionalSliceField < Field + def semantic_field? + false + end + + def rbs_class + "Location?" + end + + def rbi_class + "T.nilable(Prism::Location)" + end + + def java_type + "Location" + end + end + # This represents an integer field. class UInt8Field < Field def rbs_class @@ -398,27 +438,6 @@ def java_type end end - # This represents the same information as a location field, except it is - # represented in the source using two 32-bit integers, as opposed to two - # pointers. - class SliceField < Field - def semantic_field? - false - end - - def rbs_class - "Location" - end - - def rbi_class - "Prism::Location" - end - - def java_type - "Location" - end - end - # This class represents a node in the tree, configured by the config.yml file # in YAML format. It contains information about the name of the node and the # various child nodes it contains. @@ -515,6 +534,7 @@ def field_type_for(name) when "integer" then IntegerField when "double" then DoubleField when "slice" then SliceField + when "slice?" then OptionalSliceField else raise("Unknown field type: #{name.inspect}") end end From 31ab53859a9262eef98e3c24d4da67449a4e9c9b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sun, 30 Nov 2025 22:01:08 -0500 Subject: [PATCH 03/22] Use slices for more internal locations --- config.yml | 454 +++--- lib/prism/translation/parser/compiler.rb | 2 +- lib/prism/translation/parser/lexer.rb | 1 - lib/prism/translation/ripper.rb | 13 +- snapshots/seattlerb/symbol_empty.txt | 2 +- snapshots/spanning_heredoc_newlines.txt | 2 +- snapshots/unparser/corpus/literal/literal.txt | 2 +- src/prism.c | 1325 ++++++++--------- templates/src/token_type.c.erb | 4 - test/prism/errors_test.rb | 6 +- 10 files changed, 850 insertions(+), 961 deletions(-) diff --git a/config.yml b/config.yml index 6b052c64b3..9b7360e31a 100644 --- a/config.yml +++ b/config.yml @@ -653,10 +653,6 @@ tokens: comment: "a separator between words in a list" - name: __END__ comment: "marker for the point in the file at which the parser should stop" - - name: MISSING - comment: "a token that was expected but not found" - - name: NOT_PROVIDED - comment: "a token that was not present but it is okay" flags: - name: ArgumentsNodeFlags values: @@ -867,7 +863,7 @@ nodes: alias :"#{foo}" :"#{bar}" ^^^^^^^^^ - name: keyword_loc - type: location + type: slice comment: | Represents the location of the `alias` keyword. @@ -897,7 +893,7 @@ nodes: foo => bar | baz ^^^ - name: operator_loc - type: location + type: slice comment: | Represents the alternation operator location. @@ -933,7 +929,7 @@ nodes: 1 and 2 ^ - name: operator_loc - type: location + type: slice comment: | The location of the `and` keyword or the `&&` operator. @@ -1033,14 +1029,14 @@ nodes: foo in *bar, baz ^^^ - name: opening_loc - type: location? + type: slice? comment: | Represents the opening location of the array pattern. foo in [1, 2] ^ - name: closing_loc - type: location? + type: slice? comment: | Represents the closing location of the array pattern. @@ -1091,7 +1087,7 @@ nodes: { x: 1 } ^ - name: operator_loc - type: location? + type: slice? comment: | The location of the `=>` operator, if present. @@ -1113,7 +1109,7 @@ nodes: { **foo } ^^^ - name: operator_loc - type: location + type: slice comment: | The location of the `**` operator. @@ -1142,7 +1138,7 @@ nodes: - name: BeginNode fields: - name: begin_keyword_loc - type: location? + type: slice? comment: | Represents the location of the `begin` keyword. @@ -1181,7 +1177,7 @@ nodes: begin x; ensure y; end ^^^^^^^^ - name: end_keyword_loc - type: location? + type: slice? comment: | Represents the location of the `end` keyword. @@ -1206,7 +1202,7 @@ nodes: foo(&args) ^^^^^ - name: operator_loc - type: location + type: slice comment: | Represents the location of the `&` operator. @@ -1267,14 +1263,14 @@ nodes: [1, 2, 3].each { |i| puts x } ^^^^^^ - name: opening_loc - type: location + type: slice comment: | Represents the location of the opening `|`. [1, 2, 3].each { |i| puts x } ^ - name: closing_loc - type: location + type: slice comment: | Represents the location of the closing `|`. @@ -1297,14 +1293,14 @@ nodes: ^ end - name: name_loc - type: location? + type: slice? comment: | Represents the location of the block parameter name. def a(&b) ^ - name: operator_loc - type: location + type: slice comment: | Represents the location of the `&` operator. @@ -1344,7 +1340,7 @@ nodes: ^^^^^ end - name: opening_loc - type: location? + type: slice? comment: | Represents the opening location of the block parameters. @@ -1355,7 +1351,7 @@ nodes: ^ end - name: closing_loc - type: location? + type: slice? comment: | Represents the closing location of the block parameters. @@ -1385,7 +1381,7 @@ nodes: break foo ^^^ - name: keyword_loc - type: location + type: slice comment: | The location of the `break` keyword. @@ -1408,14 +1404,14 @@ nodes: foo.bar &&= value ^^^ - name: call_operator_loc - type: location? + type: slice? comment: | Represents the location of the call operator. foo.bar &&= value ^ - name: message_loc - type: location? + type: slice? comment: | Represents the location of the message. @@ -1436,7 +1432,7 @@ nodes: foo.bar &&= value # write_name `:bar=` ^^^ - name: operator_loc - type: location + type: slice comment: | Represents the location of the operator. @@ -1473,7 +1469,7 @@ nodes: foo + bar ^^^ - name: call_operator_loc - type: location? + type: slice? comment: | Represents the location of the call operator. @@ -1490,14 +1486,14 @@ nodes: foo.bar # name `:foo` ^^^ - name: message_loc - type: location? + type: slice? comment: | Represents the location of the message. foo.bar ^^^ - name: opening_loc - type: location? + type: slice? comment: | Represents the location of the left parenthesis. foo(bar) @@ -1511,14 +1507,14 @@ nodes: foo(bar) ^^^ - name: closing_loc - type: location? + type: slice? comment: | Represents the location of the right parenthesis. foo(bar) ^ - name: equal_loc - type: location? + type: slice? comment: | Represents the location of the equal sign, in the case that this is an attribute write. @@ -1569,14 +1565,14 @@ nodes: foo.bar += value ^^^ - name: call_operator_loc - type: location? + type: slice? comment: | Represents the location of the call operator. foo.bar += value ^ - name: message_loc - type: location? + type: slice? comment: | Represents the location of the message. @@ -1604,7 +1600,7 @@ nodes: foo.bar += value # binary_operator `:+` ^ - name: binary_operator_loc - type: location + type: slice comment: | Represents the location of the binary operator. @@ -1635,14 +1631,14 @@ nodes: foo.bar ||= value ^^^ - name: call_operator_loc - type: location? + type: slice? comment: | Represents the location of the call operator. foo.bar ||= value ^ - name: message_loc - type: location? + type: slice? comment: | Represents the location of the message. @@ -1663,7 +1659,7 @@ nodes: foo.bar ||= value # write_name `:bar=` ^^^ - name: operator_loc - type: location + type: slice comment: | Represents the location of the operator. @@ -1694,7 +1690,7 @@ nodes: foo.bar = 1 ^^^ - name: call_operator_loc - type: location + type: slice comment: | Represents the location of the call operator. @@ -1708,7 +1704,7 @@ nodes: foo.bar = 1 # name `:foo` ^^^ - name: message_loc - type: location + type: slice comment: | Represents the location of the message. @@ -1746,7 +1742,7 @@ nodes: foo => bar ^^^ - name: operator_loc - type: location + type: slice comment: | Represents the location of the `=>` operator. @@ -1784,14 +1780,14 @@ nodes: case true; in false; else; end ^^^^ - name: case_keyword_loc - type: location + type: slice comment: | Represents the location of the `case` keyword. case true; in false; end ^^^^ - name: end_keyword_loc - type: location + type: slice comment: | Represents the location of the `end` keyword. @@ -1831,14 +1827,14 @@ nodes: case true; when false; else; end ^^^^ - name: case_keyword_loc - type: location + type: slice comment: | Represents the location of the `case` keyword. case true; when false; end ^^^^ - name: end_keyword_loc - type: location + type: slice comment: | Represents the location of the `end` keyword. @@ -1856,7 +1852,7 @@ nodes: - name: locals type: constant[] - name: class_keyword_loc - type: location + type: slice comment: | Represents the location of the `class` keyword. @@ -1869,7 +1865,7 @@ nodes: - ConstantPathNode - on error: CallNode # class 0.X end - name: inheritance_operator_loc - type: location? + type: slice? comment: | Represents the location of the `<` operator. @@ -1895,7 +1891,7 @@ nodes: foo ^^^ - name: end_keyword_loc - type: location + type: slice comment: | Represents the location of the `end` keyword. @@ -1922,14 +1918,14 @@ nodes: @@target &&= value # name `:@@target` ^^^^^^^^ - name: name_loc - type: location + type: slice comment: | Represents the location of the variable name. @@target &&= value ^^^^^^^^ - name: operator_loc - type: location + type: slice comment: | Represents the location of the `&&=` operator. @@ -1953,9 +1949,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: binary_operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -1971,9 +1967,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -2017,7 +2013,7 @@ nodes: @@_test = :test # name `@@_test` - name: name_loc - type: location + type: slice comment: | The location of the variable name. @@ -2035,7 +2031,7 @@ nodes: @@_xyz = 123 ^^^ - name: operator_loc - type: location + type: slice comment: | The location of the `=` operator. @@ -2051,9 +2047,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -2067,9 +2063,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: binary_operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -2085,9 +2081,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -2102,7 +2098,7 @@ nodes: type: node kind: ConstantPathNode - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -2131,7 +2127,7 @@ nodes: type: constant? comment: The name of the constant being accessed. This could be `nil` in the event of a syntax error. - name: delimiter_loc - type: location + type: slice comment: | The location of the `::` delimiter. @@ -2141,7 +2137,7 @@ nodes: One::Two ^^ - name: name_loc - type: location + type: slice comment: | The location of the name of the constant. @@ -2161,7 +2157,7 @@ nodes: type: node kind: ConstantPathNode - name: binary_operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -2178,7 +2174,7 @@ nodes: type: node kind: ConstantPathNode - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -2195,9 +2191,9 @@ nodes: - name: name type: constant? - name: delimiter_loc - type: location + type: slice - name: name_loc - type: location + type: slice comment: | Represents writing to a constant path in a context that doesn't have an explicit value. @@ -2217,7 +2213,7 @@ nodes: ::Foo = :abc ^^^^^ - name: operator_loc - type: location + type: slice comment: | The location of the `=` operator. @@ -2277,7 +2273,7 @@ nodes: XYZ = 1 # name `:XYZ` - name: name_loc - type: location + type: slice comment: | The location of the constant name. @@ -2295,7 +2291,7 @@ nodes: MyClass = Class.new ^^^^^^^^^ - name: operator_loc - type: location + type: slice comment: | The location of the `=` operator. @@ -2311,7 +2307,7 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: receiver type: node? kind: non-void expression @@ -2326,17 +2322,17 @@ nodes: - name: locals type: constant[] - name: def_keyword_loc - type: location + type: slice - name: operator_loc - type: location? + type: slice? - name: lparen_loc - type: location? + type: slice? - name: rparen_loc - type: location? + type: slice? - name: equal_loc - type: location? + type: slice? - name: end_keyword_loc - type: location? + type: slice? comment: | Represents a method definition. @@ -2346,14 +2342,14 @@ nodes: - name: DefinedNode fields: - name: lparen_loc - type: location? + type: slice? - name: value type: node kind: Node # More than non-void expression as defined?(return) is allowed, yet defined?(BEGIN{}) is SyntaxError - name: rparen_loc - type: location? + type: slice? - name: keyword_loc - type: location + type: slice comment: | Represents the use of the `defined?` keyword. @@ -2362,12 +2358,12 @@ nodes: - name: ElseNode fields: - name: else_keyword_loc - type: location + type: slice - name: statements type: node? kind: StatementsNode - name: end_keyword_loc - type: location? + type: slice? comment: | Represents an `else` clause in a `case`, `if`, or `unless` statement. @@ -2376,12 +2372,12 @@ nodes: - name: EmbeddedStatementsNode fields: - name: opening_loc - type: location + type: slice - name: statements type: node? kind: StatementsNode - name: closing_loc - type: location + type: slice comment: | Represents an interpolated set of statements. @@ -2390,7 +2386,7 @@ nodes: - name: EmbeddedVariableNode fields: - name: operator_loc - type: location + type: slice - name: variable type: node kind: @@ -2407,12 +2403,12 @@ nodes: - name: EnsureNode fields: - name: ensure_keyword_loc - type: location + type: slice - name: statements type: node? kind: StatementsNode - name: end_keyword_loc - type: location + type: slice comment: | Represents an `ensure` clause in a `begin` statement. @@ -2476,7 +2472,7 @@ nodes: foo in Foo(*bar, baz, *qux) ^^^^ - name: opening_loc - type: location? + type: slice? comment: | The location of the opening brace. @@ -2486,7 +2482,7 @@ nodes: foo in Foo(*bar, baz, *qux) ^ - name: closing_loc - type: location? + type: slice? comment: | The location of the closing brace. @@ -2519,7 +2515,7 @@ nodes: type: node? kind: non-void expression - name: operator_loc - type: location + type: slice comment: | Represents the use of the `..` or `...` operators to create flip flops. @@ -2576,28 +2572,28 @@ nodes: ^^^^^^ end - name: for_keyword_loc - type: location + type: slice comment: | The location of the `for` keyword. for i in a end ^^^ - name: in_keyword_loc - type: location + type: slice comment: | The location of the `in` keyword. for i in a end ^^ - name: do_keyword_loc - type: location? + type: slice? comment: | The location of the `do` keyword, if present. for i in a do end ^^ - name: end_keyword_loc - type: location + type: slice comment: | The location of the `end` keyword. @@ -2645,9 +2641,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -2661,9 +2657,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: binary_operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -2679,9 +2675,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -2725,7 +2721,7 @@ nodes: $_Test = 123 # name `:$_Test` - name: name_loc - type: location + type: slice comment: | The location of the global variable's name. @@ -2743,7 +2739,7 @@ nodes: $-xyz = 123 ^^^ - name: operator_loc - type: location + type: slice comment: | The location of the `=` operator. @@ -2757,7 +2753,7 @@ nodes: - name: HashNode fields: - name: opening_loc - type: location + type: slice comment: | The location of the opening brace. @@ -2777,7 +2773,7 @@ nodes: { **foo } ^^^^^ - name: closing_loc - type: location + type: slice comment: | The location of the closing brace. @@ -2828,7 +2824,7 @@ nodes: foo => { a: 1, b:, **nil } ^^^^^ - name: opening_loc - type: location? + type: slice? comment: | The location of the opening brace. @@ -2838,7 +2834,7 @@ nodes: foo => Bar[a: 1] ^ - name: closing_loc - type: location? + type: slice? comment: | The location of the closing brace. @@ -2864,7 +2860,7 @@ nodes: - name: IfNode fields: - name: if_keyword_loc - type: location? + type: slice? comment: | The location of the `if` keyword if present. @@ -2889,7 +2885,7 @@ nodes: foo ? bar : baz ^^^ - name: then_keyword_loc - type: location? + type: slice? comment: | The location of the `then` keyword (if present) or the `?` in a ternary expression, `nil` otherwise. @@ -2930,7 +2926,7 @@ nodes: if foo then bar else baz end ^^^^^^^^^^^^ - name: end_keyword_loc - type: location? + type: slice? comment: | The location of the `end` keyword if present, `nil` otherwise. @@ -3007,9 +3003,9 @@ nodes: type: node? kind: StatementsNode - name: in_loc - type: location + type: slice - name: then_loc - type: location? + type: slice? comment: | Represents the use of the `in` keyword in a case statement. @@ -3022,19 +3018,19 @@ nodes: type: node? kind: non-void expression - name: call_operator_loc - type: location? + type: slice? - name: opening_loc - type: location + type: slice - name: arguments type: node? kind: ArgumentsNode - name: closing_loc - type: location + type: slice - name: block type: node? kind: BlockArgumentNode # foo[&b] &&= value, only valid on Ruby < 3.4 - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -3050,21 +3046,21 @@ nodes: type: node? kind: non-void expression - name: call_operator_loc - type: location? + type: slice? - name: opening_loc - type: location + type: slice - name: arguments type: node? kind: ArgumentsNode - name: closing_loc - type: location + type: slice - name: block type: node? kind: BlockArgumentNode # foo[&b] += value, only valid on Ruby < 3.4 - name: binary_operator type: constant - name: binary_operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -3080,19 +3076,19 @@ nodes: type: node? kind: non-void expression - name: call_operator_loc - type: location? + type: slice? - name: opening_loc - type: location + type: slice - name: arguments type: node? kind: ArgumentsNode - name: closing_loc - type: location + type: slice - name: block type: node? kind: BlockArgumentNode # foo[&b] ||= value, only valid on Ruby < 3.4 - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -3108,12 +3104,12 @@ nodes: type: node kind: non-void expression - name: opening_loc - type: location + type: slice - name: arguments type: node? kind: ArgumentsNode - name: closing_loc - type: location + type: slice - name: block type: node? kind: BlockArgumentNode # foo[&b], = 1, only valid on Ruby < 3.4 @@ -3135,9 +3131,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -3151,9 +3147,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: binary_operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -3169,9 +3165,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -3215,7 +3211,7 @@ nodes: @_foo = "bar" # name `@_foo` - name: name_loc - type: location + type: slice comment: | The location of the variable name. @@ -3233,7 +3229,7 @@ nodes: @_x = 1234 ^^^^ - name: operator_loc - type: location + type: slice comment: | The location of the `=` operator. @@ -3259,7 +3255,7 @@ nodes: flags: RegularExpressionFlags fields: - name: opening_loc - type: location + type: slice - name: parts type: node[] kind: @@ -3267,7 +3263,7 @@ nodes: - EmbeddedStatementsNode - EmbeddedVariableNode - name: closing_loc - type: location + type: slice newline: parts comment: | Represents a regular expression literal that contains interpolation that is being used in the predicate of a conditional to implicitly match against the last line read by an IO object. @@ -3278,7 +3274,7 @@ nodes: flags: RegularExpressionFlags fields: - name: opening_loc - type: location + type: slice - name: parts type: node[] kind: @@ -3286,7 +3282,7 @@ nodes: - EmbeddedStatementsNode - EmbeddedVariableNode - name: closing_loc - type: location + type: slice newline: parts comment: | Represents a regular expression literal that contains interpolation. @@ -3297,7 +3293,7 @@ nodes: flags: InterpolatedStringNodeFlags fields: - name: opening_loc - type: location? + type: slice? - name: parts type: node[] kind: @@ -3310,7 +3306,7 @@ nodes: - on error: SymbolNode - on error: InterpolatedSymbolNode - name: closing_loc - type: location? + type: slice? newline: parts comment: | Represents a string literal that contains interpolation. @@ -3320,7 +3316,7 @@ nodes: - name: InterpolatedSymbolNode fields: - name: opening_loc - type: location? + type: slice? - name: parts type: node[] kind: @@ -3328,7 +3324,7 @@ nodes: - EmbeddedStatementsNode - EmbeddedVariableNode - name: closing_loc - type: location? + type: slice? newline: parts comment: | Represents a symbol literal that contains interpolation. @@ -3338,7 +3334,7 @@ nodes: - name: InterpolatedXStringNode fields: - name: opening_loc - type: location + type: slice - name: parts type: node[] kind: @@ -3346,7 +3342,7 @@ nodes: - EmbeddedStatementsNode - EmbeddedVariableNode - name: closing_loc - type: location + type: slice newline: parts comment: | Represents an xstring literal that contains interpolation. @@ -3384,9 +3380,9 @@ nodes: - name: name type: constant? - name: name_loc - type: location? + type: slice? - name: operator_loc - type: location + type: slice comment: | Represents a keyword rest parameter to a method, block, or lambda definition. @@ -3398,11 +3394,11 @@ nodes: - name: locals type: constant[] - name: operator_loc - type: location + type: slice - name: opening_loc - type: location + type: slice - name: closing_loc - type: location + type: slice - name: parameters type: node? kind: @@ -3422,9 +3418,9 @@ nodes: - name: LocalVariableAndWriteNode fields: - name: name_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -3440,9 +3436,9 @@ nodes: - name: LocalVariableOperatorWriteNode fields: - name: name_loc - type: location + type: slice - name: binary_operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -3460,9 +3456,9 @@ nodes: - name: LocalVariableOrWriteNode fields: - name: name_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -3540,7 +3536,7 @@ nodes: The specific rules for calculating the depth may differ from individual Ruby implementations, as they are not specified by the language. For more information, see [the Prism documentation](https://github.com/ruby/prism/blob/main/docs/local_variable_depth.md). - name: name_loc - type: location + type: slice comment: | The location of the variable name. @@ -3562,7 +3558,7 @@ nodes: foo = foo - name: operator_loc - type: location + type: slice comment: | The location of the `=` operator. @@ -3577,11 +3573,11 @@ nodes: flags: RegularExpressionFlags fields: - name: opening_loc - type: location + type: slice - name: content_loc - type: location + type: slice - name: closing_loc - type: location + type: slice - name: unescaped type: string comment: | @@ -3598,7 +3594,7 @@ nodes: type: node kind: pattern expression - name: operator_loc - type: location + type: slice comment: | Represents the use of the modifier `in` operator. @@ -3662,7 +3658,7 @@ nodes: foo => CONST - name: operator_loc - type: location + type: slice comment: | The location of the operator. @@ -3694,7 +3690,7 @@ nodes: - name: locals type: constant[] - name: module_keyword_loc - type: location + type: slice - name: constant_path type: node kind: @@ -3707,7 +3703,7 @@ nodes: - StatementsNode - BeginNode - name: end_keyword_loc - type: location + type: slice - name: name type: constant comment: | @@ -3783,14 +3779,14 @@ nodes: a, (*, b, c) = 1, 2, 3, 4, 5 ^^^^ - name: lparen_loc - type: location? + type: slice? comment: | The location of the opening parenthesis. a, (b, c) = 1, 2, 3 ^ - name: rparen_loc - type: location? + type: slice? comment: | The location of the closing parenthesis. @@ -3872,21 +3868,21 @@ nodes: a, *, b, c = 1, 2, 3, 4, 5 ^^^^ - name: lparen_loc - type: location? + type: slice? comment: | The location of the opening parenthesis. (a, b, c) = 1, 2, 3 ^ - name: rparen_loc - type: location? + type: slice? comment: | The location of the closing parenthesis. (a, b, c) = 1, 2, 3 ^ - name: operator_loc - type: location + type: slice comment: | The location of the operator. @@ -3911,7 +3907,7 @@ nodes: type: node? kind: ArgumentsNode - name: keyword_loc - type: location + type: slice comment: | Represents the use of the `next` keyword. @@ -3926,9 +3922,9 @@ nodes: - name: NoKeywordsParameterNode fields: - name: operator_loc - type: location + type: slice - name: keyword_loc - type: location + type: slice comment: | Represents the use of `**nil` inside method arguments. @@ -3967,7 +3963,7 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -3983,9 +3979,9 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: value type: node kind: non-void expression @@ -4020,7 +4016,7 @@ nodes: 1 or 2 ^ - name: operator_loc - type: location + type: slice comment: | The location of the `or` keyword or the `||` operator. @@ -4083,9 +4079,9 @@ nodes: type: node? kind: non-void expression # Usually a StatementsNode but not always e.g. `1 in (..10)` - name: opening_loc - type: location + type: slice - name: closing_loc - type: location + type: slice newline: false comment: | Represents a parenthesized expression @@ -4103,21 +4099,21 @@ nodes: foo in ^(bar) ^^^ - name: operator_loc - type: location + type: slice comment: | The location of the `^` operator foo in ^(bar) ^ - name: lparen_loc - type: location + type: slice comment: | The location of the opening parenthesis. foo in ^(bar) ^ - name: rparen_loc - type: location + type: slice comment: | The location of the closing parenthesis. @@ -4147,7 +4143,7 @@ nodes: foo in ^bar ^^^ - name: operator_loc - type: location + type: slice comment: | The location of the `^` operator @@ -4164,11 +4160,11 @@ nodes: type: node? kind: StatementsNode - name: keyword_loc - type: location + type: slice - name: opening_loc - type: location + type: slice - name: closing_loc - type: location + type: slice comment: | Represents the use of the `END` keyword. @@ -4180,11 +4176,11 @@ nodes: type: node? kind: StatementsNode - name: keyword_loc - type: location + type: slice - name: opening_loc - type: location + type: slice - name: closing_loc - type: location + type: slice comment: | Represents the use of the `BEGIN` keyword. @@ -4225,7 +4221,7 @@ nodes: ^^^ If neither right-hand or left-hand side was included, this will be a MissingNode. - name: operator_loc - type: location + type: slice comment: | The location of the `..` or `...` operator. comment: | @@ -4266,11 +4262,11 @@ nodes: flags: RegularExpressionFlags fields: - name: opening_loc - type: location + type: slice - name: content_loc - type: location + type: slice - name: closing_loc - type: location + type: slice - name: unescaped type: string comment: | @@ -4284,7 +4280,7 @@ nodes: - name: name type: constant - name: name_loc - type: location + type: slice comment: | Represents a required keyword parameter to a method, block, or lambda definition. @@ -4308,7 +4304,7 @@ nodes: type: node kind: Node - name: keyword_loc - type: location + type: slice - name: rescue_expression type: node kind: Node @@ -4321,12 +4317,12 @@ nodes: - name: RescueNode fields: - name: keyword_loc - type: location + type: slice - name: exceptions type: node[] kind: non-void expression - name: operator_loc - type: location? + type: slice? - name: reference type: node? kind: @@ -4342,7 +4338,7 @@ nodes: - on error: NumberedReferenceReadNode # => begin; rescue => $1; end - on error: MissingNode # begin; rescue =>; end - name: then_keyword_loc - type: location? + type: slice? - name: statements type: node? kind: StatementsNode @@ -4365,9 +4361,9 @@ nodes: - name: name type: constant? - name: name_loc - type: location? + type: slice? - name: operator_loc - type: location + type: slice comment: | Represents a rest parameter to a method, block, or lambda definition. @@ -4383,7 +4379,7 @@ nodes: - name: ReturnNode fields: - name: keyword_loc - type: location + type: slice - name: arguments type: node? kind: ArgumentsNode @@ -4424,9 +4420,9 @@ nodes: - name: locals type: constant[] - name: class_keyword_loc - type: location + type: slice - name: operator_loc - type: location + type: slice - name: expression type: node kind: non-void expression @@ -4436,7 +4432,7 @@ nodes: - StatementsNode - BeginNode - name: end_keyword_loc - type: location + type: slice comment: | Represents a singleton class declaration involving the `class` keyword. @@ -4468,7 +4464,7 @@ nodes: - name: SplatNode fields: - name: operator_loc - type: location + type: slice - name: expression type: node? kind: non-void expression @@ -4491,11 +4487,11 @@ nodes: flags: StringFlags fields: - name: opening_loc - type: location? + type: slice? - name: content_loc - type: location + type: slice - name: closing_loc - type: location? + type: slice? - name: unescaped type: string comment: | @@ -4512,15 +4508,15 @@ nodes: - name: SuperNode fields: - name: keyword_loc - type: location + type: slice - name: lparen_loc - type: location? + type: slice? - name: arguments type: node? kind: ArgumentsNode comment: "Can be only `nil` when there are empty parentheses, like `super()`." - name: rparen_loc - type: location? + type: slice? - name: block type: node? kind: @@ -4540,11 +4536,11 @@ nodes: flags: SymbolFlags fields: - name: opening_loc - type: location? + type: slice? - name: value_loc - type: location? + type: slice? - name: closing_loc - type: location? + type: slice? - name: unescaped type: string comment: | @@ -4569,7 +4565,7 @@ nodes: - SymbolNode - InterpolatedSymbolNode - name: keyword_loc - type: location + type: slice comment: | Represents the use of the `undef` keyword. @@ -4578,7 +4574,7 @@ nodes: - name: UnlessNode fields: - name: keyword_loc - type: location + type: slice comment: | The location of the `unless` keyword. @@ -4599,7 +4595,7 @@ nodes: bar unless cond ^^^^ - name: then_keyword_loc - type: location? + type: slice? comment: | The location of the `then` keyword, if present. @@ -4623,7 +4619,7 @@ nodes: unless cond then bar else baz end ^^^^^^^^ - name: end_keyword_loc - type: location? + type: slice? comment: | The location of the `end` keyword, if present. @@ -4642,11 +4638,11 @@ nodes: flags: LoopFlags fields: - name: keyword_loc - type: location + type: slice - name: do_keyword_loc - type: location? + type: slice? - name: closing_loc - type: location? + type: slice? - name: predicate type: node kind: non-void expression @@ -4665,12 +4661,12 @@ nodes: - name: WhenNode fields: - name: keyword_loc - type: location + type: slice - name: conditions type: node[] kind: non-void expression - name: then_keyword_loc - type: location? + type: slice? - name: statements type: node? kind: StatementsNode @@ -4685,11 +4681,11 @@ nodes: flags: LoopFlags fields: - name: keyword_loc - type: location + type: slice - name: do_keyword_loc - type: location? + type: slice? - name: closing_loc - type: location? + type: slice? - name: predicate type: node kind: non-void expression @@ -4709,11 +4705,11 @@ nodes: flags: EncodingFlags fields: - name: opening_loc - type: location + type: slice - name: content_loc - type: location + type: slice - name: closing_loc - type: location + type: slice - name: unescaped type: string comment: | @@ -4724,14 +4720,14 @@ nodes: - name: YieldNode fields: - name: keyword_loc - type: location + type: slice - name: lparen_loc - type: location? + type: slice? - name: arguments type: node? kind: ArgumentsNode - name: rparen_loc - type: location? + type: slice? comment: | Represents the use of the `yield` keyword. diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb index 8805614603..bd3618b162 100644 --- a/lib/prism/translation/parser/compiler.rb +++ b/lib/prism/translation/parser/compiler.rb @@ -1767,7 +1767,7 @@ def visit_symbol_node(node) end else parts = - if node.value == "" + if node.value_loc.nil? [] elsif node.value.include?("\n") string_nodes_from_line_continuations(node.unescaped, node.value, node.value_loc.start_offset, node.opening) diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb index 75c48ef667..265d6c218d 100644 --- a/lib/prism/translation/parser/lexer.rb +++ b/lib/prism/translation/parser/lexer.rb @@ -19,7 +19,6 @@ class Lexer TYPES = { # These tokens should never appear in the output of the lexer. MISSING: nil, - NOT_PROVIDED: nil, EMBDOC_END: nil, EMBDOC_LINE: nil, diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index e488b7c5cf..688efa2e00 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -3106,14 +3106,13 @@ def visit_super_node(node) # :foo # ^^^^ def visit_symbol_node(node) - if (opening = node.opening)&.match?(/^%s|['"]:?$/) + if node.value_loc.nil? + bounds(node.location) + on_dyna_symbol(on_string_content) + elsif (opening = node.opening)&.match?(/^%s|['"]:?$/) bounds(node.value_loc) - content = on_string_content - - if !(value = node.value).empty? - content = on_string_add(content, on_tstring_content(value)) - end - + content = on_string_add(on_string_content, on_tstring_content(node.value)) + bounds(node.location) on_dyna_symbol(content) elsif (closing = node.closing) == ":" bounds(node.location) diff --git a/snapshots/seattlerb/symbol_empty.txt b/snapshots/seattlerb/symbol_empty.txt index 945a0e3a51..917b910891 100644 --- a/snapshots/seattlerb/symbol_empty.txt +++ b/snapshots/seattlerb/symbol_empty.txt @@ -8,6 +8,6 @@ └── @ SymbolNode (location: (1,0)-(1,3)) ├── flags: newline, static_literal, forced_us_ascii_encoding ├── opening_loc: (1,0)-(1,2) = ":'" - ├── value_loc: (1,2)-(1,2) = "" + ├── value_loc: ∅ ├── closing_loc: (1,2)-(1,3) = "'" └── unescaped: "" diff --git a/snapshots/spanning_heredoc_newlines.txt b/snapshots/spanning_heredoc_newlines.txt index 976d5d28dc..c72956cf7e 100644 --- a/snapshots/spanning_heredoc_newlines.txt +++ b/snapshots/spanning_heredoc_newlines.txt @@ -129,7 +129,7 @@ │ │ └── @ SymbolNode (location: (17,4)-(20,0)) │ │ ├── flags: static_literal, forced_us_ascii_encoding │ │ ├── opening_loc: (17,4)-(18,0) = "%s\n" - │ │ ├── value_loc: (18,0)-(18,0) = "" + │ │ ├── value_loc: ∅ │ │ ├── closing_loc: (19,0)-(20,0) = "\n" │ │ └── unescaped: "" │ ├── closing_loc: ∅ diff --git a/snapshots/unparser/corpus/literal/literal.txt b/snapshots/unparser/corpus/literal/literal.txt index 386002b557..eb0bf12d5b 100644 --- a/snapshots/unparser/corpus/literal/literal.txt +++ b/snapshots/unparser/corpus/literal/literal.txt @@ -620,7 +620,7 @@ ├── @ SymbolNode (location: (48,0)-(48,3)) │ ├── flags: newline, static_literal │ ├── opening_loc: (48,0)-(48,2) = ":\"" - │ ├── value_loc: (1,0)-(1,0) = "" + │ ├── value_loc: ∅ │ ├── closing_loc: (48,2)-(48,3) = "\"" │ └── unescaped: "" ├── @ RegularExpressionNode (location: (49,0)-(49,5)) diff --git a/src/prism.c b/src/prism.c index bb5a1b3a4d..2171b61a3f 100644 --- a/src/prism.c +++ b/src/prism.c @@ -1033,15 +1033,6 @@ pm_parser_constant_id_token(pm_parser_t *parser, const pm_token_t *token) { return pm_parser_constant_id_location(parser, token->start, token->end); } -/** - * Retrieve the constant pool id for the given token. If the token is not - * provided, then return 0. - */ -static inline pm_constant_id_t -pm_parser_optional_constant_id_token(pm_parser_t *parser, const pm_token_t *token) { - return token->type == PM_TOKEN_NOT_PROVIDED ? 0 : pm_parser_constant_id_token(parser, token); -} - /** * Check whether or not the given node is value expression. * If the node is value node, it returns NULL. @@ -1211,7 +1202,7 @@ pm_void_statement_check(pm_parser_t *parser, const pm_node_t *node) { break; case PM_CALL_NODE: { const pm_call_node_t *cast = (const pm_call_node_t *) node; - if (cast->call_operator_loc.start != NULL || cast->message_loc.start == NULL) break; + if (cast->call_operator_loc.length > 0 || cast->message_loc.length == 0) break; const pm_constant_t *message = pm_constant_pool_id_to_constant(&parser->constant_pool, cast->name); switch (message->length) { @@ -1564,19 +1555,6 @@ pm_conditional_predicate(pm_parser_t *parser, pm_node_t *node, pm_conditional_pr } } -/** - * In a lot of places in the tree you can have tokens that are not provided but - * that do not cause an error. For example, this happens in a method call - * without parentheses. In these cases we set the token to the "not provided" type. - * For example: - * - * pm_token_t token = not_provided(parser); - */ -static inline pm_token_t -not_provided(pm_parser_t *parser) { - return (pm_token_t) { .type = PM_TOKEN_NOT_PROVIDED, .start = parser->start, .end = parser->start }; -} - /** * This is a special out parameter to the parse_arguments_list function that * includes opening and closing parentheses in addition to the arguments since @@ -1972,6 +1950,10 @@ pm_missing_node_create(pm_parser_t *parser, const uint8_t *start, const uint8_t } #define TOKEN2SLICE(parser_, token_) ((pm_slice_t) { .start = (uint32_t) ((token_)->start - (parser_)->start), .length = (uint32_t) ((token_)->end - (token_)->start) }) +#define MAYBETOKEN2SLICE(parser_, token_) ((token_) == NULL ? ((pm_slice_t) { 0 }) : TOKEN2SLICE(parser_, token_)) +#define MAYBETOKENPTR(token_) ((token_).start == NULL ? NULL : &(token_)) +#define LOCATION2SLICE TOKEN2SLICE +#define MAYBELOCATION2SLICE MAYBETOKEN2SLICE /** * Allocate and initialize a new AliasGlobalVariableNode node. @@ -2003,7 +1985,7 @@ pm_alias_method_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_n .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_ALIAS_METHOD_NODE, 0, keyword, old_name), .new_name = new_name, .old_name = old_name, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword) + .keyword_loc = TOKEN2SLICE(parser, keyword) }; return node; @@ -2020,7 +2002,7 @@ pm_alternation_pattern_node_create(pm_parser_t *parser, pm_node_t *left, pm_node .base = PM_NODE_INIT_NODES(parser, PM_ALTERNATION_PATTERN_NODE, 0, left, right), .left = left, .right = right, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -2038,7 +2020,7 @@ pm_and_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *opera *node = (pm_and_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_AND_NODE, 0, left, right), .left = left, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .right = right }; @@ -2143,7 +2125,7 @@ pm_array_node_elements_append(pm_array_node_t *node, pm_node_t *element) { */ static void pm_array_node_close_set(const pm_parser_t *parser, pm_array_node_t *node, const pm_token_t *closing) { - assert(closing->type == PM_TOKEN_BRACKET_RIGHT || closing->type == PM_TOKEN_STRING_END || closing->type == PM_TOKEN_MISSING || closing->type == PM_TOKEN_NOT_PROVIDED); + assert(closing->type == PM_TOKEN_BRACKET_RIGHT || closing->type == PM_TOKEN_STRING_END || closing->type == 0); node->base.location.end = closing->end; node->closing_loc = TOKEN2SLICE(parser, closing); } @@ -2217,8 +2199,8 @@ pm_array_pattern_node_constant_create(pm_parser_t *parser, pm_node_t *constant, .base = PM_NODE_INIT_NODE_TOKEN(parser, PM_ARRAY_PATTERN_NODE, 0, constant, closing), .constant = constant, .rest = NULL, - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, closing), .requireds = { 0 }, .posts = { 0 } }; @@ -2238,8 +2220,8 @@ pm_array_pattern_node_empty_create(pm_parser_t *parser, const pm_token_t *openin .base = PM_NODE_INIT_TOKENS(parser, PM_ARRAY_PATTERN_NODE, 0, opening, closing), .constant = NULL, .rest = NULL, - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, closing), .requireds = { 0 }, .posts = { 0 } }; @@ -2262,7 +2244,7 @@ pm_assoc_node_create(pm_parser_t *parser, pm_node_t *key, const pm_token_t *oper if (value != NULL && value->location.end > key->location.end) { end = value->location.end; - } else if (operator->type != PM_TOKEN_NOT_PROVIDED) { + } else if (operator != NULL) { end = operator->end; } else { end = key->location.end; @@ -2288,7 +2270,7 @@ pm_assoc_node_create(pm_parser_t *parser, pm_node_t *key, const pm_token_t *oper *node = (pm_assoc_node_t) { .base = PM_NODE_INIT(parser, PM_ASSOC_NODE, flags, key->location.start, end), .key = key, - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), + .operator_loc = MAYBETOKEN2SLICE(parser, operator), .value = value }; @@ -2310,7 +2292,7 @@ pm_assoc_splat_node_create(pm_parser_t *parser, pm_node_t *value, const pm_token : PM_NODE_INIT_TOKEN_NODE(parser, PM_ASSOC_SPLAT_NODE, 0, operator, value) ), .value = value, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -2339,13 +2321,12 @@ static pm_begin_node_t * pm_begin_node_create(pm_parser_t *parser, const pm_token_t *begin_keyword, pm_statements_node_t *statements) { pm_begin_node_t *node = PM_NODE_ALLOC(parser, pm_begin_node_t); + const uint8_t *start = begin_keyword == NULL ? parser->start : begin_keyword->start; + const uint8_t *end = statements == NULL ? (begin_keyword == NULL ? parser->start : begin_keyword->end) : statements->base.location.end; + *node = (pm_begin_node_t) { - .base = ( - (statements == NULL) - ? PM_NODE_INIT_TOKEN(parser, PM_BEGIN_NODE, 0, begin_keyword) - : PM_NODE_INIT_TOKEN_NODE(parser, PM_BEGIN_NODE, 0, begin_keyword, statements) - ), - .begin_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(begin_keyword), + .base = PM_NODE_INIT(parser, PM_BEGIN_NODE, 0, start, end), + .begin_keyword_loc = MAYBETOKEN2SLICE(parser, begin_keyword), .statements = statements, .end_keyword_loc = { 0 } }; @@ -2359,7 +2340,7 @@ pm_begin_node_create(pm_parser_t *parser, const pm_token_t *begin_keyword, pm_st static void pm_begin_node_rescue_clause_set(pm_begin_node_t *node, pm_rescue_node_t *rescue_clause) { // If the begin keyword doesn't exist, we set the start on the begin_node - if (!node->begin_keyword_loc.start) { + if (node->begin_keyword_loc.length == 0) { node->base.location.start = rescue_clause->base.location.start; } node->base.location.end = rescue_clause->base.location.end; @@ -2388,11 +2369,10 @@ pm_begin_node_ensure_clause_set(pm_begin_node_t *node, pm_ensure_node_t *ensure_ * Set the end keyword and end location of a begin node. */ static void -pm_begin_node_end_keyword_set(pm_begin_node_t *node, const pm_token_t *end_keyword) { - assert(end_keyword->type == PM_TOKEN_KEYWORD_END || end_keyword->type == PM_TOKEN_MISSING); - +pm_begin_node_end_keyword_set(const pm_parser_t *parser, pm_begin_node_t *node, const pm_token_t *end_keyword) { + assert(end_keyword->type == PM_TOKEN_KEYWORD_END || end_keyword->type == 0); node->base.location.end = end_keyword->end; - node->end_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(end_keyword); + node->end_keyword_loc = TOKEN2SLICE(parser, end_keyword); } /** @@ -2400,6 +2380,7 @@ pm_begin_node_end_keyword_set(pm_begin_node_t *node, const pm_token_t *end_keywo */ static pm_block_argument_node_t * pm_block_argument_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_node_t *expression) { + assert(operator->type == PM_TOKEN_UAMPERSAND); pm_block_argument_node_t *node = PM_NODE_ALLOC(parser, pm_block_argument_node_t); *node = (pm_block_argument_node_t) { @@ -2409,7 +2390,7 @@ pm_block_argument_node_create(pm_parser_t *parser, const pm_token_t *operator, p : PM_NODE_INIT_TOKEN_NODE(parser, PM_BLOCK_ARGUMENT_NODE, 0, operator, expression) ), .expression = expression, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -2427,8 +2408,8 @@ pm_block_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const p .locals = *locals, .parameters = parameters, .body = body, - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing) + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, closing) }; return node; @@ -2439,18 +2420,18 @@ pm_block_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const p */ static pm_block_parameter_node_t * pm_block_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, const pm_token_t *operator) { - assert(operator->type == PM_TOKEN_NOT_PROVIDED || operator->type == PM_TOKEN_UAMPERSAND || operator->type == PM_TOKEN_AMPERSAND); + assert(operator->type == PM_TOKEN_UAMPERSAND || operator->type == PM_TOKEN_AMPERSAND); pm_block_parameter_node_t *node = PM_NODE_ALLOC(parser, pm_block_parameter_node_t); *node = (pm_block_parameter_node_t) { .base = ( - (name->type == PM_TOKEN_NOT_PROVIDED) + (name == NULL) ? PM_NODE_INIT_TOKEN(parser, PM_BLOCK_PARAMETER_NODE, 0, operator) : PM_NODE_INIT_TOKENS(parser, PM_BLOCK_PARAMETER_NODE, 0, operator, name) ), - .name = pm_parser_optional_constant_id_token(parser, name), - .name_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(name), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), + .name_loc = MAYBETOKEN2SLICE(parser, name), + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -2464,7 +2445,7 @@ pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *param pm_block_parameters_node_t *node = PM_NODE_ALLOC(parser, pm_block_parameters_node_t); const uint8_t *start; - if (opening->type != PM_TOKEN_NOT_PROVIDED) { + if (opening != NULL) { start = opening->start; } else if (parameters != NULL) { start = parameters->base.location.start; @@ -2475,7 +2456,7 @@ pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *param const uint8_t *end; if (parameters != NULL) { end = parameters->base.location.end; - } else if (opening->type != PM_TOKEN_NOT_PROVIDED) { + } else if (opening != NULL) { end = opening->end; } else { end = NULL; @@ -2484,7 +2465,7 @@ pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *param *node = (pm_block_parameters_node_t) { .base = PM_NODE_INIT(parser, PM_BLOCK_PARAMETERS_NODE, 0, start, end), .parameters = parameters, - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), + .opening_loc = MAYBETOKEN2SLICE(parser, opening), .closing_loc = { 0 }, .locals = { 0 } }; @@ -2496,11 +2477,10 @@ pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *param * Set the closing location of a BlockParametersNode node. */ static void -pm_block_parameters_node_closing_set(pm_block_parameters_node_t *node, const pm_token_t *closing) { - assert(closing->type == PM_TOKEN_PIPE || closing->type == PM_TOKEN_PARENTHESIS_RIGHT || closing->type == PM_TOKEN_MISSING); - +pm_block_parameters_node_closing_set(const pm_parser_t *parser, pm_block_parameters_node_t *node, const pm_token_t *closing) { + assert(closing->type == PM_TOKEN_PIPE || closing->type == PM_TOKEN_PARENTHESIS_RIGHT || closing->type == 0); node->base.location.end = closing->end; - node->closing_loc = PM_LOCATION_TOKEN_VALUE(closing); + node->closing_loc = TOKEN2SLICE(parser, closing); } /** @@ -2544,7 +2524,7 @@ pm_break_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument : PM_NODE_INIT_TOKEN_NODE(parser, PM_BREAK_NODE, 0, keyword, arguments) ), .arguments = arguments, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword) + .keyword_loc = TOKEN2SLICE(parser, keyword) }; return node; @@ -2560,9 +2540,9 @@ static const pm_node_flags_t PM_CALL_NODE_FLAGS_COMPARISON = ((PM_CALL_NODE_FLAG static const pm_node_flags_t PM_CALL_NODE_FLAGS_INDEX = ((PM_CALL_NODE_FLAGS_LAST - 1) << 3); /** - * Allocate and initialize a new CallNode node. This sets everything to NULL or - * PM_TOKEN_NOT_PROVIDED as appropriate such that its values can be overridden - * in the various specializations of this function. + * Allocate and initialize a new CallNode node. This sets everything to NULL + * such that its values can be overridden in the various specializations of this + * function. */ static pm_call_node_t * pm_call_node_create(pm_parser_t *parser, pm_node_flags_t flags) { @@ -2612,12 +2592,12 @@ pm_call_node_aref_create(pm_parser_t *parser, pm_node_t *receiver, pm_arguments_ node->base.location.end = pm_arguments_end(arguments); node->receiver = receiver; - node->message_loc.start = arguments->opening_loc.start; - node->message_loc.end = arguments->closing_loc.end; + node->message_loc.start = (arguments->opening_loc.start == NULL ? 0 : ((uint32_t) (arguments->opening_loc.start - parser->start))); + node->message_loc.length = (arguments->closing_loc.end == NULL ? 0 : ((uint32_t) (arguments->closing_loc.end - arguments->opening_loc.start))); - node->opening_loc = arguments->opening_loc; + node->opening_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc); node->arguments = arguments->arguments; - node->closing_loc = arguments->closing_loc; + node->closing_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc); node->block = arguments->block; node->name = pm_parser_constant_id_constant(parser, "[]", 2); @@ -2638,7 +2618,7 @@ pm_call_node_binary_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t node->base.location.end = MAX(receiver->location.end, argument->location.end); node->receiver = receiver; - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator); + node->message_loc = TOKEN2SLICE(parser, operator); pm_arguments_node_t *arguments = pm_arguments_node_create(parser); pm_arguments_node_arguments_append(arguments, argument); @@ -2667,11 +2647,11 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o node->base.location.end = end; node->receiver = receiver; - node->call_operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator); - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(message); - node->opening_loc = arguments->opening_loc; + node->call_operator_loc = TOKEN2SLICE(parser, operator); + node->message_loc = TOKEN2SLICE(parser, message); + node->opening_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc); node->arguments = arguments->arguments; - node->closing_loc = arguments->closing_loc; + node->closing_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc); node->block = arguments->block; if (operator->type == PM_TOKEN_AMPERSAND_DOT) { @@ -2696,8 +2676,6 @@ pm_call_node_call_synthesized_create(pm_parser_t *parser, pm_node_t *receiver, c node->base.location.end = parser->end; node->receiver = receiver; - node->call_operator_loc = (pm_location_t) { .start = NULL, .end = NULL }; - node->message_loc = (pm_location_t) { .start = NULL, .end = NULL }; node->arguments = arguments; node->name = pm_parser_constant_id_constant(parser, message, strlen(message)); @@ -2715,10 +2693,10 @@ pm_call_node_fcall_create(pm_parser_t *parser, pm_token_t *message, pm_arguments node->base.location.start = message->start; node->base.location.end = pm_arguments_end(arguments); - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(message); - node->opening_loc = arguments->opening_loc; + node->message_loc = TOKEN2SLICE(parser, message); + node->opening_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc); node->arguments = arguments->arguments; - node->closing_loc = arguments->closing_loc; + node->closing_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc); node->block = arguments->block; node->name = pm_parser_constant_id_token(parser, message); @@ -2759,10 +2737,10 @@ pm_call_node_not_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *me } node->receiver = receiver; - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(message); - node->opening_loc = arguments->opening_loc; + node->message_loc = TOKEN2SLICE(parser, message); + node->opening_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc); node->arguments = arguments->arguments; - node->closing_loc = arguments->closing_loc; + node->closing_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc); node->name = pm_parser_constant_id_constant(parser, "!", 1); return node; @@ -2781,10 +2759,10 @@ pm_call_node_shorthand_create(pm_parser_t *parser, pm_node_t *receiver, pm_token node->base.location.end = pm_arguments_end(arguments); node->receiver = receiver; - node->call_operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator); - node->opening_loc = arguments->opening_loc; + node->call_operator_loc = TOKEN2SLICE(parser, operator); + node->opening_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc); node->arguments = arguments->arguments; - node->closing_loc = arguments->closing_loc; + node->closing_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc); node->block = arguments->block; if (operator->type == PM_TOKEN_AMPERSAND_DOT) { @@ -2808,7 +2786,7 @@ pm_call_node_unary_create(pm_parser_t *parser, pm_token_t *operator, pm_node_t * node->base.location.end = receiver->location.end; node->receiver = receiver; - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator); + node->message_loc = TOKEN2SLICE(parser, operator); node->name = pm_parser_constant_id_constant(parser, name, strlen(name)); return node; @@ -2823,7 +2801,7 @@ pm_call_node_variable_call_create(pm_parser_t *parser, pm_token_t *message) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); node->base.location = PM_LOCATION_TOKEN_VALUE(message); - node->message_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(message); + node->message_loc = TOKEN2SLICE(parser, message); node->name = pm_parser_constant_id_token(parser, message); return node; @@ -2836,11 +2814,11 @@ pm_call_node_variable_call_create(pm_parser_t *parser, pm_token_t *message) { static inline bool pm_call_node_writable_p(const pm_parser_t *parser, const pm_call_node_t *node) { return ( - (node->message_loc.start != NULL) && - (node->message_loc.end[-1] != '!') && - (node->message_loc.end[-1] != '?') && - char_is_identifier_start(parser, node->message_loc.start, parser->end - node->message_loc.start) && - (node->opening_loc.start == NULL) && + (node->message_loc.length > 0) && + (parser->start[node->message_loc.start + node->message_loc.length - 1] != '!') && + (parser->start[node->message_loc.start + node->message_loc.length - 1] != '?') && + char_is_identifier_start(parser, parser->start + node->message_loc.start, node->message_loc.length) && + (node->opening_loc.length == 0) && (node->arguments == NULL) && (node->block == NULL) ); @@ -2882,7 +2860,7 @@ pm_call_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -2938,7 +2916,7 @@ pm_index_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, cons .arguments = target->arguments, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -2966,7 +2944,7 @@ pm_call_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .read_name = 0, .write_name = target->name, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -2999,7 +2977,7 @@ pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3027,7 +3005,7 @@ pm_call_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3060,7 +3038,7 @@ pm_index_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const .arguments = target->arguments, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3135,7 +3113,7 @@ pm_capture_pattern_node_create(pm_parser_t *parser, pm_node_t *value, pm_local_v .base = PM_NODE_INIT_NODES(parser, PM_CAPTURE_PATTERN_NODE, 0, value, target), .value = value, .target = target, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -3149,11 +3127,11 @@ pm_case_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node pm_case_node_t *node = PM_NODE_ALLOC(parser, pm_case_node_t); *node = (pm_case_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_CASE_NODE, 0, case_keyword, end_keyword), + .base = PM_NODE_INIT(parser, PM_CASE_NODE, 0, case_keyword->start, end_keyword == NULL ? case_keyword->end : end_keyword->end), .predicate = predicate, .else_clause = NULL, - .case_keyword_loc = PM_LOCATION_TOKEN_VALUE(case_keyword), - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword), + .case_keyword_loc = TOKEN2SLICE(parser, case_keyword), + .end_keyword_loc = MAYBETOKEN2SLICE(parser, end_keyword), .conditions = { 0 } }; @@ -3184,24 +3162,24 @@ pm_case_node_else_clause_set(pm_case_node_t *node, pm_else_node_t *else_clause) * Set the end location for a CaseNode node. */ static void -pm_case_node_end_keyword_loc_set(pm_case_node_t *node, const pm_token_t *end_keyword) { +pm_case_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_node_t *node, const pm_token_t *end_keyword) { node->base.location.end = end_keyword->end; - node->end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword); + node->end_keyword_loc = TOKEN2SLICE(parser, end_keyword); } /** * Allocate and initialize a new CaseMatchNode node. */ static pm_case_match_node_t * -pm_case_match_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node_t *predicate, const pm_token_t *end_keyword) { +pm_case_match_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node_t *predicate) { pm_case_match_node_t *node = PM_NODE_ALLOC(parser, pm_case_match_node_t); *node = (pm_case_match_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_CASE_MATCH_NODE, 0, case_keyword, end_keyword), + .base = PM_NODE_INIT_TOKEN(parser, PM_CASE_MATCH_NODE, 0, case_keyword), .predicate = predicate, .else_clause = NULL, - .case_keyword_loc = PM_LOCATION_TOKEN_VALUE(case_keyword), - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword), + .case_keyword_loc = TOKEN2SLICE(parser, case_keyword), + .end_keyword_loc = { 0 }, .conditions = { 0 } }; @@ -3232,9 +3210,9 @@ pm_case_match_node_else_clause_set(pm_case_match_node_t *node, pm_else_node_t *e * Set the end location for a CaseMatchNode node. */ static void -pm_case_match_node_end_keyword_loc_set(pm_case_match_node_t *node, const pm_token_t *end_keyword) { +pm_case_match_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_match_node_t *node, const pm_token_t *end_keyword) { node->base.location.end = end_keyword->end; - node->end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword); + node->end_keyword_loc = TOKEN2SLICE(parser, end_keyword); } /** @@ -3247,12 +3225,12 @@ pm_class_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const p *node = (pm_class_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_CLASS_NODE, 0, class_keyword, end_keyword), .locals = *locals, - .class_keyword_loc = PM_LOCATION_TOKEN_VALUE(class_keyword), + .class_keyword_loc = TOKEN2SLICE(parser, class_keyword), .constant_path = constant_path, - .inheritance_operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(inheritance_operator), + .inheritance_operator_loc = MAYBETOKEN2SLICE(parser, inheritance_operator), .superclass = superclass, .body = body, - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword), + .end_keyword_loc = TOKEN2SLICE(parser, end_keyword), .name = pm_parser_constant_id_token(parser, name) }; @@ -3270,8 +3248,8 @@ pm_class_variable_and_write_node_create(pm_parser_t *parser, pm_class_variable_r *node = (pm_class_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->base.location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3288,8 +3266,8 @@ pm_class_variable_operator_write_node_create(pm_parser_t *parser, pm_class_varia *node = (pm_class_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = target->base.location, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->base.location), + .binary_operator_loc = TOKEN2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -3308,8 +3286,8 @@ pm_class_variable_or_write_node_create(pm_parser_t *parser, pm_class_variable_re *node = (pm_class_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->base.location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3357,8 +3335,8 @@ pm_class_variable_write_node_create(pm_parser_t *parser, pm_class_variable_read_ *node = (pm_class_variable_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_WRITE_NODE, flags, read_node, value), .name = read_node->name, - .name_loc = PM_LOCATION_NODE_VALUE(UP(read_node)), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &read_node->base.location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3376,7 +3354,7 @@ pm_constant_path_and_write_node_create(pm_parser_t *parser, pm_constant_path_nod *node = (pm_constant_path_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_AND_WRITE_NODE, 0, target, value), .target = target, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3393,7 +3371,7 @@ pm_constant_path_operator_write_node_create(pm_parser_t *parser, pm_constant_pat *node = (pm_constant_path_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_OPERATOR_WRITE_NODE, 0, target, value), .target = target, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .binary_operator_loc = TOKEN2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -3412,7 +3390,7 @@ pm_constant_path_or_write_node_create(pm_parser_t *parser, pm_constant_path_node *node = (pm_constant_path_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_OR_WRITE_NODE, 0, target, value), .target = target, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3432,23 +3410,17 @@ pm_constant_path_node_create(pm_parser_t *parser, pm_node_t *parent, const pm_to name = pm_parser_constant_id_token(parser, name_token); } - if (parent == NULL) { - *node = (pm_constant_path_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_CONSTANT_PATH_NODE, 0, delimiter, name_token), - .parent = parent, - .name = name, - .delimiter_loc = PM_LOCATION_TOKEN_VALUE(delimiter), - .name_loc = PM_LOCATION_TOKEN_VALUE(name_token) - }; - } else { - *node = (pm_constant_path_node_t) { - .base = PM_NODE_INIT_NODE_TOKEN(parser, PM_CONSTANT_PATH_NODE, 0, parent, name_token), - .parent = parent, - .name = name, - .delimiter_loc = PM_LOCATION_TOKEN_VALUE(delimiter), - .name_loc = PM_LOCATION_TOKEN_VALUE(name_token) - }; - } + *node = (pm_constant_path_node_t) { + .base = ( + (parent == NULL) + ? PM_NODE_INIT_TOKENS(parser, PM_CONSTANT_PATH_NODE, 0, delimiter, name_token) + : PM_NODE_INIT_NODE_TOKEN(parser, PM_CONSTANT_PATH_NODE, 0, parent, name_token) + ), + .parent = parent, + .name = name, + .delimiter_loc = TOKEN2SLICE(parser, delimiter), + .name_loc = TOKEN2SLICE(parser, name_token) + }; return node; } @@ -3464,7 +3436,7 @@ pm_constant_path_write_node_create(pm_parser_t *parser, pm_constant_path_node_t *node = (pm_constant_path_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_WRITE_NODE, flags, target, value), .target = target, - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3482,8 +3454,8 @@ pm_constant_and_write_node_create(pm_parser_t *parser, pm_constant_read_node_t * *node = (pm_constant_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_AND_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->base.location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3500,8 +3472,8 @@ pm_constant_operator_write_node_create(pm_parser_t *parser, pm_constant_read_nod *node = (pm_constant_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = target->base.location, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->base.location), + .binary_operator_loc = TOKEN2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -3520,8 +3492,8 @@ pm_constant_or_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *t *node = (pm_constant_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_OR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->base.location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3533,7 +3505,7 @@ pm_constant_or_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *t */ static pm_constant_read_node_t * pm_constant_read_node_create(pm_parser_t *parser, const pm_token_t *name) { - assert(name->type == PM_TOKEN_CONSTANT || name->type == PM_TOKEN_MISSING); + assert(name->type == PM_TOKEN_CONSTANT || name->type == 0); pm_constant_read_node_t *node = PM_NODE_ALLOC(parser, pm_constant_read_node_t); *node = (pm_constant_read_node_t) { @@ -3555,8 +3527,8 @@ pm_constant_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *targ *node = (pm_constant_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_WRITE_NODE, flags, target, value), .name = target->name, - .name_loc = target->base.location, - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->base.location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -3634,22 +3606,22 @@ pm_def_node_create( *node = (pm_def_node_t) { .base = ( - (end_keyword->type == PM_TOKEN_NOT_PROVIDED) + (end_keyword == NULL) ? PM_NODE_INIT_TOKEN_NODE(parser, PM_DEF_NODE, 0, def_keyword, body) : PM_NODE_INIT_TOKENS(parser, PM_DEF_NODE, 0, def_keyword, end_keyword) ), .name = name, - .name_loc = PM_LOCATION_TOKEN_VALUE(name_loc), + .name_loc = TOKEN2SLICE(parser, name_loc), .receiver = receiver, .parameters = parameters, .body = body, .locals = *locals, - .def_keyword_loc = PM_LOCATION_TOKEN_VALUE(def_keyword), - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), - .lparen_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(lparen), - .rparen_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(rparen), - .equal_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(equal), - .end_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(end_keyword) + .def_keyword_loc = TOKEN2SLICE(parser, def_keyword), + .operator_loc = MAYBETOKEN2SLICE(parser, operator), + .lparen_loc = MAYBETOKEN2SLICE(parser, lparen), + .rparen_loc = MAYBETOKEN2SLICE(parser, rparen), + .equal_loc = MAYBETOKEN2SLICE(parser, equal), + .end_keyword_loc = MAYBETOKEN2SLICE(parser, end_keyword) }; return node; @@ -3664,14 +3636,14 @@ pm_defined_node_create(pm_parser_t *parser, const pm_token_t *lparen, pm_node_t *node = (pm_defined_node_t) { .base = ( - (rparen->type == PM_TOKEN_NOT_PROVIDED) + (rparen == NULL) ? PM_NODE_INIT_TOKEN_NODE(parser, PM_DEFINED_NODE, 0, keyword, value) : PM_NODE_INIT_TOKENS(parser, PM_DEFINED_NODE, 0, keyword, rparen) ), - .lparen_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(lparen), + .lparen_loc = MAYBETOKEN2SLICE(parser, lparen), .value = value, - .rparen_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(rparen), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword) + .rparen_loc = MAYBETOKEN2SLICE(parser, rparen), + .keyword_loc = TOKEN2SLICE(parser, keyword) }; return node; @@ -3686,13 +3658,13 @@ pm_else_node_create(pm_parser_t *parser, const pm_token_t *else_keyword, pm_stat *node = (pm_else_node_t) { .base = ( - ((end_keyword->type == PM_TOKEN_NOT_PROVIDED) && (statements != NULL)) + ((end_keyword == NULL) && (statements != NULL)) ? PM_NODE_INIT_TOKEN_NODE(parser, PM_ELSE_NODE, 0, else_keyword, statements) : PM_NODE_INIT_TOKENS(parser, PM_ELSE_NODE, 0, else_keyword, end_keyword) ), - .else_keyword_loc = PM_LOCATION_TOKEN_VALUE(else_keyword), + .else_keyword_loc = TOKEN2SLICE(parser, else_keyword), .statements = statements, - .end_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(end_keyword) + .end_keyword_loc = MAYBETOKEN2SLICE(parser, end_keyword) }; return node; @@ -3707,9 +3679,9 @@ pm_embedded_statements_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_embedded_statements_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_EMBEDDED_STATEMENTS_NODE, 0, opening, closing), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), + .opening_loc = TOKEN2SLICE(parser, opening), .statements = statements, - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing) + .closing_loc = TOKEN2SLICE(parser, closing) }; return node; @@ -3724,7 +3696,7 @@ pm_embedded_variable_node_create(pm_parser_t *parser, const pm_token_t *operator *node = (pm_embedded_variable_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_EMBEDDED_VARIABLE_NODE, 0, operator, variable), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .variable = variable }; @@ -3740,9 +3712,9 @@ pm_ensure_node_create(pm_parser_t *parser, const pm_token_t *ensure_keyword, pm_ *node = (pm_ensure_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_ENSURE_NODE, 0, ensure_keyword, end_keyword), - .ensure_keyword_loc = PM_LOCATION_TOKEN_VALUE(ensure_keyword), + .ensure_keyword_loc = TOKEN2SLICE(parser, ensure_keyword), .statements = statements, - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword) + .end_keyword_loc = TOKEN2SLICE(parser, end_keyword) }; return node; @@ -4010,10 +3982,10 @@ pm_for_node_create( .index = index, .collection = collection, .statements = statements, - .for_keyword_loc = PM_LOCATION_TOKEN_VALUE(for_keyword), - .in_keyword_loc = PM_LOCATION_TOKEN_VALUE(in_keyword), - .do_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(do_keyword), - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword) + .for_keyword_loc = TOKEN2SLICE(parser, for_keyword), + .in_keyword_loc = TOKEN2SLICE(parser, in_keyword), + .do_keyword_loc = MAYBETOKEN2SLICE(parser, do_keyword), + .end_keyword_loc = TOKEN2SLICE(parser, end_keyword) }; return node; @@ -4086,8 +4058,8 @@ pm_hash_pattern_node_empty_create(pm_parser_t *parser, const pm_token_t *opening *node = (pm_hash_pattern_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_HASH_PATTERN_NODE, 0, opening, closing), .constant = NULL, - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, closing), .elements = { 0 }, .rest = NULL }; @@ -4167,8 +4139,8 @@ pm_global_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, *node = (pm_global_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -4185,8 +4157,8 @@ pm_global_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *ta *node = (pm_global_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = target->location, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->location), + .binary_operator_loc = TOKEN2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -4205,8 +4177,8 @@ pm_global_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, *node = (pm_global_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -4254,8 +4226,8 @@ pm_global_variable_write_node_create(pm_parser_t *parser, pm_node_t *target, con *node = (pm_global_variable_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_WRITE_NODE, flags, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = PM_LOCATION_NODE_VALUE(target), - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -4272,8 +4244,8 @@ pm_global_variable_write_node_synthesized_create(pm_parser_t *parser, pm_constan *node = (pm_global_variable_write_node_t) { .base = PM_NODE_INIT_BASE(parser, PM_GLOBAL_VARIABLE_WRITE_NODE, 0), .name = name, - .name_loc = PM_LOCATION_NULL_VALUE(parser), - .operator_loc = PM_LOCATION_NULL_VALUE(parser), + .name_loc = { 0 }, + .operator_loc = { 0 }, .value = value }; @@ -4290,8 +4262,8 @@ pm_hash_node_create(pm_parser_t *parser, const pm_token_t *opening) { *node = (pm_hash_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_HASH_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_NULL_VALUE(parser), + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = { 0 }, .elements = { 0 } }; @@ -4319,9 +4291,9 @@ pm_hash_node_elements_append(pm_hash_node_t *hash, pm_node_t *element) { } static inline void -pm_hash_node_closing_loc_set(pm_hash_node_t *hash, pm_token_t *token) { +pm_hash_node_closing_loc_set(const pm_parser_t *parser, pm_hash_node_t *hash, pm_token_t *token) { hash->base.location.end = token->end; - hash->closing_loc = PM_LOCATION_TOKEN_VALUE(token); + hash->closing_loc = TOKEN2SLICE(parser, token); } /** @@ -4340,7 +4312,7 @@ pm_if_node_create(pm_parser_t *parser, pm_if_node_t *node = PM_NODE_ALLOC(parser, pm_if_node_t); const uint8_t *end; - if (end_keyword->type != PM_TOKEN_NOT_PROVIDED) { + if (end_keyword != NULL) { end = end_keyword->end; } else if (subsequent != NULL) { end = subsequent->location.end; @@ -4352,12 +4324,12 @@ pm_if_node_create(pm_parser_t *parser, *node = (pm_if_node_t) { .base = PM_NODE_INIT(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, if_keyword->start, end), - .if_keyword_loc = PM_LOCATION_TOKEN_VALUE(if_keyword), + .if_keyword_loc = TOKEN2SLICE(parser, if_keyword), .predicate = predicate, - .then_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(then_keyword), + .then_keyword_loc = MAYBETOKEN2SLICE(parser, then_keyword), .statements = statements, .subsequent = subsequent, - .end_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(end_keyword) + .end_keyword_loc = MAYBETOKEN2SLICE(parser, end_keyword) }; return node; @@ -4376,7 +4348,7 @@ pm_if_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const pm_t *node = (pm_if_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, statement, predicate), - .if_keyword_loc = PM_LOCATION_TOKEN_VALUE(if_keyword), + .if_keyword_loc = TOKEN2SLICE(parser, if_keyword), .predicate = predicate, .then_keyword_loc = { 0 }, .statements = statements, @@ -4401,16 +4373,14 @@ pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_to pm_statements_node_t *else_statements = pm_statements_node_create(parser); pm_statements_node_body_append(parser, else_statements, false_expression, true); - pm_token_t end_keyword = not_provided(parser); - pm_else_node_t *else_node = pm_else_node_create(parser, colon, else_statements, &end_keyword); - + pm_else_node_t *else_node = pm_else_node_create(parser, colon, else_statements, NULL); pm_if_node_t *node = PM_NODE_ALLOC(parser, pm_if_node_t); *node = (pm_if_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, predicate, false_expression), .if_keyword_loc = { 0 }, .predicate = predicate, - .then_keyword_loc = PM_LOCATION_TOKEN_VALUE(qmark), + .then_keyword_loc = TOKEN2SLICE(parser, qmark), .statements = if_statements, .subsequent = UP(else_node), .end_keyword_loc = { 0 } @@ -4421,15 +4391,15 @@ pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_to } static inline void -pm_if_node_end_keyword_loc_set(pm_if_node_t *node, const pm_token_t *keyword) { +pm_if_node_end_keyword_loc_set(const pm_parser_t *parser, pm_if_node_t *node, const pm_token_t *keyword) { node->base.location.end = keyword->end; - node->end_keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword); + node->end_keyword_loc = TOKEN2SLICE(parser, keyword); } static inline void -pm_else_node_end_keyword_loc_set(pm_else_node_t *node, const pm_token_t *keyword) { +pm_else_node_end_keyword_loc_set(const pm_parser_t *parser, pm_else_node_t *node, const pm_token_t *keyword) { node->base.location.end = keyword->end; - node->end_keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword); + node->end_keyword_loc = TOKEN2SLICE(parser, keyword); } /** @@ -4570,7 +4540,7 @@ pm_in_node_create(pm_parser_t *parser, pm_node_t *pattern, pm_statements_node_t const uint8_t *end; if (statements != NULL) { end = statements->base.location.end; - } else if (then_keyword->type != PM_TOKEN_NOT_PROVIDED) { + } else if (then_keyword != NULL) { end = then_keyword->end; } else { end = pattern->location.end; @@ -4580,8 +4550,8 @@ pm_in_node_create(pm_parser_t *parser, pm_node_t *pattern, pm_statements_node_t .base = PM_NODE_INIT(parser, PM_IN_NODE, 0, in_keyword->start, end), .pattern = pattern, .statements = statements, - .in_loc = PM_LOCATION_TOKEN_VALUE(in_keyword), - .then_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(then_keyword) + .in_loc = TOKEN2SLICE(parser, in_keyword), + .then_loc = MAYBETOKEN2SLICE(parser, then_keyword) }; return node; @@ -4598,8 +4568,8 @@ pm_instance_variable_and_write_node_create(pm_parser_t *parser, pm_instance_vari *node = (pm_instance_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->base.location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -4616,8 +4586,8 @@ pm_instance_variable_operator_write_node_create(pm_parser_t *parser, pm_instance *node = (pm_instance_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = target->base.location, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->base.location), + .binary_operator_loc = TOKEN2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -4636,8 +4606,8 @@ pm_instance_variable_or_write_node_create(pm_parser_t *parser, pm_instance_varia *node = (pm_instance_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = target->base.location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->base.location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -4672,8 +4642,8 @@ pm_instance_variable_write_node_create(pm_parser_t *parser, pm_instance_variable *node = (pm_instance_variable_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_WRITE_NODE, flags, read_node, value), .name = read_node->name, - .name_loc = PM_LOCATION_NODE_VALUE(read_node), - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &read_node->base.location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -4733,8 +4703,8 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok *node = (pm_interpolated_regular_expression_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(opening), + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, opening), .parts = { 0 } }; @@ -4755,7 +4725,7 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio static inline void pm_interpolated_regular_expression_node_closing_set(pm_parser_t *parser, pm_interpolated_regular_expression_node_t *node, const pm_token_t *closing) { - node->closing_loc = PM_LOCATION_TOKEN_VALUE(closing); + node->closing_loc = TOKEN2SLICE(parser, closing); node->base.location.end = closing->end; pm_node_flag_set(UP(node), pm_regular_expression_flags_create(parser, closing)); } @@ -4791,7 +4761,7 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ #define MUTABLE_FLAGS(node) \ node->base.flags = (pm_node_flags_t) ((FL(node) | PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE) & ~PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); - if (node->parts.size == 0 && node->opening_loc.start == NULL) { + if (node->parts.size == 0 && node->opening_loc.length == 0) { node->base.location.start = part->location.start; } @@ -4891,9 +4861,9 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin } *node = (pm_interpolated_string_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_INTERPOLATED_STRING_NODE, flags, opening, closing), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_INTERPOLATED_STRING_NODE, flags, opening == NULL ? NULL : opening->start, closing == NULL ? NULL : closing->end), + .opening_loc = MAYBETOKEN2SLICE(parser, opening), + .closing_loc = MAYBETOKEN2SLICE(parser, closing), .parts = { 0 } }; @@ -4911,14 +4881,14 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin * Set the closing token of the given InterpolatedStringNode node. */ static void -pm_interpolated_string_node_closing_set(pm_interpolated_string_node_t *node, const pm_token_t *closing) { - node->closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing); +pm_interpolated_string_node_closing_set(const pm_parser_t *parser, pm_interpolated_string_node_t *node, const pm_token_t *closing) { + node->closing_loc = TOKEN2SLICE(parser, closing); node->base.location.end = closing->end; } static void pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_t *part) { - if (node->parts.size == 0 && node->opening_loc.start == NULL) { + if (node->parts.size == 0 && node->opening_loc.length == 0) { node->base.location.start = part->location.start; } @@ -4927,8 +4897,8 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ } static void -pm_interpolated_symbol_node_closing_loc_set(pm_interpolated_symbol_node_t *node, const pm_token_t *closing) { - node->closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing); +pm_interpolated_symbol_node_closing_loc_set(const pm_parser_t *parser, pm_interpolated_symbol_node_t *node, const pm_token_t *closing) { + node->closing_loc = TOKEN2SLICE(parser, closing); node->base.location.end = closing->end; } @@ -4940,9 +4910,9 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin pm_interpolated_symbol_node_t *node = PM_NODE_ALLOC(parser, pm_interpolated_symbol_node_t); *node = (pm_interpolated_symbol_node_t) { - .base = PM_NODE_INIT_TOKENS(parser, PM_INTERPOLATED_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening, closing), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .base = PM_NODE_INIT(parser, PM_INTERPOLATED_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening == NULL ? NULL : opening->start, closing == NULL ? NULL : closing->end), + .opening_loc = MAYBETOKEN2SLICE(parser, opening), + .closing_loc = MAYBETOKEN2SLICE(parser, closing), .parts = { 0 } }; @@ -4965,8 +4935,8 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi *node = (pm_interpolated_x_string_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_INTERPOLATED_X_STRING_NODE, 0, opening, closing), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, closing), .parts = { 0 } }; @@ -4980,8 +4950,8 @@ pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_no } static inline void -pm_interpolated_xstring_node_closing_set(pm_interpolated_x_string_node_t *node, const pm_token_t *closing) { - node->closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing); +pm_interpolated_xstring_node_closing_set(const pm_parser_t *parser, pm_interpolated_x_string_node_t *node, const pm_token_t *closing) { + node->closing_loc = TOKEN2SLICE(parser, closing); node->base.location.end = closing->end; } @@ -5056,7 +5026,7 @@ pm_required_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t *node = (pm_required_keyword_parameter_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_REQUIRED_KEYWORD_PARAMETER_NODE, 0, name), .name = pm_parser_constant_id_location(parser, name->start, name->end - 1), - .name_loc = PM_LOCATION_TOKEN_VALUE(name), + .name_loc = TOKEN2SLICE(parser, name), }; return node; @@ -5072,7 +5042,7 @@ pm_optional_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t *node = (pm_optional_keyword_parameter_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_OPTIONAL_KEYWORD_PARAMETER_NODE, 0, name, value), .name = pm_parser_constant_id_location(parser, name->start, name->end - 1), - .name_loc = PM_LOCATION_TOKEN_VALUE(name), + .name_loc = TOKEN2SLICE(parser, name), .value = value }; @@ -5088,13 +5058,13 @@ pm_keyword_rest_parameter_node_create(pm_parser_t *parser, const pm_token_t *ope *node = (pm_keyword_rest_parameter_node_t) { .base = ( - (name->type == PM_TOKEN_NOT_PROVIDED) + (name == NULL) ? PM_NODE_INIT_TOKEN(parser, PM_KEYWORD_REST_PARAMETER_NODE, 0, operator) : PM_NODE_INIT_TOKENS(parser, PM_KEYWORD_REST_PARAMETER_NODE, 0, operator, name) ), - .name = pm_parser_optional_constant_id_token(parser, name), - .name_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(name), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), + .name_loc = MAYBETOKEN2SLICE(parser, name), + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -5118,9 +5088,9 @@ pm_lambda_node_create( *node = (pm_lambda_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_LAMBDA_NODE, 0, operator, closing), .locals = *locals, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .operator_loc = TOKEN2SLICE(parser, operator), + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, closing), .parameters = parameters, .body = body }; @@ -5139,8 +5109,8 @@ pm_local_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, *node = (pm_local_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_AND_WRITE_NODE, 0, target, value), - .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value, .name = name, .depth = depth @@ -5158,8 +5128,8 @@ pm_local_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *tar *node = (pm_local_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), - .name_loc = target->location, - .binary_operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->location), + .binary_operator_loc = TOKEN2SLICE(parser, operator), .value = value, .name = name, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), @@ -5180,8 +5150,8 @@ pm_local_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, c *node = (pm_local_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_OR_WRITE_NODE, 0, target, value), - .name_loc = target->location, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = LOCATION2SLICE(parser, &target->location), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value, .name = name, .depth = depth @@ -5240,8 +5210,8 @@ pm_local_variable_write_node_create(pm_parser_t *parser, pm_constant_id_t name, .name = name, .depth = depth, .value = value, - .name_loc = *name_loc, - .operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator) + .name_loc = LOCATION2SLICE(parser, name_loc), + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -5306,7 +5276,7 @@ pm_match_predicate_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t .base = PM_NODE_INIT_NODES(parser, PM_MATCH_PREDICATE_NODE, 0, value, pattern), .value = value, .pattern = pattern, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -5325,7 +5295,7 @@ pm_match_required_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t * .base = PM_NODE_INIT_NODES(parser, PM_MATCH_REQUIRED_NODE, 0, value, pattern), .value = value, .pattern = pattern, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -5357,10 +5327,10 @@ pm_module_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const *node = (pm_module_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_MODULE_NODE, 0, module_keyword, end_keyword), .locals = (locals == NULL ? ((pm_constant_id_list_t) { .ids = NULL, .size = 0, .capacity = 0 }) : *locals), - .module_keyword_loc = PM_LOCATION_TOKEN_VALUE(module_keyword), + .module_keyword_loc = TOKEN2SLICE(parser, module_keyword), .constant_path = constant_path, .body = body, - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword), + .end_keyword_loc = TOKEN2SLICE(parser, end_keyword), .name = pm_parser_constant_id_token(parser, name) }; @@ -5424,18 +5394,18 @@ pm_multi_target_node_targets_append(pm_parser_t *parser, pm_multi_target_node_t * Set the opening of a MultiTargetNode node. */ static void -pm_multi_target_node_opening_set(pm_multi_target_node_t *node, const pm_token_t *lparen) { +pm_multi_target_node_opening_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *lparen) { node->base.location.start = lparen->start; - node->lparen_loc = PM_LOCATION_TOKEN_VALUE(lparen); + node->lparen_loc = TOKEN2SLICE(parser, lparen); } /** * Set the closing of a MultiTargetNode node. */ static void -pm_multi_target_node_closing_set(pm_multi_target_node_t *node, const pm_token_t *rparen) { +pm_multi_target_node_closing_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *rparen) { node->base.location.end = rparen->end; - node->rparen_loc = PM_LOCATION_TOKEN_VALUE(rparen); + node->rparen_loc = TOKEN2SLICE(parser, rparen); } /** @@ -5453,7 +5423,7 @@ pm_multi_write_node_create(pm_parser_t *parser, pm_multi_target_node_t *target, .rights = target->rights, .lparen_loc = target->lparen_loc, .rparen_loc = target->rparen_loc, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -5478,7 +5448,7 @@ pm_next_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_arguments ? PM_NODE_INIT_TOKEN(parser, PM_NEXT_NODE, 0, keyword) : PM_NODE_INIT_TOKEN_NODE(parser, PM_NEXT_NODE, 0, keyword, arguments) ), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .keyword_loc = TOKEN2SLICE(parser, keyword), .arguments = arguments }; @@ -5511,8 +5481,8 @@ pm_no_keywords_parameter_node_create(pm_parser_t *parser, const pm_token_t *oper *node = (pm_no_keywords_parameter_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_NO_KEYWORDS_PARAMETER_NODE, 0, operator, keyword), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword) + .operator_loc = TOKEN2SLICE(parser, operator), + .keyword_loc = TOKEN2SLICE(parser, keyword) }; return node; @@ -5608,8 +5578,8 @@ pm_optional_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, c *node = (pm_optional_parameter_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_OPTIONAL_PARAMETER_NODE, 0, name, value), .name = pm_parser_constant_id_token(parser, name), - .name_loc = PM_LOCATION_TOKEN_VALUE(name), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .name_loc = TOKEN2SLICE(parser, name), + .operator_loc = TOKEN2SLICE(parser, operator), .value = value }; @@ -5629,7 +5599,7 @@ pm_or_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *operat .base = PM_NODE_INIT_NODES(parser, PM_OR_NODE, 0, left, right), .left = left, .right = right, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -5765,8 +5735,8 @@ pm_parentheses_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_no *node = (pm_parentheses_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_PARENTHESES_NODE, flags, opening, closing), .body = body, - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing) + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, closing) }; return node; @@ -5782,9 +5752,9 @@ pm_pinned_expression_node_create(pm_parser_t *parser, pm_node_t *expression, con *node = (pm_pinned_expression_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_PINNED_EXPRESSION_NODE, 0, operator, rparen), .expression = expression, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), - .lparen_loc = PM_LOCATION_TOKEN_VALUE(lparen), - .rparen_loc = PM_LOCATION_TOKEN_VALUE(rparen) + .operator_loc = TOKEN2SLICE(parser, operator), + .lparen_loc = TOKEN2SLICE(parser, lparen), + .rparen_loc = TOKEN2SLICE(parser, rparen) }; return node; @@ -5800,7 +5770,7 @@ pm_pinned_variable_node_create(pm_parser_t *parser, const pm_token_t *operator, *node = (pm_pinned_variable_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_PINNED_VARIABLE_NODE, 0, operator, variable), .variable = variable, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -5816,9 +5786,9 @@ pm_post_execution_node_create(pm_parser_t *parser, const pm_token_t *keyword, co *node = (pm_post_execution_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_POST_EXECUTION_NODE, 0, keyword, closing), .statements = statements, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing) + .keyword_loc = TOKEN2SLICE(parser, keyword), + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, closing) }; return node; @@ -5834,9 +5804,9 @@ pm_pre_execution_node_create(pm_parser_t *parser, const pm_token_t *keyword, con *node = (pm_pre_execution_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_PRE_EXECUTION_NODE, 0, keyword, closing), .statements = statements, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing) + .keyword_loc = TOKEN2SLICE(parser, keyword), + .opening_loc = TOKEN2SLICE(parser, opening), + .closing_loc = TOKEN2SLICE(parser, closing) }; return node; @@ -5872,7 +5842,7 @@ pm_range_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *ope .base = PM_NODE_INIT(parser, PM_RANGE_NODE, flags, (left == NULL ? operator->start : left->location.start), (right == NULL ? operator->end : right->location.end)), .left = left, .right = right, - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -5904,9 +5874,9 @@ pm_regular_expression_node_create_unescaped(pm_parser_t *parser, const pm_token_ *node = (pm_regular_expression_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_REGULAR_EXPRESSION_NODE, flags, opening, closing), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .content_loc = PM_LOCATION_TOKEN_VALUE(content), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .opening_loc = TOKEN2SLICE(parser, opening), + .content_loc = TOKEN2SLICE(parser, content), + .closing_loc = TOKEN2SLICE(parser, closing), .unescaped = *unescaped }; @@ -5946,7 +5916,7 @@ pm_rescue_modifier_node_create(pm_parser_t *parser, pm_node_t *expression, const *node = (pm_rescue_modifier_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_RESCUE_MODIFIER_NODE, 0, expression, rescue_expression), .expression = expression, - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .keyword_loc = TOKEN2SLICE(parser, keyword), .rescue_expression = rescue_expression }; @@ -5962,7 +5932,7 @@ pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword) { *node = (pm_rescue_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_RESCUE_NODE, 0, keyword), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .keyword_loc = TOKEN2SLICE(parser, keyword), .operator_loc = { 0 }, .then_keyword_loc = { 0 }, .reference = NULL, @@ -5975,8 +5945,8 @@ pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword) { } static inline void -pm_rescue_node_operator_set(pm_rescue_node_t *node, const pm_token_t *operator) { - node->operator_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(operator); +pm_rescue_node_operator_set(const pm_parser_t *parser, pm_rescue_node_t *node, const pm_token_t *operator) { + node->operator_loc = TOKEN2SLICE(parser, operator); } /** @@ -6026,13 +5996,13 @@ pm_rest_parameter_node_create(pm_parser_t *parser, const pm_token_t *operator, c *node = (pm_rest_parameter_node_t) { .base = ( - (name->type == PM_TOKEN_NOT_PROVIDED) + (name == NULL) ? PM_NODE_INIT_TOKEN(parser, PM_REST_PARAMETER_NODE, 0, operator) : PM_NODE_INIT_TOKENS(parser, PM_REST_PARAMETER_NODE, 0, operator, name) ), - .name = pm_parser_optional_constant_id_token(parser, name), - .name_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(name), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator) + .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), + .name_loc = MAYBETOKEN2SLICE(parser, name), + .operator_loc = TOKEN2SLICE(parser, operator) }; return node; @@ -6066,7 +6036,7 @@ pm_return_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argumen ? PM_NODE_INIT_TOKEN(parser, PM_RETURN_NODE, 0, keyword) : PM_NODE_INIT_TOKEN_NODE(parser, PM_RETURN_NODE, 0, keyword, arguments) ), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .keyword_loc = TOKEN2SLICE(parser, keyword), .arguments = arguments }; @@ -6113,11 +6083,11 @@ pm_singleton_class_node_create(pm_parser_t *parser, pm_constant_id_list_t *local *node = (pm_singleton_class_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_SINGLETON_CLASS_NODE, 0, class_keyword, end_keyword), .locals = *locals, - .class_keyword_loc = PM_LOCATION_TOKEN_VALUE(class_keyword), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .class_keyword_loc = TOKEN2SLICE(parser, class_keyword), + .operator_loc = TOKEN2SLICE(parser, operator), .expression = expression, .body = body, - .end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword) + .end_keyword_loc = TOKEN2SLICE(parser, end_keyword) }; return node; @@ -6193,7 +6163,7 @@ pm_splat_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_node_t ? PM_NODE_INIT_TOKEN(parser, PM_SPLAT_NODE, 0, operator) : PM_NODE_INIT_TOKEN_NODE(parser, PM_SPLAT_NODE, 0, operator, expression) ), - .operator_loc = PM_LOCATION_TOKEN_VALUE(operator), + .operator_loc = TOKEN2SLICE(parser, operator), .expression = expression }; @@ -6300,14 +6270,14 @@ pm_string_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, break; } - const uint8_t *start = (opening->type == PM_TOKEN_NOT_PROVIDED ? content->start : opening->start); - const uint8_t *end = (closing->type == PM_TOKEN_NOT_PROVIDED ? content->end : closing->end); + const uint8_t *start = opening == NULL ? content->start : opening->start; + const uint8_t *end = closing == NULL ? content->end : closing->end; *node = (pm_string_node_t) { .base = PM_NODE_INIT(parser, PM_STRING_NODE, flags, start, end), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .content_loc = PM_LOCATION_TOKEN_VALUE(content), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .opening_loc = MAYBETOKEN2SLICE(parser, opening), + .content_loc = TOKEN2SLICE(parser, content), + .closing_loc = MAYBETOKEN2SLICE(parser, closing), .unescaped = *string }; @@ -6348,10 +6318,10 @@ pm_super_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument *node = (pm_super_node_t) { .base = PM_NODE_INIT(parser, PM_SUPER_NODE, 0, keyword->start, end), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), - .lparen_loc = arguments->opening_loc, + .keyword_loc = TOKEN2SLICE(parser, keyword), + .lparen_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc), .arguments = arguments->arguments, - .rparen_loc = arguments->closing_loc, + .rparen_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc), .block = arguments->block }; @@ -6576,14 +6546,14 @@ static pm_symbol_node_t * pm_symbol_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *value, const pm_token_t *closing, const pm_string_t *unescaped, pm_node_flags_t flags) { pm_symbol_node_t *node = PM_NODE_ALLOC(parser, pm_symbol_node_t); - const uint8_t *start = (opening->type == PM_TOKEN_NOT_PROVIDED ? value->start : opening->start); - const uint8_t *end = (closing->type == PM_TOKEN_NOT_PROVIDED ? value->end : closing->end); + const uint8_t *start = opening == NULL ? value->start : opening->start; + const uint8_t *end = closing == NULL ? value->end : closing->end; *node = (pm_symbol_node_t) { .base = PM_NODE_INIT(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL | flags, start, end), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), - .value_loc = PM_LOCATION_TOKEN_VALUE(value), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .opening_loc = MAYBETOKEN2SLICE(parser, opening), + .value_loc = MAYBETOKEN2SLICE(parser, value), + .closing_loc = MAYBETOKEN2SLICE(parser, closing), .unescaped = *unescaped }; @@ -6613,35 +6583,15 @@ pm_symbol_node_create_current_string(pm_parser_t *parser, const pm_token_t *open */ static pm_symbol_node_t * pm_symbol_node_label_create(pm_parser_t *parser, const pm_token_t *token) { - pm_symbol_node_t *node; - - switch (token->type) { - case PM_TOKEN_LABEL: { - pm_token_t opening = not_provided(parser); - pm_token_t closing = { .type = PM_TOKEN_LABEL_END, .start = token->end - 1, .end = token->end }; + assert(token->type == PM_TOKEN_LABEL); - pm_token_t label = { .type = PM_TOKEN_LABEL, .start = token->start, .end = token->end - 1 }; - node = pm_symbol_node_create(parser, &opening, &label, &closing); + pm_token_t closing = { .type = PM_TOKEN_LABEL_END, .start = token->end - 1, .end = token->end }; + pm_token_t label = { .type = PM_TOKEN_LABEL, .start = token->start, .end = token->end - 1 }; + pm_symbol_node_t *node = pm_symbol_node_create(parser, NULL, &label, &closing); - assert((label.end - label.start) >= 0); - pm_string_shared_init(&node->unescaped, label.start, label.end); - pm_node_flag_set(UP(node), parse_symbol_encoding(parser, &label, &node->unescaped, false)); - - break; - } - case PM_TOKEN_MISSING: { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - - pm_token_t label = { .type = PM_TOKEN_LABEL, .start = token->start, .end = token->end }; - node = pm_symbol_node_create(parser, &opening, &label, &closing); - break; - } - default: - assert(false && "unreachable"); - node = NULL; - break; - } + assert((label.end - label.start) >= 0); + pm_string_shared_init(&node->unescaped, label.start, label.end); + pm_node_flag_set(UP(node), parse_symbol_encoding(parser, &label, &node->unescaped, false)); return node; } @@ -6655,7 +6605,7 @@ pm_symbol_node_synthesized_create(pm_parser_t *parser, const char *content) { *node = (pm_symbol_node_t) { .base = PM_NODE_INIT_BASE(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL | PM_SYMBOL_FLAGS_FORCED_US_ASCII_ENCODING), - .value_loc = PM_LOCATION_NULL_VALUE(parser), + .value_loc = { 0 }, .unescaped = { 0 } }; @@ -6667,16 +6617,24 @@ pm_symbol_node_synthesized_create(pm_parser_t *parser, const char *content) { * Check if the given node is a label in a hash. */ static bool -pm_symbol_node_label_p(pm_node_t *node) { +pm_symbol_node_label_p(const pm_parser_t *parser, const pm_node_t *node) { const uint8_t *end = NULL; switch (PM_NODE_TYPE(node)) { - case PM_SYMBOL_NODE: - end = ((pm_symbol_node_t *) node)->closing_loc.end; + case PM_SYMBOL_NODE: { + const pm_symbol_node_t *cast = (pm_symbol_node_t *) node; + if (cast->closing_loc.length != 0) { + end = parser->start + cast->closing_loc.start + cast->closing_loc.length; + } break; - case PM_INTERPOLATED_SYMBOL_NODE: - end = ((pm_interpolated_symbol_node_t *) node)->closing_loc.end; + } + case PM_INTERPOLATED_SYMBOL_NODE: { + const pm_interpolated_symbol_node_t *cast = (pm_interpolated_symbol_node_t *) node; + if (cast->closing_loc.length != 0) { + end = parser->start + cast->closing_loc.start + cast->closing_loc.length; + } break; + } default: return false; } @@ -6693,13 +6651,18 @@ pm_string_node_to_symbol_node(pm_parser_t *parser, pm_string_node_t *node, const *new_node = (pm_symbol_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening, closing), - .opening_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(opening), + .opening_loc = TOKEN2SLICE(parser, opening), .value_loc = node->content_loc, - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .closing_loc = TOKEN2SLICE(parser, closing), .unescaped = node->unescaped }; - pm_token_t content = { .type = PM_TOKEN_IDENTIFIER, .start = node->content_loc.start, .end = node->content_loc.end }; + pm_token_t content = { + .type = PM_TOKEN_IDENTIFIER, + .start = parser->start + node->content_loc.start, + .end = parser->start + node->content_loc.start + node->content_loc.length + }; + pm_node_flag_set(UP(new_node), parse_symbol_encoding(parser, &content, &node->unescaped, true)); // We are explicitly _not_ using pm_node_destroy here because we don't want @@ -6782,7 +6745,7 @@ pm_undef_node_create(pm_parser_t *parser, const pm_token_t *token) { *node = (pm_undef_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_UNDEF_NODE, 0, token), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(token), + .keyword_loc = TOKEN2SLICE(parser, token), .names = { 0 } }; @@ -6810,9 +6773,9 @@ pm_unless_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_t *node = (pm_unless_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_UNLESS_NODE, PM_NODE_FLAG_NEWLINE, keyword, end), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .keyword_loc = TOKEN2SLICE(parser, keyword), .predicate = predicate, - .then_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(then_keyword), + .then_keyword_loc = MAYBETOKEN2SLICE(parser, then_keyword), .statements = statements, .else_clause = NULL, .end_keyword_loc = { 0 } @@ -6834,7 +6797,7 @@ pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const *node = (pm_unless_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_UNLESS_NODE, PM_NODE_FLAG_NEWLINE, statement, predicate), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(unless_keyword), + .keyword_loc = TOKEN2SLICE(parser, unless_keyword), .predicate = predicate, .then_keyword_loc = { 0 }, .statements = statements, @@ -6846,8 +6809,8 @@ pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const } static inline void -pm_unless_node_end_keyword_loc_set(pm_unless_node_t *node, const pm_token_t *end_keyword) { - node->end_keyword_loc = PM_LOCATION_TOKEN_VALUE(end_keyword); +pm_unless_node_end_keyword_loc_set(const pm_parser_t *parser, pm_unless_node_t *node, const pm_token_t *end_keyword) { + node->end_keyword_loc = TOKEN2SLICE(parser, end_keyword); node->base.location.end = end_keyword->end; } @@ -6884,9 +6847,9 @@ pm_until_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_to *node = (pm_until_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_UNTIL_NODE, flags, keyword, closing), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), - .do_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(do_keyword), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .keyword_loc = TOKEN2SLICE(parser, keyword), + .do_keyword_loc = MAYBETOKEN2SLICE(parser, do_keyword), + .closing_loc = TOKEN2SLICE(parser, closing), .predicate = predicate, .statements = statements }; @@ -6905,7 +6868,7 @@ pm_until_node_modifier_create(pm_parser_t *parser, const pm_token_t *keyword, pm *node = (pm_until_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_UNTIL_NODE, flags, statements, predicate), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .keyword_loc = TOKEN2SLICE(parser, keyword), .do_keyword_loc = { 0 }, .closing_loc = { 0 }, .predicate = predicate, @@ -6924,7 +6887,7 @@ pm_when_node_create(pm_parser_t *parser, const pm_token_t *keyword) { *node = (pm_when_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_WHEN_NODE, 0, keyword), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .keyword_loc = TOKEN2SLICE(parser, keyword), .statements = NULL, .then_keyword_loc = { 0 }, .conditions = { 0 } @@ -6946,9 +6909,9 @@ pm_when_node_conditions_append(pm_when_node_t *node, pm_node_t *condition) { * Set the location of the then keyword of a when node. */ static inline void -pm_when_node_then_keyword_loc_set(pm_when_node_t *node, const pm_token_t *then_keyword) { +pm_when_node_then_keyword_loc_set(const pm_parser_t *parser, pm_when_node_t *node, const pm_token_t *then_keyword) { node->base.location.end = then_keyword->end; - node->then_keyword_loc = PM_LOCATION_TOKEN_VALUE(then_keyword); + node->then_keyword_loc = TOKEN2SLICE(parser, then_keyword); } /** @@ -6973,9 +6936,9 @@ pm_while_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_to *node = (pm_while_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_WHILE_NODE, flags, keyword, closing), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), - .do_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(do_keyword), - .closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .keyword_loc = TOKEN2SLICE(parser, keyword), + .do_keyword_loc = MAYBETOKEN2SLICE(parser, do_keyword), + .closing_loc = TOKEN2SLICE(parser, closing), .predicate = predicate, .statements = statements }; @@ -6994,7 +6957,7 @@ pm_while_node_modifier_create(pm_parser_t *parser, const pm_token_t *keyword, pm *node = (pm_while_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_WHILE_NODE, flags, statements, predicate), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), + .keyword_loc = TOKEN2SLICE(parser, keyword), .do_keyword_loc = { 0 }, .closing_loc = { 0 }, .predicate = predicate, @@ -7013,9 +6976,9 @@ pm_while_node_synthesized_create(pm_parser_t *parser, pm_node_t *predicate, pm_s *node = (pm_while_node_t) { .base = PM_NODE_INIT_BASE(parser, PM_WHILE_NODE, 0), - .keyword_loc = PM_LOCATION_NULL_VALUE(parser), - .do_keyword_loc = PM_LOCATION_NULL_VALUE(parser), - .closing_loc = PM_LOCATION_NULL_VALUE(parser), + .keyword_loc = { 0 }, + .do_keyword_loc = { 0 }, + .closing_loc = { 0 }, .predicate = predicate, .statements = statements }; @@ -7033,9 +6996,9 @@ pm_xstring_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, *node = (pm_x_string_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_X_STRING_NODE, PM_STRING_FLAGS_FROZEN, opening, closing), - .opening_loc = PM_LOCATION_TOKEN_VALUE(opening), - .content_loc = PM_LOCATION_TOKEN_VALUE(content), - .closing_loc = PM_LOCATION_TOKEN_VALUE(closing), + .opening_loc = TOKEN2SLICE(parser, opening), + .content_loc = TOKEN2SLICE(parser, content), + .closing_loc = TOKEN2SLICE(parser, closing), .unescaped = *unescaped }; @@ -7058,11 +7021,11 @@ pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_lo pm_yield_node_t *node = PM_NODE_ALLOC(parser, pm_yield_node_t); const uint8_t *end; - if (rparen_loc->start != NULL) { + if (rparen_loc != NULL) { end = rparen_loc->end; } else if (arguments != NULL) { end = arguments->base.location.end; - } else if (lparen_loc->start != NULL) { + } else if (lparen_loc != NULL) { end = lparen_loc->end; } else { end = keyword->end; @@ -7070,10 +7033,10 @@ pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_lo *node = (pm_yield_node_t) { .base = PM_NODE_INIT(parser, PM_YIELD_NODE, 0, keyword->start, end), - .keyword_loc = PM_LOCATION_TOKEN_VALUE(keyword), - .lparen_loc = *lparen_loc, + .keyword_loc = TOKEN2SLICE(parser, keyword), + .lparen_loc = MAYBELOCATION2SLICE(parser, lparen_loc), .arguments = arguments, - .rparen_loc = *rparen_loc + .rparen_loc = MAYBELOCATION2SLICE(parser, rparen_loc) }; return node; @@ -8442,8 +8405,8 @@ current_token_starts_line(pm_parser_t *parser) { * handle interpolation. This function performs that check. It returns a token * type representing what it found. Those cases are: * - * * PM_TOKEN_NOT_PROVIDED - No interpolation was found at this point. The - * caller should keep lexing. + * * 0 - No interpolation was found at this point. The caller should keep + * lexing. * * PM_TOKEN_STRING_CONTENT - No interpolation was found at this point. The * caller should return this token type. * * PM_TOKEN_EMBEXPR_BEGIN - An embedded expression was found. The caller @@ -8460,9 +8423,9 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) { return PM_TOKEN_STRING_CONTENT; } - // Now we'll check against the character that follows the #. If it constitutes - // valid interplation, we'll handle that, otherwise we'll return - // PM_TOKEN_NOT_PROVIDED. + // Now we'll check against the character that follows the #. If it + // constitutes valid interplation, we'll handle that, otherwise we'll return + // 0. switch (pound[1]) { case '@': { // In this case we may have hit an embedded instance or class variable. @@ -8496,7 +8459,7 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) { // string content. This is like if we get "#@-". In this case the caller // should keep lexing. parser->current.end = pound + 1; - return PM_TOKEN_NOT_PROVIDED; + return 0; } case '$': // In this case we may have hit an embedded global variable. If there's @@ -8546,7 +8509,7 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) { // In this case we've hit a #$ that does not indicate a global variable. // In this case we'll continue lexing past it. parser->current.end = pound + 1; - return PM_TOKEN_NOT_PROVIDED; + return 0; case '{': // In this case it's the start of an embedded expression. If we have // already consumed content, then we need to return that content as string @@ -8570,7 +8533,7 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) { // mark that by returning the not provided token type. This tells the // consumer to keep lexing forward. parser->current.end = pound + 1; - return PM_TOKEN_NOT_PROVIDED; + return 0; } } @@ -11240,7 +11203,7 @@ parser_lex(pm_parser_t *parser) { if (*breakpoint == '#') { pm_token_type_t type = lex_interpolation(parser, breakpoint); - if (type == PM_TOKEN_NOT_PROVIDED) { + if (!type) { // If we haven't returned at this point then we had something // that looked like an interpolated class or instance variable // like "#@" but wasn't actually. In this case we'll just skip @@ -11492,7 +11455,7 @@ parser_lex(pm_parser_t *parser) { // interpolation. pm_token_type_t type = lex_interpolation(parser, breakpoint); - if (type == PM_TOKEN_NOT_PROVIDED) { + if (!type) { // If we haven't returned at this point then we had // something that looked like an interpolated class or // instance variable like "#@" but wasn't actually. In @@ -11738,7 +11701,7 @@ parser_lex(pm_parser_t *parser) { case '#': { pm_token_type_t type = lex_interpolation(parser, breakpoint); - if (type == PM_TOKEN_NOT_PROVIDED) { + if (!type) { // If we haven't returned at this point then we had something that // looked like an interpolated class or instance variable like "#@" // but wasn't actually. In this case we'll just skip to the next @@ -12059,7 +12022,7 @@ parser_lex(pm_parser_t *parser) { case '#': { pm_token_type_t type = lex_interpolation(parser, breakpoint); - if (type == PM_TOKEN_NOT_PROVIDED) { + if (!type) { // If we haven't returned at this point then we had // something that looked like an interpolated class // or instance variable like "#@" but wasn't @@ -12376,7 +12339,7 @@ expect1(pm_parser_t *parser, pm_token_type_t type, pm_diagnostic_id_t diag_id) { pm_parser_err(parser, location, location, diag_id); parser->previous.start = location; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } /** @@ -12391,7 +12354,7 @@ expect2(pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_di pm_parser_err(parser, location, location, diag_id); parser->previous.start = location; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } /** @@ -12405,7 +12368,7 @@ expect1_heredoc_term(pm_parser_t *parser, const uint8_t *ident_start, size_t ide } else { pm_parser_err_heredoc_term(parser, ident_start, ident_length); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } } @@ -12741,10 +12704,10 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p // target then this is either a method call or a local variable // write. if ( - (call->message_loc.start != NULL) && - (call->message_loc.end[-1] != '!') && - (call->message_loc.end[-1] != '?') && - (call->opening_loc.start == NULL) && + (call->message_loc.length > 0) && + (parser->start[call->message_loc.start + call->message_loc.length - 1] != '!') && + (parser->start[call->message_loc.start + call->message_loc.length - 1] != '?') && + (call->opening_loc.length == 0) && (call->arguments == NULL) && (call->block == NULL) ) { @@ -12758,7 +12721,10 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p // When it was parsed in the prefix position, foo was seen as a // method call with no receiver and no arguments. Now we have an // =, so we know it's a local variable write. - const pm_location_t message_loc = call->message_loc; + const pm_location_t message_loc = { + .start = parser->start + call->message_loc.start, + .end = parser->start + call->message_loc.start + call->message_loc.length + }; pm_constant_id_t name = pm_parser_local_add_location(parser, message_loc.start, message_loc.end, 0); pm_node_destroy(parser, target); @@ -12766,7 +12732,7 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p return UP(pm_local_variable_target_node_create(parser, &message_loc, name, 0)); } - if (*call->message_loc.start == '_' || parser->encoding->alnum_char(call->message_loc.start, call->message_loc.end - call->message_loc.start)) { + if (parser->start[call->message_loc.start] == '_' || parser->encoding->alnum_char(parser->start + call->message_loc.start, call->message_loc.length)) { if (multiple && PM_NODE_FLAG_P(call, PM_CALL_NODE_FLAGS_SAFE_NAVIGATION)) { pm_parser_err_node(parser, (const pm_node_t *) call, PM_ERR_UNEXPECTED_SAFE_NAVIGATION); } @@ -12926,10 +12892,10 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod // target then this is either a method call or a local variable // write. if ( - (call->message_loc.start != NULL) && - (call->message_loc.end[-1] != '!') && - (call->message_loc.end[-1] != '?') && - (call->opening_loc.start == NULL) && + (call->message_loc.length > 0) && + (parser->start[call->message_loc.start + call->message_loc.length - 1] != '!') && + (parser->start[call->message_loc.start + call->message_loc.length - 1] != '?') && + (call->opening_loc.length == 0) && (call->arguments == NULL) && (call->block == NULL) ) { @@ -12943,7 +12909,10 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod // When it was parsed in the prefix position, foo was seen as a // method call with no receiver and no arguments. Now we have an // =, so we know it's a local variable write. - const pm_location_t message = call->message_loc; + const pm_location_t message = { + .start = parser->start + call->message_loc.start, + .end = parser->start + call->message_loc.start + call->message_loc.length + }; pm_parser_local_add_location(parser, message.start, message.end, 0); pm_node_destroy(parser, target); @@ -12955,7 +12924,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod return target; } - if (char_is_identifier_start(parser, call->message_loc.start, parser->end - call->message_loc.start)) { + if (char_is_identifier_start(parser, parser->start + call->message_loc.start, call->message_loc.length)) { // When we get here, we have a method call, because it was // previously marked as a method call but now we have an =. This // looks like: @@ -12971,7 +12940,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod pm_arguments_node_arguments_append(arguments, value); call->base.location.end = arguments->base.location.end; - call->equal_loc = PM_LOCATION_TOKEN_VALUE(operator); + call->equal_loc = TOKEN2SLICE(parser, operator); parse_write_name(parser, &call->name); pm_node_flag_set(UP(call), PM_CALL_NODE_FLAGS_ATTRIBUTE_WRITE | pm_implicit_array_write_flags(value, PM_CALL_NODE_FLAGS_IMPLICIT_ARRAY)); @@ -12993,7 +12962,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod // Replace the name with "[]=". call->name = pm_parser_constant_id_constant(parser, "[]=", 3); - call->equal_loc = PM_LOCATION_TOKEN_VALUE(operator); + call->equal_loc = TOKEN2SLICE(parser, operator); // Ensure that the arguments for []= don't contain keywords pm_index_arguments_check(parser, call->arguments, call->block); @@ -13208,7 +13177,7 @@ parse_statements(pm_parser_t *parser, pm_context_t context, uint16_t depth) { // probably extract a helper function. PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } } @@ -13314,7 +13283,6 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod pm_node_t *key = UP(pm_symbol_node_label_create(parser, &label)); pm_hash_key_static_literals_add(parser, literals, key); - pm_token_t operator = not_provided(parser); pm_node_t *value = NULL; if (token_begins_expression_p(parser->current.type)) { @@ -13344,7 +13312,7 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod value = UP(pm_implicit_node_create(parser, value)); } - element = UP(pm_assoc_node_create(parser, key, &operator, value)); + element = UP(pm_assoc_node_create(parser, key, NULL, value)); break; } default: { @@ -13358,16 +13326,14 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod pm_hash_key_static_literals_add(parser, literals, key); - pm_token_t operator; - if (pm_symbol_node_label_p(key)) { - operator = not_provided(parser); - } else { + pm_token_t operator = { 0 }; + if (!pm_symbol_node_label_p(parser, key)) { expect1(parser, PM_TOKEN_EQUAL_GREATER, PM_ERR_HASH_ROCKET); operator = parser->previous; } pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1)); - element = UP(pm_assoc_node_create(parser, key, &operator, value)); + element = UP(pm_assoc_node_create(parser, key, MAYBETOKENPTR(operator), value)); break; } } @@ -13525,7 +13491,8 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for // ... operator. if (PM_NODE_TYPE_P(right, PM_RANGE_NODE)) { pm_range_node_t *range = (pm_range_node_t *) right; - pm_parser_err(parser, range->operator_loc.start, range->operator_loc.end, PM_ERR_UNEXPECTED_RANGE_OPERATOR); + const uint8_t *start = parser->start + range->operator_loc.start; + pm_parser_err(parser, start, start + range->operator_loc.length, PM_ERR_UNEXPECTED_RANGE_OPERATOR); } argument = UP(pm_range_node_create(parser, NULL, &operator, right)); @@ -13553,16 +13520,14 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for bool contains_keywords = false; bool contains_keyword_splat = false; - if (pm_symbol_node_label_p(argument) || accept1(parser, PM_TOKEN_EQUAL_GREATER)) { + if (pm_symbol_node_label_p(parser, argument) || accept1(parser, PM_TOKEN_EQUAL_GREATER)) { if (parsed_bare_hash) { pm_parser_err_previous(parser, PM_ERR_ARGUMENT_BARE_HASH); } - pm_token_t operator; + pm_token_t operator = { 0 }; if (parser->previous.type == PM_TOKEN_EQUAL_GREATER) { operator = parser->previous; - } else { - operator = not_provided(parser); } pm_keyword_hash_node_t *bare_hash = pm_keyword_hash_node_create(parser); @@ -13574,7 +13539,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for // Finish parsing the one we are part way through. pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1)); - argument = UP(pm_assoc_node_create(parser, argument, &operator, value)); + argument = UP(pm_assoc_node_create(parser, argument, MAYBETOKENPTR(operator), value)); pm_keyword_hash_node_elements_append(bare_hash, argument); argument = UP(bare_hash); @@ -13631,7 +13596,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for // `foo(bar 1 do end, 2)` should be rejected. if (PM_NODE_TYPE_P(argument, PM_CALL_NODE)) { pm_call_node_t *call = (pm_call_node_t *) argument; - if (call->opening_loc.start == NULL && call->arguments != NULL && call->block != NULL) { + if (call->opening_loc.length == 0 && call->arguments != NULL && call->block != NULL) { pm_parser_err_previous(parser, PM_ERR_INVALID_COMMA); break; } @@ -13663,7 +13628,7 @@ parse_required_destructured_parameter(pm_parser_t *parser) { expect1(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_ERR_EXPECT_LPAREN_REQ_PARAMETER); pm_multi_target_node_t *node = pm_multi_target_node_create(parser); - pm_multi_target_node_opening_set(node, &parser->previous); + pm_multi_target_node_opening_set(parser, node, &parser->previous); do { pm_node_t *param; @@ -13711,7 +13676,7 @@ parse_required_destructured_parameter(pm_parser_t *parser) { accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN_REQ_PARAMETER); - pm_multi_target_node_closing_set(node, &parser->previous); + pm_multi_target_node_closing_set(parser, node, &parser->previous); return node; } @@ -13827,7 +13792,7 @@ parse_parameters( parser_lex(parser); pm_token_t operator = parser->previous; - pm_token_t name; + pm_token_t name = { 0 }; bool repeated = false; if (accept1(parser, PM_TOKEN_IDENTIFIER)) { @@ -13835,11 +13800,10 @@ parse_parameters( repeated = pm_parser_parameter_name_check(parser, &name); pm_parser_local_add_token(parser, &name, 1); } else { - name = not_provided(parser); parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_BLOCK; } - pm_block_parameter_node_t *param = pm_block_parameter_node_create(parser, &name, &operator); + pm_block_parameter_node_t *param = pm_block_parameter_node_create(parser, MAYBETOKENPTR(name), &operator); if (repeated) { pm_node_flag_set_repeated_parameter(UP(param)); } @@ -14060,7 +14024,7 @@ parse_parameters( parser_lex(parser); pm_token_t operator = parser->previous; - pm_token_t name; + pm_token_t name = { 0 }; bool repeated = false; if (accept1(parser, PM_TOKEN_IDENTIFIER)) { @@ -14068,11 +14032,10 @@ parse_parameters( repeated = pm_parser_parameter_name_check(parser, &name); pm_parser_local_add_token(parser, &name, 1); } else { - name = not_provided(parser); parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS; } - pm_node_t *param = UP(pm_rest_parameter_node_create(parser, &operator, &name)); + pm_node_t *param = UP(pm_rest_parameter_node_create(parser, &operator, MAYBETOKENPTR(name))); if (repeated) { pm_node_flag_set_repeated_parameter(param); } @@ -14102,7 +14065,7 @@ parse_parameters( param = UP(pm_no_keywords_parameter_node_create(parser, &operator, &parser->previous)); } else { - pm_token_t name; + pm_token_t name = { 0 }; bool repeated = false; if (accept1(parser, PM_TOKEN_IDENTIFIER)) { @@ -14110,11 +14073,10 @@ parse_parameters( repeated = pm_parser_parameter_name_check(parser, &name); pm_parser_local_add_token(parser, &name, 1); } else { - name = not_provided(parser); parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS; } - param = UP(pm_keyword_rest_parameter_node_create(parser, &operator, &name)); + param = UP(pm_keyword_rest_parameter_node_create(parser, &operator, MAYBETOKENPTR(name))); if (repeated) { pm_node_flag_set_repeated_parameter(param); } @@ -14315,7 +14277,7 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ // we're going to have an empty list of exceptions to rescue (which // implies StandardError). parser_lex(parser); - pm_rescue_node_operator_set(rescue, &parser->previous); + pm_rescue_node_operator_set(parser, rescue, &parser->previous); pm_node_t *reference = parse_expression(parser, PM_BINDING_POWER_INDEX, false, false, PM_ERR_RESCUE_VARIABLE, (uint16_t) (depth + 1)); reference = parse_target(parser, reference, false, false); @@ -14345,7 +14307,7 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ // If we hit a `=>` then we're going to parse the exception variable. Once // we've done that, we'll break out of the loop and parse the statements. if (accept1(parser, PM_TOKEN_EQUAL_GREATER)) { - pm_rescue_node_operator_set(rescue, &parser->previous); + pm_rescue_node_operator_set(parser, rescue, &parser->previous); pm_node_t *reference = parse_expression(parser, PM_BINDING_POWER_INDEX, false, false, PM_ERR_RESCUE_VARIABLE, (uint16_t) (depth + 1)); reference = parse_target(parser, reference, false, false); @@ -14360,11 +14322,11 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) { - rescue->then_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(&parser->previous); + rescue->then_keyword_loc = TOKEN2SLICE(parser, &parser->previous); } } else { expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_RESCUE_TERM); - rescue->then_keyword_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(&parser->previous); + rescue->then_keyword_loc = TOKEN2SLICE(parser, &parser->previous); } if (!match3(parser, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_END)) { @@ -14487,10 +14449,10 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ if (match1(parser, PM_TOKEN_KEYWORD_END)) { if (opening != NULL) parser_warn_indentation_mismatch(parser, opening_newline_index, opening, false, false); - pm_begin_node_end_keyword_set(parent_node, &parser->current); + pm_begin_node_end_keyword_set(parser, parent_node, &parser->current); } else { - pm_token_t end_keyword = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; - pm_begin_node_end_keyword_set(parent_node, &end_keyword); + pm_token_t end_keyword = (pm_token_t) { .type = PM_TOKEN_KEYWORD_END, .start = parser->previous.end, .end = parser->previous.end }; + pm_begin_node_end_keyword_set(parser, parent_node, &end_keyword); } } @@ -14500,9 +14462,7 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ */ static pm_begin_node_t * parse_rescues_implicit_begin(pm_parser_t *parser, size_t opening_newline_index, const pm_token_t *opening, const uint8_t *start, pm_statements_node_t *statements, pm_rescues_type_t type, uint16_t depth) { - pm_token_t begin_keyword = not_provided(parser); - pm_begin_node_t *node = pm_begin_node_create(parser, &begin_keyword, statements); - + pm_begin_node_t *node = pm_begin_node_create(parser, NULL, statements); parse_rescues(parser, opening_newline_index, opening, node, type, (uint16_t) (depth + 1)); node->base.location.start = start; @@ -14542,7 +14502,7 @@ parse_block_parameters( } pm_block_parameters_node_t *block_parameters = pm_block_parameters_node_create(parser, parameters, opening); - if ((opening->type != PM_TOKEN_NOT_PROVIDED)) { + if (opening != NULL) { accept1(parser, PM_TOKEN_NEWLINE); if (accept1(parser, PM_TOKEN_SEMICOLON)) { @@ -14713,7 +14673,7 @@ parse_block(pm_parser_t *parser, uint16_t depth) { expect1(parser, PM_TOKEN_PIPE, PM_ERR_BLOCK_PARAM_PIPE_TERM); } - pm_block_parameters_node_closing_set(block_parameters, &parser->previous); + pm_block_parameters_node_closing_set(parser, block_parameters, &parser->previous); } accept1(parser, PM_TOKEN_NEWLINE); @@ -14774,7 +14734,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_ARGUMENT_TERM_PAREN, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } pm_accepts_block_stack_pop(parser); @@ -15097,7 +15057,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl pm_node_list_t *previous_block_exits = push_block_exits(parser, ¤t_block_exits); pm_token_t keyword = parser->previous; - pm_token_t then_keyword = not_provided(parser); + pm_token_t then_keyword = { 0 }; pm_node_t *predicate = parse_predicate(parser, PM_BINDING_POWER_MODIFIER, context, &then_keyword, (uint16_t) (depth + 1)); pm_statements_node_t *statements = NULL; @@ -15109,15 +15069,14 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON); } - pm_token_t end_keyword = not_provided(parser); pm_node_t *parent = NULL; switch (context) { case PM_CONTEXT_IF: - parent = UP(pm_if_node_create(parser, &keyword, predicate, &then_keyword, statements, NULL, &end_keyword)); + parent = UP(pm_if_node_create(parser, &keyword, predicate, MAYBETOKENPTR(then_keyword), statements, NULL, NULL)); break; case PM_CONTEXT_UNLESS: - parent = UP(pm_unless_node_create(parser, &keyword, predicate, &then_keyword, statements)); + parent = UP(pm_unless_node_create(parser, &keyword, predicate, MAYBETOKENPTR(then_keyword), statements)); break; default: assert(false && "unreachable"); @@ -15145,7 +15104,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl pm_accepts_block_stack_pop(parser); accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON); - pm_node_t *elsif = UP(pm_if_node_create(parser, &elsif_keyword, predicate, &then_keyword, statements, NULL, &end_keyword)); + pm_node_t *elsif = UP(pm_if_node_create(parser, &elsif_keyword, predicate, MAYBETOKENPTR(then_keyword), statements, NULL, NULL)); ((pm_if_node_t *) current)->subsequent = elsif; current = elsif; } @@ -15193,12 +15152,12 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl while (recursing) { switch (PM_NODE_TYPE(current)) { case PM_IF_NODE: - pm_if_node_end_keyword_loc_set((pm_if_node_t *) current, &parser->previous); + pm_if_node_end_keyword_loc_set(parser, (pm_if_node_t *) current, &parser->previous); current = ((pm_if_node_t *) current)->subsequent; recursing = current != NULL; break; case PM_ELSE_NODE: - pm_else_node_end_keyword_loc_set((pm_else_node_t *) current, &parser->previous); + pm_else_node_end_keyword_loc_set(parser, (pm_else_node_t *) current, &parser->previous); recursing = false; break; default: { @@ -15210,7 +15169,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl break; } case PM_CONTEXT_UNLESS: - pm_unless_node_end_keyword_loc_set((pm_unless_node_t *) parent, &parser->previous); + pm_unless_node_end_keyword_loc_set(parser, (pm_unless_node_t *) parent, &parser->previous); break; default: assert(false && "unreachable"); @@ -15325,10 +15284,7 @@ parse_string_part(pm_parser_t *parser, uint16_t depth) { // "aaa #{bbb} #@ccc ddd" // ^^^^ ^ ^^^^ case PM_TOKEN_STRING_CONTENT: { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - - pm_node_t *node = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing)); + pm_node_t *node = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); pm_node_flag_set(node, parse_unescaped_encoding(parser)); parser_lex(parser); @@ -15363,9 +15319,7 @@ parse_string_part(pm_parser_t *parser, uint16_t depth) { parser->brace_nesting = brace_nesting; lex_state_set(parser, state); - expect1(parser, PM_TOKEN_EMBEXPR_END, PM_ERR_EMBEXPR_END); - pm_token_t closing = parser->previous; // If this set of embedded statements only contains a single // statement, then Ruby does not consider it as a possible statement @@ -15374,7 +15328,7 @@ parse_string_part(pm_parser_t *parser, uint16_t depth) { pm_node_flag_unset(statements->body.nodes[0], PM_NODE_FLAG_NEWLINE); } - return UP(pm_embedded_statements_node_create(parser, &opening, statements, &closing)); + return UP(pm_embedded_statements_node_create(parser, &opening, statements, &parser->previous)); } // Here the lexer has returned the beginning of an embedded variable. @@ -15462,9 +15416,7 @@ parse_operator_symbol_name(const pm_token_t *name) { static pm_node_t * parse_operator_symbol(pm_parser_t *parser, const pm_token_t *opening, pm_lex_state_t next_state) { - pm_token_t closing = not_provided(parser); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, opening, &parser->current, &closing); - + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, opening, &parser->current, NULL); const uint8_t *end = parse_operator_symbol_name(&parser->current); if (next_state != PM_LEX_STATE_NONE) lex_state_set(parser, next_state); @@ -15507,9 +15459,7 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s break; } - pm_token_t closing = not_provided(parser); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, &closing); - + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, NULL); pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end); pm_node_flag_set(UP(symbol), parse_symbol_encoding(parser, &parser->previous, &symbol->unescaped, false)); @@ -15521,10 +15471,13 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s if (match1(parser, PM_TOKEN_STRING_END)) { if (next_state != PM_LEX_STATE_NONE) lex_state_set(parser, next_state); parser_lex(parser); + pm_token_t content = { + .type = PM_TOKEN_STRING_CONTENT, + .start = parser->previous.start, + .end = parser->previous.start + }; - pm_token_t content = not_provided(parser); - pm_token_t closing = parser->previous; - return UP(pm_symbol_node_create(parser, &opening, &content, &closing)); + return UP(pm_symbol_node_create(parser, &opening, &content, &parser->previous)); } // Now we can parse the first part of the symbol. @@ -15555,7 +15508,7 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_INTERPOLATED); } - pm_interpolated_symbol_node_closing_loc_set(symbol, &parser->previous); + pm_interpolated_symbol_node_closing_loc_set(parser, symbol, &parser->previous); return UP(symbol); } @@ -15578,12 +15531,10 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s // interpolated string node, so that's what we'll do here. if (match1(parser, PM_TOKEN_STRING_CONTENT)) { pm_interpolated_symbol_node_t *symbol = pm_interpolated_symbol_node_create(parser, &opening, NULL, &opening); - pm_token_t bounds = not_provided(parser); - - pm_node_t *part = UP(pm_string_node_create_unescaped(parser, &bounds, &content, &bounds, &unescaped)); + pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &unescaped)); pm_interpolated_symbol_node_append(symbol, part); - part = UP(pm_string_node_create_unescaped(parser, &bounds, &parser->current, &bounds, &parser->current_string)); + part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->current, NULL, &parser->current_string)); pm_interpolated_symbol_node_append(symbol, part); if (next_state != PM_LEX_STATE_NONE) { @@ -15593,7 +15544,7 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s parser_lex(parser); expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_DYNAMIC); - pm_interpolated_symbol_node_closing_loc_set(symbol, &parser->previous); + pm_interpolated_symbol_node_closing_loc_set(parser, symbol, &parser->previous); return UP(symbol); } } else { @@ -15621,20 +15572,15 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s static inline pm_node_t * parse_undef_argument(pm_parser_t *parser, uint16_t depth) { switch (parser->current.type) { - case PM_CASE_OPERATOR: { - const pm_token_t opening = not_provided(parser); - return parse_operator_symbol(parser, &opening, PM_LEX_STATE_NONE); - } + case PM_CASE_OPERATOR: + return parse_operator_symbol(parser, NULL, PM_LEX_STATE_NONE); case PM_CASE_KEYWORD: case PM_TOKEN_CONSTANT: case PM_TOKEN_IDENTIFIER: case PM_TOKEN_METHOD_NAME: { parser_lex(parser); - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, &closing); - + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, NULL, &parser->previous, NULL); pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end); pm_node_flag_set(UP(symbol), parse_symbol_encoding(parser, &parser->previous, &symbol->unescaped, false)); @@ -15661,10 +15607,8 @@ parse_undef_argument(pm_parser_t *parser, uint16_t depth) { static inline pm_node_t * parse_alias_argument(pm_parser_t *parser, bool first, uint16_t depth) { switch (parser->current.type) { - case PM_CASE_OPERATOR: { - const pm_token_t opening = not_provided(parser); - return parse_operator_symbol(parser, &opening, first ? PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM : PM_LEX_STATE_NONE); - } + case PM_CASE_OPERATOR: + return parse_operator_symbol(parser, NULL, first ? PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM : PM_LEX_STATE_NONE); case PM_CASE_KEYWORD: case PM_TOKEN_CONSTANT: case PM_TOKEN_IDENTIFIER: @@ -15672,10 +15616,7 @@ parse_alias_argument(pm_parser_t *parser, bool first, uint16_t depth) { if (first) lex_state_set(parser, PM_LEX_STATE_FNAME | PM_LEX_STATE_FITEM); parser_lex(parser); - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &parser->previous, &closing); - + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, NULL, &parser->previous, NULL); pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.end); pm_node_flag_set(UP(symbol), parse_symbol_encoding(parser, &parser->previous, &symbol->unescaped, false)); @@ -15789,7 +15730,7 @@ parse_method_definition_name(pm_parser_t *parser) { return parser->previous; default: PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_DEF_NAME, pm_token_type_human(parser->current.type)); - return (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->current.start, .end = parser->current.end }; + return (pm_token_t) { .type = 0, .start = parser->current.start, .end = parser->current.end }; } } @@ -15917,10 +15858,8 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 // If we get here, then we have an end of a label immediately // after a start. In that case we'll create an empty symbol // node. - pm_token_t content = parse_strings_empty_content(parser->previous.start); - pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, &content, &parser->previous); - - pm_string_shared_init(&symbol->unescaped, content.start, content.end); + pm_symbol_node_t *symbol = pm_symbol_node_create(parser, &opening, NULL, &parser->previous); + pm_string_shared_init(&symbol->unescaped, parser->previous.start, parser->previous.start); node = UP(symbol); if (!label_allowed) pm_parser_err_node(parser, node, PM_ERR_UNEXPECTED_LABEL); @@ -15932,7 +15871,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 if (match1(parser, PM_TOKEN_EOF)) { unescaped = PM_STRING_EMPTY; - content = not_provided(parser); + content = (pm_token_t) { .type = PM_TOKEN_STRING_CONTENT, .start = parser->start, .end = parser->start }; } else { unescaped = parser->current_string; expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_EXPECT_STRING_CONTENT); @@ -15952,13 +15891,11 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 // be able to contain all of the parts. if (match1(parser, PM_TOKEN_STRING_CONTENT)) { pm_node_list_t parts = { 0 }; - - pm_token_t delimiters = not_provided(parser); - pm_node_t *part = UP(pm_string_node_create_unescaped(parser, &delimiters, &content, &delimiters, &unescaped)); + pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &unescaped)); pm_node_list_append(&parts, part); do { - part = UP(pm_string_node_create_current_string(parser, &delimiters, &parser->current, &delimiters)); + part = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); pm_node_list_append(&parts, part); parser_lex(parser); } while (match1(parser, PM_TOKEN_STRING_CONTENT)); @@ -15978,7 +15915,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 } else { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_STRING_LITERAL_TERM, pm_token_type_human(parser->previous.type)); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped)); } } else if (match1(parser, PM_TOKEN_STRING_CONTENT)) { @@ -16004,7 +15941,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 pm_parser_err(parser, location, location, PM_ERR_STRING_LITERAL_EOF); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } } else if (accept1(parser, PM_TOKEN_LABEL_END)) { node = UP(pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &content, &unescaped, true))); @@ -16013,10 +15950,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 // If we get here, then we have interpolation so we'll need // to create a string or symbol node with interpolation. pm_node_list_t parts = { 0 }; - pm_token_t string_opening = not_provided(parser); - pm_token_t string_closing = not_provided(parser); - - pm_node_t *part = UP(pm_string_node_create_unescaped(parser, &string_opening, &parser->previous, &string_closing, &unescaped)); + pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->previous, NULL, &unescaped)); pm_node_flag_set(part, parse_unescaped_encoding(parser)); pm_node_list_append(&parts, part); @@ -16093,9 +16027,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 } concating = true; - pm_token_t bounds = not_provided(parser); - - pm_interpolated_string_node_t *container = pm_interpolated_string_node_create(parser, &bounds, NULL, &bounds); + pm_interpolated_string_node_t *container = pm_interpolated_string_node_create(parser, NULL, NULL, NULL); pm_interpolated_string_node_append(container, current); current = UP(container); } @@ -16194,13 +16126,13 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures case PM_ARRAY_PATTERN_NODE: { pm_array_pattern_node_t *pattern_node = (pm_array_pattern_node_t *) inner; - if (pattern_node->constant == NULL && pattern_node->opening_loc.start == NULL) { + if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { pattern_node->base.location.start = node->location.start; pattern_node->base.location.end = closing.end; pattern_node->constant = node; - pattern_node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pattern_node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + pattern_node->opening_loc = TOKEN2SLICE(parser, &opening); + pattern_node->closing_loc = TOKEN2SLICE(parser, &closing); return UP(pattern_node); } @@ -16210,13 +16142,13 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures case PM_FIND_PATTERN_NODE: { pm_find_pattern_node_t *pattern_node = (pm_find_pattern_node_t *) inner; - if (pattern_node->constant == NULL && pattern_node->opening_loc.start == NULL) { + if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { pattern_node->base.location.start = node->location.start; pattern_node->base.location.end = closing.end; pattern_node->constant = node; - pattern_node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pattern_node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + pattern_node->opening_loc = TOKEN2SLICE(parser, &opening); + pattern_node->closing_loc = TOKEN2SLICE(parser, &closing); return UP(pattern_node); } @@ -16226,13 +16158,13 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures case PM_HASH_PATTERN_NODE: { pm_hash_pattern_node_t *pattern_node = (pm_hash_pattern_node_t *) inner; - if (pattern_node->constant == NULL && pattern_node->opening_loc.start == NULL) { + if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { pattern_node->base.location.start = node->location.start; pattern_node->base.location.end = closing.end; pattern_node->constant = node; - pattern_node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pattern_node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + pattern_node->opening_loc = TOKEN2SLICE(parser, &opening); + pattern_node->closing_loc = TOKEN2SLICE(parser, &closing); return UP(pattern_node); } @@ -16353,8 +16285,13 @@ pm_slice_is_valid_local(const pm_parser_t *parser, const uint8_t *start, const u */ static pm_node_t * parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_symbol_node_t *key) { - const pm_location_t *value_loc = &((pm_symbol_node_t *) key)->value_loc; - + const pm_slice_t *value_slice = &((pm_symbol_node_t *) key)->value_loc; + const pm_location_t value_location = { + .start = parser->start + value_slice->start, + .end = parser->start + value_slice->start + value_slice->length + }; + const pm_location_t *value_loc = &value_location; + pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); int depth = -1; @@ -16409,7 +16346,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node rest = first_node; break; case PM_SYMBOL_NODE: { - if (pm_symbol_node_label_p(first_node)) { + if (pm_symbol_node_label_p(parser, first_node)) { parse_pattern_hash_key(parser, &keys, first_node); pm_node_t *value; @@ -16423,9 +16360,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY, (uint16_t) (depth + 1)); } - pm_token_t operator = not_provided(parser); - pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, &operator, value)); - + pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, NULL, value)); pm_node_list_append(&assocs, assoc); break; } @@ -16438,9 +16373,8 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node pm_diagnostic_id_t diag_id = PM_NODE_TYPE_P(first_node, PM_INTERPOLATED_SYMBOL_NODE) ? PM_ERR_PATTERN_HASH_KEY_INTERPOLATED : PM_ERR_PATTERN_HASH_KEY_LABEL; pm_parser_err_node(parser, first_node, diag_id); - pm_token_t operator = not_provided(parser); pm_node_t *value = UP(pm_missing_node_create(parser, first_node->location.start, first_node->location.end)); - pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, &operator, value)); + pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, NULL, value)); pm_node_list_append(&assocs, assoc); break; @@ -16476,12 +16410,16 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node if (PM_NODE_TYPE_P(key, PM_INTERPOLATED_SYMBOL_NODE)) { pm_parser_err_node(parser, key, PM_ERR_PATTERN_HASH_KEY_INTERPOLATED); - } else if (!pm_symbol_node_label_p(key)) { + } else if (!pm_symbol_node_label_p(parser, key)) { pm_parser_err_node(parser, key, PM_ERR_PATTERN_LABEL_AFTER_COMMA); } + } else if (accept1(parser, PM_TOKEN_LABEL)) { + key = UP(pm_symbol_node_label_create(parser, &parser->previous)); } else { expect1(parser, PM_TOKEN_LABEL, PM_ERR_PATTERN_LABEL_AFTER_COMMA); - key = UP(pm_symbol_node_label_create(parser, &parser->previous)); + + pm_token_t label = { .type = PM_TOKEN_LABEL, .start = parser->previous.end, .end = parser->previous.end }; + key = UP(pm_symbol_node_create(parser, NULL, &label, NULL)); } parse_pattern_hash_key(parser, &keys, key); @@ -16497,8 +16435,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY, (uint16_t) (depth + 1)); } - pm_token_t operator = not_provided(parser); - pm_node_t *assoc = UP(pm_assoc_node_create(parser, key, &operator, value)); + pm_node_t *assoc = UP(pm_assoc_node_create(parser, key, NULL, value)); if (rest != NULL) { pm_parser_err_node(parser, assoc, PM_ERR_PATTERN_EXPRESSION_AFTER_REST); @@ -16560,12 +16497,12 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm switch (PM_NODE_TYPE(inner)) { case PM_ARRAY_PATTERN_NODE: { pm_array_pattern_node_t *pattern_node = (pm_array_pattern_node_t *) inner; - if (pattern_node->opening_loc.start == NULL) { + if (pattern_node->opening_loc.length == 0) { pattern_node->base.location.start = opening.start; pattern_node->base.location.end = closing.end; - pattern_node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pattern_node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + pattern_node->opening_loc = TOKEN2SLICE(parser, &opening); + pattern_node->closing_loc = TOKEN2SLICE(parser, &closing); return UP(pattern_node); } @@ -16574,12 +16511,12 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm } case PM_FIND_PATTERN_NODE: { pm_find_pattern_node_t *pattern_node = (pm_find_pattern_node_t *) inner; - if (pattern_node->opening_loc.start == NULL) { + if (pattern_node->opening_loc.length == 0) { pattern_node->base.location.start = opening.start; pattern_node->base.location.end = closing.end; - pattern_node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pattern_node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + pattern_node->opening_loc = TOKEN2SLICE(parser, &opening); + pattern_node->closing_loc = TOKEN2SLICE(parser, &closing); return UP(pattern_node); } @@ -16638,8 +16575,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm node->base.location.start = opening.start; node->base.location.end = closing.end; - node->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - node->closing_loc = PM_LOCATION_TOKEN_VALUE(&closing); + node->opening_loc = TOKEN2SLICE(parser, &opening); + node->closing_loc = TOKEN2SLICE(parser, &closing); } parser->pattern_matching_newlines = previous_pattern_matching_newlines; @@ -16668,7 +16605,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pm_node_t *node = parse_expression(parser, PM_BINDING_POWER_MAX, false, true, diag_id, (uint16_t) (depth + 1)); // If we found a label, we need to immediately return to the caller. - if (pm_symbol_node_label_p(node)) return node; + if (pm_symbol_node_label_p(parser, node)) return node; // Call nodes (arithmetic operations) are not allowed in patterns if (PM_NODE_TYPE(node) == PM_CALL_NODE) { @@ -16946,7 +16883,7 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flag // be dynamic symbols leading to hash patterns. node = parse_pattern_primitive(parser, captures, diag_id, (uint16_t) (depth + 1)); - if (pm_symbol_node_label_p(node)) { + if (pm_symbol_node_label_p(parser, node)) { node = UP(parse_pattern_hash(parser, captures, node, (uint16_t) (depth + 1))); if (!(flags & PM_PARSE_PATTERN_TOP)) { @@ -16975,7 +16912,7 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flag // If we got a dynamic label symbol, then we need to treat it like the // beginning of a hash pattern. - if (pm_symbol_node_label_p(node)) { + if (pm_symbol_node_label_p(parser, node)) { return UP(parse_pattern_hash(parser, captures, node, (uint16_t) (depth + 1))); } @@ -17393,7 +17330,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b PM_PARSER_ERR_FORMAT(parser, location, location, PM_ERR_ARRAY_SEPARATOR, pm_token_type_human(parser->current.type)); parser->previous.start = location; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } } @@ -17432,7 +17369,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } else { element = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, true, PM_ERR_ARRAY_EXPRESSION, (uint16_t) (depth + 1)); - if (pm_symbol_node_label_p(element) || accept1(parser, PM_TOKEN_EQUAL_GREATER)) { + if (pm_symbol_node_label_p(parser, element) || accept1(parser, PM_TOKEN_EQUAL_GREATER)) { if (parsed_bare_hash) { pm_parser_err_previous(parser, PM_ERR_EXPRESSION_BARE_HASH); } @@ -17441,15 +17378,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_static_literals_t hash_keys = { 0 }; pm_hash_key_static_literals_add(parser, &hash_keys, element); - pm_token_t operator; + pm_token_t operator = { 0 }; if (parser->previous.type == PM_TOKEN_EQUAL_GREATER) { operator = parser->previous; - } else { - operator = not_provided(parser); } pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1)); - pm_node_t *assoc = UP(pm_assoc_node_create(parser, element, &operator, value)); + pm_node_t *assoc = UP(pm_assoc_node_create(parser, element, MAYBETOKENPTR(operator), value)); pm_keyword_hash_node_elements_append(hash, assoc); element = UP(hash); @@ -17471,7 +17406,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_ARRAY_TERM, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } pm_array_node_close_set(parser, array, &parser->previous); @@ -17556,20 +17491,17 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // multiple target node. pm_multi_target_node_t *multi_target; - if (PM_NODE_TYPE_P(statement, PM_MULTI_TARGET_NODE) && ((pm_multi_target_node_t *) statement)->lparen_loc.start == NULL) { + if (PM_NODE_TYPE_P(statement, PM_MULTI_TARGET_NODE) && ((pm_multi_target_node_t *) statement)->lparen_loc.length == 0) { multi_target = (pm_multi_target_node_t *) statement; } else { multi_target = pm_multi_target_node_create(parser); pm_multi_target_node_targets_append(parser, multi_target, statement); } - pm_location_t lparen_loc = PM_LOCATION_TOKEN_VALUE(&opening); - pm_location_t rparen_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); - - multi_target->lparen_loc = lparen_loc; - multi_target->rparen_loc = rparen_loc; - multi_target->base.location.start = lparen_loc.start; - multi_target->base.location.end = rparen_loc.end; + multi_target->lparen_loc = TOKEN2SLICE(parser, &opening); + multi_target->rparen_loc = TOKEN2SLICE(parser, &parser->previous); + multi_target->base.location.start = opening.start; + multi_target->base.location.end = parser->previous.end; pm_node_t *result; if (match1(parser, PM_TOKEN_COMMA) && (binding_power == PM_BINDING_POWER_STATEMENT)) { @@ -17722,12 +17654,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_accepts_block_stack_pop(parser); expect1(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_HASH_TERM); - pm_hash_node_closing_loc_set(node, &parser->previous); + pm_hash_node_closing_loc_set(parser, node, &parser->previous); return UP(node); } case PM_TOKEN_CHARACTER_LITERAL: { - pm_token_t closing = not_provided(parser); pm_node_t *node = UP(pm_string_node_create_current_string( parser, &(pm_token_t) { @@ -17740,7 +17671,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b .start = parser->current.start + 1, .end = parser->current.end }, - &closing + NULL )); pm_node_flag_set(node, parse_unescaped_encoding(parser)); @@ -17885,14 +17816,14 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // variable call bit in the flags. pm_node_flag_unset(UP(call), PM_CALL_NODE_FLAGS_VARIABLE_CALL); - call->opening_loc = arguments.opening_loc; + call->opening_loc = arguments.opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments.opening_loc); call->arguments = arguments.arguments; - call->closing_loc = arguments.closing_loc; + call->closing_loc = arguments.closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments.closing_loc); call->block = arguments.block; const uint8_t *end = pm_arguments_end(&arguments); if (!end) { - end = call->message_loc.end; + end = parser->start + call->message_loc.start + call->message_loc.length; } call->base.location.end = end; } @@ -17984,9 +17915,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_flag_set(part, parse_unescaped_encoding(parser)); pm_string_node_t *cast = (pm_string_node_t *) part; - cast->opening_loc = PM_LOCATION_TOKEN_VALUE(&opening); - cast->closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->current); - cast->base.location = cast->opening_loc; + cast->opening_loc = TOKEN2SLICE(parser, &opening); + cast->closing_loc = TOKEN2SLICE(parser, &parser->current); + cast->base.location = (pm_location_t) { + .start = parser->start + cast->opening_loc.start, + .end = parser->start + cast->opening_loc.start + cast->opening_loc.length + }; if (lex_mode.quote == PM_HEREDOC_QUOTE_BACKTICK) { assert(sizeof(pm_string_node_t) == sizeof(pm_x_string_node_t)); @@ -18019,18 +17953,24 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b cast->parts = parts; expect1_heredoc_term(parser, lex_mode.ident_start, lex_mode.ident_length); - pm_interpolated_xstring_node_closing_set(cast, &parser->previous); + pm_interpolated_xstring_node_closing_set(parser, cast, &parser->previous); - cast->base.location = cast->opening_loc; + cast->base.location = (pm_location_t) { + .start = parser->start + cast->opening_loc.start, + .end = parser->start + cast->opening_loc.start + cast->opening_loc.length + }; node = UP(cast); } else { pm_interpolated_string_node_t *cast = pm_interpolated_string_node_create(parser, &opening, &parts, &opening); pm_node_list_free(&parts); expect1_heredoc_term(parser, lex_mode.ident_start, lex_mode.ident_length); - pm_interpolated_string_node_closing_set(cast, &parser->previous); + pm_interpolated_string_node_closing_set(parser, cast, &parser->previous); - cast->base.location = cast->opening_loc; + cast->base.location = (pm_location_t) { + .start = parser->start + cast->opening_loc.start, + .end = parser->start + cast->opening_loc.start + cast->opening_loc.length + }; node = UP(cast); } @@ -18164,11 +18104,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // At this point we can create a case node, though we don't yet know // if it is a case-in or case-when node. - pm_token_t end_keyword = not_provided(parser); pm_node_t *node; if (match1(parser, PM_TOKEN_KEYWORD_WHEN)) { - pm_case_node_t *case_node = pm_case_node_create(parser, &case_keyword, predicate, &end_keyword); + pm_case_node_t *case_node = pm_case_node_create(parser, &case_keyword, predicate, NULL); pm_static_literals_t literals = { 0 }; // At this point we've seen a when keyword, so we know this is a @@ -18212,11 +18151,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) { - pm_when_node_then_keyword_loc_set(when_node, &parser->previous); + pm_when_node_then_keyword_loc_set(parser, when_node, &parser->previous); } } else { expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_EXPECT_WHEN_DELIMITER); - pm_when_node_then_keyword_loc_set(when_node, &parser->previous); + pm_when_node_then_keyword_loc_set(parser, when_node, &parser->previous); } if (!match3(parser, PM_TOKEN_KEYWORD_WHEN, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) { @@ -18238,7 +18177,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_static_literals_free(&literals); node = UP(case_node); } else { - pm_case_match_node_t *case_node = pm_case_match_node_create(parser, &case_keyword, predicate, &end_keyword); + pm_case_match_node_t *case_node = pm_case_match_node_create(parser, &case_keyword, predicate); // If this is a case-match node (i.e., it is a pattern matching // case statement) then we must have a predicate. @@ -18283,12 +18222,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // Now we need to check for the terminator of the in node's // pattern. It can be a newline or semicolon optionally // followed by a `then` keyword. - pm_token_t then_keyword; + pm_token_t then_keyword = { 0 }; if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) { then_keyword = parser->previous; - } else { - then_keyword = not_provided(parser); } } else { expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_EXPECT_IN_DELIMITER); @@ -18306,7 +18243,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // Now that we have the full pattern and statements, we can // create the node and attach it to the case node. - pm_node_t *condition = UP(pm_in_node_create(parser, pattern, statements, &in_keyword, &then_keyword)); + pm_node_t *condition = UP(pm_in_node_create(parser, pattern, statements, &in_keyword, MAYBETOKENPTR(then_keyword))); pm_case_match_node_condition_append(case_node, condition); } @@ -18341,9 +18278,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_CASE_TERM); if (PM_NODE_TYPE_P(node, PM_CASE_NODE)) { - pm_case_node_end_keyword_loc_set((pm_case_node_t *) node, &parser->previous); + pm_case_node_end_keyword_loc_set(parser, (pm_case_node_t *) node, &parser->previous); } else { - pm_case_match_node_end_keyword_loc_set((pm_case_match_node_t *) node, &parser->previous); + pm_case_match_node_end_keyword_loc_set(parser, (pm_case_match_node_t *) node, &parser->previous); } pop_block_exits(parser, previous_block_exits); @@ -18374,7 +18311,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_BEGIN_TERM); begin_node->base.location.end = parser->previous.end; - pm_begin_node_end_keyword_set(begin_node, &parser->previous); + pm_begin_node_end_keyword_set(parser, begin_node, &parser->previous); pop_block_exits(parser, previous_block_exits); pm_node_list_free(¤t_block_exits); @@ -18487,7 +18424,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b arguments.block = NULL; } - pm_node_t *node = UP(pm_yield_node_create(parser, &keyword, &arguments.opening_loc, arguments.arguments, &arguments.closing_loc)); + pm_node_t *node = UP(pm_yield_node_create(parser, &keyword, arguments.opening_loc.start == NULL ? NULL : &arguments.opening_loc, arguments.arguments, arguments.closing_loc.start == NULL ? NULL : &arguments.closing_loc)); if (!parser->parsing_eval && !parser->partial_script) parse_yield(parser, node); return node; @@ -18545,7 +18482,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_parser_err_token(parser, &name, PM_ERR_CLASS_NAME); } - pm_token_t inheritance_operator; + pm_token_t inheritance_operator = { 0 }; pm_node_t *superclass; if (match1(parser, PM_TOKEN_LESS)) { @@ -18557,13 +18494,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b superclass = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, true, false, PM_ERR_CLASS_SUPERCLASS, (uint16_t) (depth + 1)); } else { - inheritance_operator = not_provided(parser); superclass = NULL; } pm_parser_scope_push(parser, true); - if (inheritance_operator.type != PM_TOKEN_NOT_PROVIDED) { + if (inheritance_operator.start != NULL) { expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CLASS_UNEXPECTED_END); } else { accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON); @@ -18602,7 +18538,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pop_block_exits(parser, previous_block_exits); pm_node_list_free(¤t_block_exits); - return UP(pm_class_node_create(parser, &locals, &class_keyword, constant_path, &name, &inheritance_operator, superclass, statements, &parser->previous)); + return UP(pm_class_node_create(parser, &locals, &class_keyword, constant_path, &name, MAYBETOKENPTR(inheritance_operator), superclass, statements, &parser->previous)); } case PM_TOKEN_KEYWORD_DEF: { pm_node_list_t current_block_exits = { 0 }; @@ -18612,7 +18548,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b size_t opening_newline_index = token_newline_index(parser); pm_node_t *receiver = NULL; - pm_token_t operator = not_provided(parser); + pm_token_t operator = { 0 }; pm_token_t name; // This context is necessary for lexing `...` in a bare params @@ -18759,8 +18695,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } - pm_token_t lparen; - pm_token_t rparen; + pm_token_t lparen = { 0 }; + pm_token_t rparen = { 0 }; pm_parameters_node_t *params; bool accept_endless_def = true; @@ -18782,7 +18718,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_DEF_PARAMS_TERM_PAREN, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; - parser->previous.type = PM_TOKEN_MISSING; + parser->previous.type = 0; } rparen = parser->previous; @@ -18795,8 +18731,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b lex_state_set(parser, parser->lex_state | PM_LEX_STATE_LABEL); } - lparen = not_provided(parser); - rparen = not_provided(parser); params = parse_parameters(parser, PM_BINDING_POWER_DEFINED, false, false, true, true, false, (uint16_t) (depth + 1)); // Reject `def * = 1` and similar. We have to specifically check @@ -18807,18 +18741,15 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } default: { - lparen = not_provided(parser); - rparen = not_provided(parser); params = NULL; - context_pop(parser); break; } } pm_node_t *statements = NULL; - pm_token_t equal; - pm_token_t end_keyword; + pm_token_t equal = { 0 }; + pm_token_t end_keyword = { 0 }; if (accept1(parser, PM_TOKEN_EQUAL)) { if (token_is_setter_name(&name)) { @@ -18862,11 +18793,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_statements_node_body_append(parser, (pm_statements_node_t *) statements, statement, false); pm_do_loop_stack_pop(parser); context_pop(parser); - end_keyword = not_provided(parser); } else { - equal = not_provided(parser); - - if (lparen.type == PM_TOKEN_NOT_PROVIDED) { + if (lparen.start == NULL) { lex_state_set(parser, PM_LEX_STATE_BEG); parser->command_start = true; expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_DEF_PARAMS_TERM); @@ -18920,19 +18848,19 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b statements, &locals, &def_keyword, - &operator, - &lparen, - &rparen, - &equal, - &end_keyword + MAYBETOKENPTR(operator), + MAYBETOKENPTR(lparen), + MAYBETOKENPTR(rparen), + MAYBETOKENPTR(equal), + MAYBETOKENPTR(end_keyword) )); } case PM_TOKEN_KEYWORD_DEFINED: { parser_lex(parser); - pm_token_t keyword = parser->previous; - pm_token_t lparen; - pm_token_t rparen; + pm_token_t keyword = parser->previous; + pm_token_t lparen = { 0 }; + pm_token_t rparen = { 0 }; pm_node_t *expression; context_push(parser, PM_CONTEXT_DEFINED); @@ -18943,31 +18871,26 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (newline && accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { expression = UP(pm_parentheses_node_create(parser, &lparen, NULL, &parser->previous, 0)); - lparen = not_provided(parser); - rparen = not_provided(parser); + lparen = (pm_token_t) { 0 }; } else { expression = parse_expression(parser, PM_BINDING_POWER_COMPOSITION, true, false, PM_ERR_DEFINED_EXPRESSION, (uint16_t) (depth + 1)); - if (parser->recovering) { - rparen = not_provided(parser); - } else { + if (!parser->recovering) { accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN); rparen = parser->previous; } } } else { - lparen = not_provided(parser); - rparen = not_provided(parser); expression = parse_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_DEFINED_EXPRESSION, (uint16_t) (depth + 1)); } context_pop(parser); return UP(pm_defined_node_create( parser, - &lparen, + MAYBETOKENPTR(lparen), expression, - &rparen, + MAYBETOKENPTR(rparen), &keyword )); } @@ -19035,11 +18958,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_t *collection = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, true, false, PM_ERR_FOR_COLLECTION, (uint16_t) (depth + 1)); pm_do_loop_stack_pop(parser); - pm_token_t do_keyword; + pm_token_t do_keyword = { 0 }; if (accept1(parser, PM_TOKEN_KEYWORD_DO_LOOP)) { do_keyword = parser->previous; } else { - do_keyword = not_provided(parser); if (!match2(parser, PM_TOKEN_SEMICOLON, PM_TOKEN_NEWLINE)) { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_FOR_DELIMITER, pm_token_type_human(parser->current.type)); } @@ -19053,7 +18975,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_warn_indentation_mismatch(parser, opening_newline_index, &for_keyword, false, false); expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_FOR_TERM); - return UP(pm_for_node_create(parser, index, collection, statements, &for_keyword, &in_keyword, &do_keyword, &parser->previous)); + return UP(pm_for_node_create(parser, index, collection, statements, &for_keyword, &in_keyword, MAYBETOKENPTR(do_keyword), &parser->previous)); } case PM_TOKEN_KEYWORD_IF: if (parser_end_of_line_p(parser)) { @@ -19162,7 +19084,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pop_block_exits(parser, previous_block_exits); pm_node_list_free(¤t_block_exits); - pm_token_t missing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + pm_token_t missing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; return UP(pm_module_node_create(parser, NULL, &module_keyword, constant_path, &missing, NULL, &missing)); } @@ -19251,11 +19173,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_do_loop_stack_pop(parser); context_pop(parser); - pm_token_t do_keyword; + pm_token_t do_keyword = { 0 }; if (accept1(parser, PM_TOKEN_KEYWORD_DO_LOOP)) { do_keyword = parser->previous; } else { - do_keyword = not_provided(parser); expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CONDITIONAL_UNTIL_PREDICATE); } @@ -19270,7 +19191,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false); expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_UNTIL_TERM); - return UP(pm_until_node_create(parser, &keyword, &do_keyword, &parser->previous, predicate, statements, 0)); + return UP(pm_until_node_create(parser, &keyword, MAYBETOKENPTR(do_keyword), &parser->previous, predicate, statements, 0)); } case PM_TOKEN_KEYWORD_WHILE: { size_t opening_newline_index = token_newline_index(parser); @@ -19285,11 +19206,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_do_loop_stack_pop(parser); context_pop(parser); - pm_token_t do_keyword; + pm_token_t do_keyword = { 0 }; if (accept1(parser, PM_TOKEN_KEYWORD_DO_LOOP)) { do_keyword = parser->previous; } else { - do_keyword = not_provided(parser); expect2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON, PM_ERR_CONDITIONAL_WHILE_PREDICATE); } @@ -19304,7 +19224,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false); expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_WHILE_TERM); - return UP(pm_while_node_create(parser, &keyword, &do_keyword, &parser->previous, predicate, statements, 0)); + return UP(pm_while_node_create(parser, &keyword, MAYBETOKENPTR(do_keyword), &parser->previous, predicate, statements, 0)); } case PM_TOKEN_PERCENT_LOWER_I: { parser_lex(parser); @@ -19316,9 +19236,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (match1(parser, PM_TOKEN_STRING_END)) break; if (match1(parser, PM_TOKEN_STRING_CONTENT)) { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_array_node_elements_append(array, UP(pm_symbol_node_create_current_string(parser, &opening, &parser->current, &closing))); + pm_array_node_elements_append(array, UP(pm_symbol_node_create_current_string(parser, NULL, &parser->current, NULL))); } expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_LIST_I_LOWER_ELEMENT); @@ -19327,7 +19245,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_LIST_I_LOWER_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_I_LOWER_TERM); } @@ -19361,20 +19279,17 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } case PM_TOKEN_STRING_CONTENT: { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - if (current == NULL) { // If we hit content and the current node is NULL, then this is // the first string content we've seen. In that case we're going // to create a new string node and set that to the current. - current = UP(pm_symbol_node_create_current_string(parser, &opening, &parser->current, &closing)); + current = UP(pm_symbol_node_create_current_string(parser, NULL, &parser->current, NULL)); parser_lex(parser); } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_SYMBOL_NODE)) { // If we hit string content and the current node is an // interpolated string, then we need to append the string content // to the list of child nodes. - pm_node_t *string = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing)); + pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); parser_lex(parser); pm_interpolated_symbol_node_append((pm_interpolated_symbol_node_t *) current, string); @@ -19383,14 +19298,17 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // then we need to convert the current node into an interpolated // string and add the string content to the list of child nodes. pm_symbol_node_t *cast = (pm_symbol_node_t *) current; - pm_token_t bounds = not_provided(parser); - - pm_token_t content = { .type = PM_TOKEN_STRING_CONTENT, .start = cast->value_loc.start, .end = cast->value_loc.end }; - pm_node_t *first_string = UP(pm_string_node_create_unescaped(parser, &bounds, &content, &bounds, &cast->unescaped)); - pm_node_t *second_string = UP(pm_string_node_create_current_string(parser, &opening, &parser->previous, &closing)); + pm_token_t content = { + .type = PM_TOKEN_STRING_CONTENT, + .start = parser->start + cast->value_loc.start, + .end = parser->start + cast->value_loc.start + cast->value_loc.length + }; + + pm_node_t *first_string = UP(pm_string_node_create_unescaped(parser, NULL, &content, NULL, &cast->unescaped)); + pm_node_t *second_string = UP(pm_string_node_create_current_string(parser, NULL, &parser->previous, NULL)); parser_lex(parser); - pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing); + pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL); pm_interpolated_symbol_node_append(interpolated, first_string); pm_interpolated_symbol_node_append(interpolated, second_string); @@ -19408,16 +19326,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we hit an embedded variable and the current node is NULL, // then this is the start of a new string. We'll set the current // node to a new interpolated string. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - current = UP(pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing)); + current = UP(pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL)); } else if (PM_NODE_TYPE_P(current, PM_SYMBOL_NODE)) { // If we hit an embedded variable and the current node is a string // node, then we'll convert the current into an interpolated // string and add the string node to the list of parts. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing); + pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL); current = UP(pm_symbol_node_to_string_node(parser, (pm_symbol_node_t *) current)); pm_interpolated_symbol_node_append(interpolated, current); @@ -19442,17 +19356,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we hit an embedded expression and the current node is NULL, // then this is the start of a new string. We'll set the current // node to a new interpolated string. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - current = UP(pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing)); + current = UP(pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL)); } else if (PM_NODE_TYPE_P(current, PM_SYMBOL_NODE)) { // If we hit an embedded expression and the current node is a // string node, then we'll convert the current into an // interpolated string and add the string node to the list of // parts. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing); + pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, NULL, NULL, NULL); current = UP(pm_symbol_node_to_string_node(parser, (pm_symbol_node_t *) current)); pm_interpolated_symbol_node_append(interpolated, current); @@ -19488,7 +19398,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_LIST_I_UPPER_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_I_UPPER_TERM); } @@ -19509,10 +19419,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (match1(parser, PM_TOKEN_STRING_END)) break; if (match1(parser, PM_TOKEN_STRING_CONTENT)) { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - - pm_node_t *string = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing)); + pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); pm_array_node_elements_append(array, string); } @@ -19522,7 +19429,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_LIST_W_LOWER_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_W_LOWER_TERM); } @@ -19561,10 +19468,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b break; } case PM_TOKEN_STRING_CONTENT: { - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - - pm_node_t *string = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing)); + pm_node_t *string = UP(pm_string_node_create_current_string(parser, NULL, &parser->current, NULL)); pm_node_flag_set(string, parse_unescaped_encoding(parser)); parser_lex(parser); @@ -19584,7 +19488,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // a string node, then we need to convert the // current node into an interpolated string and add // the string content to the list of child nodes. - pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); + pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, NULL, NULL, NULL); pm_interpolated_string_node_append(interpolated, current); pm_interpolated_string_node_append(interpolated, string); current = UP(interpolated); @@ -19600,17 +19504,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // node is NULL, then this is the start of a new // string. We'll set the current node to a new // interpolated string. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - current = UP(pm_interpolated_string_node_create(parser, &opening, NULL, &closing)); + current = UP(pm_interpolated_string_node_create(parser, NULL, NULL, NULL)); } else if (PM_NODE_TYPE_P(current, PM_STRING_NODE)) { // If we hit an embedded variable and the current // node is a string node, then we'll convert the // current into an interpolated string and add the // string node to the list of parts. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); + pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, NULL, NULL, NULL); pm_interpolated_string_node_append(interpolated, current); current = UP(interpolated); } else { @@ -19629,17 +19529,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // node is NULL, then this is the start of a new // string. We'll set the current node to a new // interpolated string. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - current = UP(pm_interpolated_string_node_create(parser, &opening, NULL, &closing)); + current = UP(pm_interpolated_string_node_create(parser, NULL, NULL, NULL)); } else if (PM_NODE_TYPE_P(current, PM_STRING_NODE)) { // If we hit an embedded expression and the current // node is a string node, then we'll convert the // current into an interpolated string and add the // string node to the list of parts. - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); + pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, NULL, NULL, NULL); pm_interpolated_string_node_append(interpolated, current); current = UP(interpolated); } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_STRING_NODE)) { @@ -19669,7 +19565,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_LIST_W_UPPER_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_LIST_W_UPPER_TERM); } @@ -19733,10 +19629,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // a regular expression node with interpolation. interpolated = pm_interpolated_regular_expression_node_create(parser, &opening); - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - pm_node_t *part = UP(pm_string_node_create_unescaped(parser, &opening, &parser->previous, &closing, &unescaped)); - + pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->previous, NULL, &unescaped)); if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) { // This is extremely strange, but the first string part of a // regular expression will always be tagged as binary if we @@ -19764,7 +19657,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_REGEXP_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_REGEXP_END, PM_ERR_REGEXP_TERM); } @@ -19817,10 +19710,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // create a string node with interpolation. node = pm_interpolated_xstring_node_create(parser, &opening, &opening); - pm_token_t opening = not_provided(parser); - pm_token_t closing = not_provided(parser); - - pm_node_t *part = UP(pm_string_node_create_unescaped(parser, &opening, &parser->previous, &closing, &unescaped)); + pm_node_t *part = UP(pm_string_node_create_unescaped(parser, NULL, &parser->previous, NULL, &unescaped)); pm_node_flag_set(part, parse_unescaped_encoding(parser)); pm_interpolated_xstring_node_append(node, part); @@ -19841,11 +19731,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t closing = parser->current; if (match1(parser, PM_TOKEN_EOF)) { pm_parser_err_token(parser, &opening, PM_ERR_XSTRING_TERM); - closing = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + closing = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } else { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_XSTRING_TERM); } - pm_interpolated_xstring_node_closing_set(node, &closing); + pm_interpolated_xstring_node_closing_set(parser, node, &closing); return UP(node); } @@ -19967,13 +19857,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN); - pm_block_parameters_node_closing_set(block_parameters, &parser->previous); + pm_block_parameters_node_closing_set(parser, block_parameters, &parser->previous); break; } case PM_CASE_PARAMETER: { pm_accepts_block_stack_push(parser, false); - pm_token_t opening = not_provided(parser); - block_parameters = parse_block_parameters(parser, false, &opening, true, false, (uint16_t) (depth + 1)); + block_parameters = parse_block_parameters(parser, false, NULL, true, false, (uint16_t) (depth + 1)); pm_accepts_block_stack_pop(parser); break; } @@ -20196,7 +20085,7 @@ parse_assignment_values(pm_parser_t *parser, pm_binding_power_t previous_binding // but without parenthesis. if (PM_NODE_TYPE_P(value, PM_CALL_NODE)) { pm_call_node_t *call_node = (pm_call_node_t *) value; - if ((call_node->arguments != NULL) && (call_node->opening_loc.start == NULL)) { + if ((call_node->arguments != NULL) && (call_node->opening_loc.length == 0)) { accepts_command_call_inner = true; } } @@ -20512,7 +20401,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // is parsed because it could be referenced in the value. pm_call_node_t *call_node = (pm_call_node_t *) node; if (PM_NODE_FLAG_P(call_node, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_parser_local_add_location(parser, call_node->message_loc.start, call_node->message_loc.end, 0); + pm_parser_local_add_location(parser, parser->start + call_node->message_loc.start, parser->start + call_node->message_loc.start + call_node->message_loc.length, 0); } } PRISM_FALLTHROUGH @@ -20646,10 +20535,13 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // receiver that could have been a local variable) then we // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_location_t *message_loc = &cast->message_loc; - pm_refute_numbered_parameter(parser, message_loc->start, message_loc->end); + pm_location_t message_loc = { + .start = parser->start + cast->message_loc.start, + .end = parser->start + cast->message_loc.start + cast->message_loc.length + }; - pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc->start, message_loc->end, 1); + pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); + pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc.start, message_loc.end, 1); parser_lex(parser); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1)); @@ -20780,10 +20672,13 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // receiver that could have been a local variable) then we // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_location_t *message_loc = &cast->message_loc; - pm_refute_numbered_parameter(parser, message_loc->start, message_loc->end); + pm_location_t message_loc = (pm_location_t) { + .start = parser->start + cast->message_loc.start, + .end = parser->start + cast->message_loc.start + cast->message_loc.length + }; - pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc->start, message_loc->end, 1); + pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); + pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc.start, message_loc.end, 1); parser_lex(parser); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1)); @@ -20925,10 +20820,14 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // receiver that could have been a local variable) then we // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_location_t *message_loc = &cast->message_loc; - pm_refute_numbered_parameter(parser, message_loc->start, message_loc->end); - - pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc->start, message_loc->end, 1); + pm_location_t message_loc = (pm_location_t) { + .start = parser->start + cast->message_loc.start, + .end = parser->start + cast->message_loc.start + cast->message_loc.length + }; + + pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); + + pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc.start, message_loc.end, 1); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1)); pm_node_t *result = UP(pm_local_variable_operator_write_node_create(parser, UP(cast), &token, value, constant_id, 0)); @@ -21166,7 +21065,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t } default: { PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_MESSAGE, pm_token_type_human(parser->current.type)); - message = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + message = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } } @@ -21242,7 +21141,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // before the `expect` function call to make sure it doesn't // accidentally move past a ':' token that occurs after the syntax // error. - pm_token_t colon = (pm_token_t) { .type = PM_TOKEN_MISSING, .start = parser->previous.end, .end = parser->previous.end }; + pm_token_t colon = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; pm_node_t *false_expression = UP(pm_missing_node_create(parser, colon.start, colon.end)); context_pop(parser); @@ -21442,7 +21341,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t static inline bool pm_call_node_command_p(const pm_call_node_t *node) { return ( - (node->opening_loc.start == NULL) && + (node->opening_loc.length == 0) && (node->block == NULL || PM_NODE_TYPE_P(node->block, PM_BLOCK_ARGUMENT_NODE)) && (node->arguments != NULL || node->block != NULL) ); @@ -21496,7 +21395,7 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc // If we have a symbol node that is being parsed as a label, then we // need to immediately return, because there should never be an // infix operator following this node. - if (pm_symbol_node_label_p(node)) { + if (pm_symbol_node_label_p(parser, node)) { return node; } break; @@ -21601,22 +21500,22 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc if ( // (1) foo[1] !( - cast->call_operator_loc.start == NULL && - cast->message_loc.start != NULL && - cast->message_loc.start[0] == '[' && - cast->message_loc.end[-1] == ']' + cast->call_operator_loc.length == 0 && + cast->message_loc.length > 0 && + parser->start[cast->message_loc.start] == '[' && + parser->start[cast->message_loc.start + cast->message_loc.length - 1] == ']' ) && // (2) foo.bar !( - cast->call_operator_loc.start != NULL && + cast->call_operator_loc.length > 0 && cast->arguments == NULL && cast->block == NULL && - cast->opening_loc.start == NULL + cast->opening_loc.length == 0 ) && // (3) foo.bar(1) !( - cast->call_operator_loc.start != NULL && - cast->opening_loc.start != NULL + cast->call_operator_loc.length > 0 && + cast->opening_loc.length > 0 ) && // (4) foo.bar do end !( @@ -21699,7 +21598,7 @@ wrap_statements(pm_parser_t *parser, pm_statements_node_t *statements) { pm_keyword_hash_node_elements_append(keywords, UP(pm_assoc_node_create( parser, UP(pm_symbol_node_synthesized_create(parser, "chomp")), - &(pm_token_t) { .type = PM_TOKEN_NOT_PROVIDED, .start = parser->start, .end = parser->start }, + NULL, UP(pm_true_node_synthesized_create(parser)) ))); diff --git a/templates/src/token_type.c.erb b/templates/src/token_type.c.erb index f196393ee1..5c6f271310 100644 --- a/templates/src/token_type.c.erb +++ b/templates/src/token_type.c.erb @@ -31,10 +31,6 @@ pm_token_type_human(pm_token_type_t token_type) { switch (token_type) { case PM_TOKEN_EOF: return "end-of-input"; - case PM_TOKEN_MISSING: - return "missing token"; - case PM_TOKEN_NOT_PROVIDED: - return "not provided token"; case PM_TOKEN_AMPERSAND: return "'&'"; case PM_TOKEN_AMPERSAND_AMPERSAND: diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 706b739557..40bb252d61 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -45,19 +45,19 @@ def test_embdoc_ending def test_unterminated_string_closing statement = Prism.parse_statement("'hello") assert_equal statement.unescaped, "hello" - assert_empty statement.closing + assert_nil statement.closing end def test_unterminated_interpolated_string_closing statement = Prism.parse_statement('"hello') assert_equal statement.unescaped, "hello" - assert_empty statement.closing + assert_nil statement.closing end def test_unterminated_empty_string_closing statement = Prism.parse_statement('"') assert_empty statement.unescaped - assert_empty statement.closing + assert_nil statement.closing end def test_invalid_message_name From 456ec95561e79193bc40b4226f40f95b2fed5c03 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 15:05:16 -0500 Subject: [PATCH 04/22] Switch magic comments over to using slices --- ext/prism/extension.c | 10 +++++----- include/prism/parser.h | 14 ++++---------- src/prism.c | 6 ++---- templates/src/serialize.c.erb | 16 ++++++++-------- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/ext/prism/extension.c b/ext/prism/extension.c index 71c2d91b98..fd19bb8245 100644 --- a/ext/prism/extension.c +++ b/ext/prism/extension.c @@ -455,8 +455,8 @@ rb_class_new_instance_freeze(int argc, const VALUE *argv, VALUE klass, bool free * Create a new Location instance from the given parser and bounds. */ static inline VALUE -parser_location(const pm_parser_t *parser, VALUE source, bool freeze, const uint8_t *start, size_t length) { - VALUE argv[] = { source, LONG2FIX(start - parser->start), LONG2FIX(length) }; +parser_location(const pm_parser_t *parser, VALUE source, bool freeze, uint32_t start, uint32_t length) { + VALUE argv[] = { source, LONG2FIX(start), LONG2FIX(length) }; return rb_class_new_instance_freeze(3, argv, rb_cPrismLocation, freeze); } @@ -464,7 +464,7 @@ parser_location(const pm_parser_t *parser, VALUE source, bool freeze, const uint * Create a new Location instance from the given parser and location. */ #define PARSER_LOCATION_LOC(parser, source, freeze, loc) \ - parser_location(parser, source, freeze, loc.start, (size_t) (loc.end - loc.start)) + parser_location(parser, source, freeze, (uint32_t) (loc.start - parser->start), (uint32_t) (loc.end - loc.start)) /** * Build a new Comment instance from the given parser and comment. @@ -501,8 +501,8 @@ parser_comments(const pm_parser_t *parser, VALUE source, bool freeze) { */ static inline VALUE parser_magic_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_magic_comment_t *magic_comment) { - VALUE key_loc = parser_location(parser, source, freeze, magic_comment->key_start, magic_comment->key_length); - VALUE value_loc = parser_location(parser, source, freeze, magic_comment->value_start, magic_comment->value_length); + VALUE key_loc = parser_location(parser, source, freeze, magic_comment->key.start, magic_comment->key.length); + VALUE value_loc = parser_location(parser, source, freeze, magic_comment->value.start, magic_comment->value.length); VALUE argv[] = { key_loc, value_loc }; return rb_class_new_instance_freeze(2, argv, rb_cPrismMagicComment, freeze); } diff --git a/include/prism/parser.h b/include/prism/parser.h index 95d7aac710..462a343890 100644 --- a/include/prism/parser.h +++ b/include/prism/parser.h @@ -479,17 +479,11 @@ typedef struct { /** The embedded base node. */ pm_list_node_t node; - /** A pointer to the start of the key in the source. */ - const uint8_t *key_start; + /** The key of the magic comment. */ + pm_slice_t key; - /** A pointer to the start of the value in the source. */ - const uint8_t *value_start; - - /** The length of the key in the source. */ - uint32_t key_length; - - /** The length of the value in the source. */ - uint32_t value_length; + /** The value of the magic comment. */ + pm_slice_t value; } pm_magic_comment_t; /** diff --git a/src/prism.c b/src/prism.c index 2171b61a3f..b9d37417b4 100644 --- a/src/prism.c +++ b/src/prism.c @@ -7642,10 +7642,8 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { // Allocate a new magic comment node to append to the parser's list. pm_magic_comment_t *magic_comment; if ((magic_comment = (pm_magic_comment_t *) xcalloc(1, sizeof(pm_magic_comment_t))) != NULL) { - magic_comment->key_start = key_start; - magic_comment->value_start = value_start; - magic_comment->key_length = (uint32_t) key_length; - magic_comment->value_length = value_length; + magic_comment->key = (pm_slice_t) { .start = (uint32_t) (key_start - parser->start), .length = (uint32_t) key_length }; + magic_comment->value = (pm_slice_t) { .start = (uint32_t) (value_start - parser->start), .length = value_length }; pm_list_append(&parser->magic_comment_list, (pm_list_node_t *) magic_comment); } } diff --git a/templates/src/serialize.c.erb b/templates/src/serialize.c.erb index 82751d3821..207ec9fe73 100644 --- a/templates/src/serialize.c.erb +++ b/templates/src/serialize.c.erb @@ -210,23 +210,23 @@ pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buf } static void -pm_serialize_magic_comment(pm_parser_t *parser, pm_magic_comment_t *magic_comment, pm_buffer_t *buffer) { +pm_serialize_magic_comment(pm_magic_comment_t *magic_comment, pm_buffer_t *buffer) { // serialize key location - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(magic_comment->key_start - parser->start)); - pm_buffer_append_varuint(buffer, pm_sizet_to_u32(magic_comment->key_length)); + pm_buffer_append_varuint(buffer, magic_comment->key.start); + pm_buffer_append_varuint(buffer, magic_comment->key.length); // serialize value location - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(magic_comment->value_start - parser->start)); - pm_buffer_append_varuint(buffer, pm_sizet_to_u32(magic_comment->value_length)); + pm_buffer_append_varuint(buffer, magic_comment->value.start); + pm_buffer_append_varuint(buffer, magic_comment->value.length); } static void -pm_serialize_magic_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) { +pm_serialize_magic_comment_list(pm_list_t *list, pm_buffer_t *buffer) { pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list))); pm_magic_comment_t *magic_comment; for (magic_comment = (pm_magic_comment_t *) list->head; magic_comment != NULL; magic_comment = (pm_magic_comment_t *) magic_comment->node.next) { - pm_serialize_magic_comment(parser, magic_comment, buffer); + pm_serialize_magic_comment(magic_comment, buffer); } } @@ -284,7 +284,7 @@ pm_serialize_metadata(pm_parser_t *parser, pm_buffer_t *buffer) { <%- unless Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS -%> pm_serialize_comment_list(parser, &parser->comment_list, buffer); <%- end -%> - pm_serialize_magic_comment_list(parser, &parser->magic_comment_list, buffer); + pm_serialize_magic_comment_list(&parser->magic_comment_list, buffer); pm_serialize_data_loc(parser, buffer); pm_serialize_diagnostic_list(parser, &parser->error_list, buffer); pm_serialize_diagnostic_list(parser, &parser->warning_list, buffer); From a16e2519b5fc87f5b255b8cf2264d9ecaddadf9e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 15:08:12 -0500 Subject: [PATCH 05/22] Switch data_loc over to using a slice --- ext/prism/extension.c | 4 ++-- include/prism/parser.h | 2 +- src/prism.c | 6 +++--- templates/src/serialize.c.erb | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/prism/extension.c b/ext/prism/extension.c index fd19bb8245..eeb26d82c2 100644 --- a/ext/prism/extension.c +++ b/ext/prism/extension.c @@ -533,10 +533,10 @@ parser_magic_comments(const pm_parser_t *parser, VALUE source, bool freeze) { */ static VALUE parser_data_loc(const pm_parser_t *parser, VALUE source, bool freeze) { - if (parser->data_loc.end == NULL) { + if (parser->data_loc.length == 0) { return Qnil; } else { - return PARSER_LOCATION_LOC(parser, source, freeze, parser->data_loc); + return parser_location(parser, source, freeze, parser->data_loc.start, parser->data_loc.length); } } diff --git a/include/prism/parser.h b/include/prism/parser.h index 462a343890..c4edd128f0 100644 --- a/include/prism/parser.h +++ b/include/prism/parser.h @@ -722,7 +722,7 @@ struct pm_parser { * and the rest of the content of the file. This content is loaded into the * DATA constant when the file being parsed is the main file being executed. */ - pm_location_t data_loc; + pm_slice_t data_loc; /** The list of warnings that have been found while parsing. */ pm_list_t warning_list; diff --git a/src/prism.c b/src/prism.c index b9d37417b4..db2acbe200 100644 --- a/src/prism.c +++ b/src/prism.c @@ -11004,8 +11004,8 @@ parser_lex(pm_parser_t *parser) { parser->current.type = PM_TOKEN___END__; parser_lex_callback(parser); - parser->data_loc.start = parser->current.start; - parser->data_loc.end = parser->current.end; + parser->data_loc.start = (uint32_t) (parser->current.start - parser->start); + parser->data_loc.length = (uint32_t) (parser->current.end - parser->current.start); LEX(PM_TOKEN_EOF); } @@ -21761,7 +21761,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm .current = { .type = PM_TOKEN_EOF, .start = source, .end = source }, .next_start = NULL, .heredoc_end = NULL, - .data_loc = { .start = NULL, .end = NULL }, + .data_loc = { 0 }, .comment_list = { 0 }, .magic_comment_list = { 0 }, .warning_list = { 0 }, diff --git a/templates/src/serialize.c.erb b/templates/src/serialize.c.erb index 207ec9fe73..eb2e9a21d3 100644 --- a/templates/src/serialize.c.erb +++ b/templates/src/serialize.c.erb @@ -232,11 +232,11 @@ pm_serialize_magic_comment_list(pm_list_t *list, pm_buffer_t *buffer) { static void pm_serialize_data_loc(const pm_parser_t *parser, pm_buffer_t *buffer) { - if (parser->data_loc.end == NULL) { + if (parser->data_loc.length == 0) { pm_buffer_append_byte(buffer, 0); } else { pm_buffer_append_byte(buffer, 1); - pm_serialize_location(parser, &parser->data_loc, buffer); + pm_serialize_slice(&parser->data_loc, buffer); } } From 801a6f931d424cdd4afdee2d7e07b7e1f4a2a973 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 15:24:07 -0500 Subject: [PATCH 06/22] Switch diagnostics over to using slices --- ext/prism/extension.c | 24 +++++++++++++++--------- src/prism.c | 22 +++++++++++----------- src/util/pm_strpbrk.c | 20 ++++++++++---------- templates/include/prism/diagnostic.h.erb | 14 +++++++------- templates/src/diagnostic.c.erb | 14 +++++++------- templates/src/serialize.c.erb | 12 ++++++------ 6 files changed, 56 insertions(+), 50 deletions(-) diff --git a/ext/prism/extension.c b/ext/prism/extension.c index eeb26d82c2..96bfa36b20 100644 --- a/ext/prism/extension.c +++ b/ext/prism/extension.c @@ -455,16 +455,22 @@ rb_class_new_instance_freeze(int argc, const VALUE *argv, VALUE klass, bool free * Create a new Location instance from the given parser and bounds. */ static inline VALUE -parser_location(const pm_parser_t *parser, VALUE source, bool freeze, uint32_t start, uint32_t length) { +parser_location(VALUE source, bool freeze, uint32_t start, uint32_t length) { VALUE argv[] = { source, LONG2FIX(start), LONG2FIX(length) }; return rb_class_new_instance_freeze(3, argv, rb_cPrismLocation, freeze); } +/** + * Create a new Location instance from the given parser and slice. + */ +#define PARSER_LOCATION_SLICE(parser, source, freeze, slice) \ + parser_location(source, freeze, slice.start, slice.length) + /** * Create a new Location instance from the given parser and location. */ #define PARSER_LOCATION_LOC(parser, source, freeze, loc) \ - parser_location(parser, source, freeze, (uint32_t) (loc.start - parser->start), (uint32_t) (loc.end - loc.start)) + parser_location(source, freeze, (uint32_t) (loc.start - parser->start), (uint32_t) (loc.end - loc.start)) /** * Build a new Comment instance from the given parser and comment. @@ -500,9 +506,9 @@ parser_comments(const pm_parser_t *parser, VALUE source, bool freeze) { * Build a new MagicComment instance from the given parser and magic comment. */ static inline VALUE -parser_magic_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_magic_comment_t *magic_comment) { - VALUE key_loc = parser_location(parser, source, freeze, magic_comment->key.start, magic_comment->key.length); - VALUE value_loc = parser_location(parser, source, freeze, magic_comment->value.start, magic_comment->value.length); +parser_magic_comment(VALUE source, bool freeze, const pm_magic_comment_t *magic_comment) { + VALUE key_loc = parser_location(source, freeze, magic_comment->key.start, magic_comment->key.length); + VALUE value_loc = parser_location(source, freeze, magic_comment->value.start, magic_comment->value.length); VALUE argv[] = { key_loc, value_loc }; return rb_class_new_instance_freeze(2, argv, rb_cPrismMagicComment, freeze); } @@ -519,7 +525,7 @@ parser_magic_comments(const pm_parser_t *parser, VALUE source, bool freeze) { magic_comment != NULL; magic_comment = (const pm_magic_comment_t *) magic_comment->node.next ) { - VALUE value = parser_magic_comment(parser, source, freeze, magic_comment); + VALUE value = parser_magic_comment(source, freeze, magic_comment); rb_ary_push(magic_comments, value); } @@ -536,7 +542,7 @@ parser_data_loc(const pm_parser_t *parser, VALUE source, bool freeze) { if (parser->data_loc.length == 0) { return Qnil; } else { - return parser_location(parser, source, freeze, parser->data_loc.start, parser->data_loc.length); + return parser_location(source, freeze, parser->data_loc.start, parser->data_loc.length); } } @@ -554,7 +560,7 @@ parser_errors(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bo ) { VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(error->diag_id))); VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(error->message, encoding)); - VALUE location = PARSER_LOCATION_LOC(parser, source, freeze, error->location); + VALUE location = PARSER_LOCATION_SLICE(parser, source, freeze, error->location); VALUE level = Qnil; switch (error->level) { @@ -594,7 +600,7 @@ parser_warnings(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, ) { VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(warning->diag_id))); VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(warning->message, encoding)); - VALUE location = PARSER_LOCATION_LOC(parser, source, freeze, warning->location); + VALUE location = PARSER_LOCATION_SLICE(parser, source, freeze, warning->location); VALUE level = Qnil; switch (warning->level) { diff --git a/src/prism.c b/src/prism.c index db2acbe200..445aadf217 100644 --- a/src/prism.c +++ b/src/prism.c @@ -423,14 +423,14 @@ debug_lex_state_set(pm_parser_t *parser, pm_lex_state_t state, char const * call */ static inline void pm_parser_err(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { - pm_diagnostic_list_append(&parser->error_list, start, end, diag_id); + pm_diagnostic_list_append(&parser->error_list, (uint32_t) (start - parser->start), (uint32_t) (end - start), diag_id); } /** * Append an error to the list of errors on the parser using a format string. */ -#define PM_PARSER_ERR_FORMAT(parser, start, end, diag_id, ...) \ - pm_diagnostic_list_append_format(&parser->error_list, start, end, diag_id, __VA_ARGS__) +#define PM_PARSER_ERR_FORMAT(parser_, start_, end_, diag_id_, ...) \ + pm_diagnostic_list_append_format(&(parser_)->error_list, (uint32_t) ((start_) - (parser_)->start), (uint32_t) ((end_) - (start_)), diag_id_, __VA_ARGS__) /** * Append an error to the list of errors on the parser using the location of the @@ -508,7 +508,7 @@ pm_parser_err_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_ */ static inline void pm_parser_warn(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { - pm_diagnostic_list_append(&parser->warning_list, start, end, diag_id); + pm_diagnostic_list_append(&parser->warning_list, (uint32_t) (start - parser->start), (uint32_t) (end - start), diag_id); } /** @@ -532,8 +532,8 @@ pm_parser_warn_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id /** * Append a warning to the list of warnings on the parser using a format string. */ -#define PM_PARSER_WARN_FORMAT(parser, start, end, diag_id, ...) \ - pm_diagnostic_list_append_format(&parser->warning_list, start, end, diag_id, __VA_ARGS__) +#define PM_PARSER_WARN_FORMAT(parser_, start_, end_, diag_id_, ...) \ + pm_diagnostic_list_append_format(&(parser_)->warning_list, (uint32_t) ((start_) - (parser_)->start), (uint32_t) ((end_) - (start_)), diag_id_, __VA_ARGS__) /** * Append a warning to the list of warnings on the parser using the location of @@ -3847,7 +3847,7 @@ pm_double_parse(pm_parser_t *parser, const pm_token_t *token) { ellipsis = ""; } - pm_diagnostic_list_append_format(&parser->warning_list, token->start, token->end, PM_WARN_FLOAT_OUT_OF_RANGE, warn_width, (const char *) token->start, ellipsis); + pm_diagnostic_list_append_format(&parser->warning_list, (uint32_t) (token->start - parser->start), (uint32_t) (token->end - token->start), PM_WARN_FLOAT_OUT_OF_RANGE, warn_width, (const char *) token->start, ellipsis); value = (value < 0.0) ? -HUGE_VAL : HUGE_VAL; } @@ -13208,8 +13208,8 @@ pm_hash_key_static_literals_add(pm_parser_t *parser, pm_static_literals_t *liter pm_diagnostic_list_append_format( &parser->warning_list, - duplicated->location.start, - duplicated->location.end, + (uint32_t) (duplicated->location.start - parser->start), + (uint32_t) (duplicated->location.end - duplicated->location.start), PM_WARN_DUPLICATED_HASH_KEY, (int) pm_buffer_length(&buffer), pm_buffer_value(&buffer), @@ -13231,8 +13231,8 @@ pm_when_clause_static_literals_add(pm_parser_t *parser, pm_static_literals_t *li if ((previous = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node, false)) != NULL) { pm_diagnostic_list_append_format( &parser->warning_list, - node->location.start, - node->location.end, + (uint32_t) (node->location.start - parser->start), + (uint32_t) (node->location.end - node->location.start), PM_WARN_DUPLICATED_WHEN_CLAUSE, pm_newline_list_line_column(&parser->newline_list, node->location.start, parser->start_line).line, pm_newline_list_line_column(&parser->newline_list, previous->location.start, parser->start_line).line diff --git a/src/util/pm_strpbrk.c b/src/util/pm_strpbrk.c index 916a4cc3fd..60c67b2983 100644 --- a/src/util/pm_strpbrk.c +++ b/src/util/pm_strpbrk.c @@ -4,22 +4,22 @@ * Add an invalid multibyte character error to the parser. */ static inline void -pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { - pm_diagnostic_list_append_format(&parser->error_list, start, end, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *start); +pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, uint32_t start, uint32_t length) { + pm_diagnostic_list_append_format(&parser->error_list, start, length, PM_ERR_INVALID_MULTIBYTE_CHARACTER, parser->start[start]); } /** * Set the explicit encoding for the parser to the current encoding. */ static inline void -pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, const uint8_t *source, size_t width) { +pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, uint32_t start, uint32_t length) { if (parser->explicit_encoding != NULL) { if (parser->explicit_encoding == parser->encoding) { // Okay, we already locked to this encoding. } else if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) { // Not okay, we already found a Unicode escape sequence and this // conflicts. - pm_diagnostic_list_append_format(&parser->error_list, source, source + width, PM_ERR_MIXED_ENCODING, parser->encoding->name); + pm_diagnostic_list_append_format(&parser->error_list, start, length, PM_ERR_MIXED_ENCODING, parser->encoding->name); } else { // Should not be anything else. assert(false && "unreachable"); @@ -61,7 +61,7 @@ pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *chars index++; } while (index < maximum && pm_encoding_utf_8_char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); - pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index); + pm_strpbrk_invalid_multibyte_character(parser, (uint32_t) ((source + start) - parser->start), (uint32_t) (index - start)); } } } @@ -81,7 +81,7 @@ pm_strpbrk_ascii_8bit(pm_parser_t *parser, const uint8_t *source, const uint8_t return source + index; } - if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, source, 1); + if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, (uint32_t) (source - parser->start), 1); index++; } @@ -105,7 +105,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t index++; } else { size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); - if (validate) pm_strpbrk_explicit_encoding_set(parser, source, width); + if (validate) pm_strpbrk_explicit_encoding_set(parser, (uint32_t) (source - parser->start), (uint32_t) width); if (width > 0) { index += width; @@ -122,7 +122,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t index++; } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); - pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index); + pm_strpbrk_invalid_multibyte_character(parser, (uint32_t) ((source + start) - parser->start), (uint32_t) (index - start)); } } } @@ -148,7 +148,7 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t index++; } else { size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index)); - pm_strpbrk_explicit_encoding_set(parser, source, width); + pm_strpbrk_explicit_encoding_set(parser, (uint32_t) (source - parser->start), (uint32_t) width); if (width > 0) { index += width; @@ -163,7 +163,7 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t index++; } while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0); - pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index); + pm_strpbrk_invalid_multibyte_character(parser, (uint32_t) ((source + start) - parser->start), (uint32_t) (index - start)); } } } diff --git a/templates/include/prism/diagnostic.h.erb b/templates/include/prism/diagnostic.h.erb index 07bbc8fae7..ce4102639d 100644 --- a/templates/include/prism/diagnostic.h.erb +++ b/templates/include/prism/diagnostic.h.erb @@ -40,7 +40,7 @@ typedef struct { pm_list_node_t node; /** The location of the diagnostic in the source. */ - pm_location_t location; + pm_slice_t location; /** The ID of the diagnostic. */ pm_diagnostic_id_t diag_id; @@ -100,25 +100,25 @@ const char * pm_diagnostic_id_human(pm_diagnostic_id_t diag_id); * memory for its message. * * @param list The list to append to. - * @param start The start of the diagnostic. - * @param end The end of the diagnostic. + * @param start The source offset of the start of the diagnostic. + * @param length The length of the diagnostic. * @param diag_id The diagnostic ID. * @return Whether the diagnostic was successfully appended. */ -bool pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id); +bool pm_diagnostic_list_append(pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id); /** * Append a diagnostic to the given list of diagnostics that is using a format * string for its message. * * @param list The list to append to. - * @param start The start of the diagnostic. - * @param end The end of the diagnostic. + * @param start The source offset of the start of the diagnostic. + * @param length The length of the diagnostic. * @param diag_id The diagnostic ID. * @param ... The arguments to the format string for the message. * @return Whether the diagnostic was successfully appended. */ -bool pm_diagnostic_list_append_format(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id, ...); +bool pm_diagnostic_list_append_format(pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id, ...); /** * Deallocate the internal state of the given diagnostic list. diff --git a/templates/src/diagnostic.c.erb b/templates/src/diagnostic.c.erb index 121dd4b2b6..88f8525f80 100644 --- a/templates/src/diagnostic.c.erb +++ b/templates/src/diagnostic.c.erb @@ -447,12 +447,12 @@ pm_diagnostic_level(pm_diagnostic_id_t diag_id) { * Append an error to the given list of diagnostic. */ bool -pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { +pm_diagnostic_list_append(pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id) { pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(1, sizeof(pm_diagnostic_t)); if (diagnostic == NULL) return false; *diagnostic = (pm_diagnostic_t) { - .location = { start, end }, + .location = { .start = start, .length = length }, .diag_id = diag_id, .message = pm_diagnostic_message(diag_id), .owned = false, @@ -468,7 +468,7 @@ pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t * * string for its message. */ bool -pm_diagnostic_list_append_format(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id, ...) { +pm_diagnostic_list_append_format(pm_list_t *list, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id, ...) { va_list arguments; va_start(arguments, diag_id); @@ -485,19 +485,19 @@ pm_diagnostic_list_append_format(pm_list_t *list, const uint8_t *start, const ui return false; } - size_t length = (size_t) (result + 1); - char *message = (char *) xmalloc(length); + size_t message_length = (size_t) (result + 1); + char *message = (char *) xmalloc(message_length); if (message == NULL) { xfree(diagnostic); return false; } va_start(arguments, diag_id); - vsnprintf(message, length, format, arguments); + vsnprintf(message, message_length, format, arguments); va_end(arguments); *diagnostic = (pm_diagnostic_t) { - .location = { start, end }, + .location = { .start = start, .length = length }, .diag_id = diag_id, .message = message, .owned = true, diff --git a/templates/src/serialize.c.erb b/templates/src/serialize.c.erb index eb2e9a21d3..ac79b643dd 100644 --- a/templates/src/serialize.c.erb +++ b/templates/src/serialize.c.erb @@ -241,7 +241,7 @@ pm_serialize_data_loc(const pm_parser_t *parser, pm_buffer_t *buffer) { } static void -pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) { +pm_serialize_diagnostic(pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) { // serialize the type pm_buffer_append_varuint(buffer, (uint32_t) diagnostic->diag_id); @@ -251,18 +251,18 @@ pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buf pm_buffer_append_string(buffer, diagnostic->message, message_length); // serialize location - pm_serialize_location(parser, &diagnostic->location, buffer); + pm_serialize_slice(&diagnostic->location, buffer); pm_buffer_append_byte(buffer, diagnostic->level); } static void -pm_serialize_diagnostic_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) { +pm_serialize_diagnostic_list(pm_list_t *list, pm_buffer_t *buffer) { pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list))); pm_diagnostic_t *diagnostic; for (diagnostic = (pm_diagnostic_t *) list->head; diagnostic != NULL; diagnostic = (pm_diagnostic_t *) diagnostic->node.next) { - pm_serialize_diagnostic(parser, diagnostic, buffer); + pm_serialize_diagnostic(diagnostic, buffer); } } @@ -286,8 +286,8 @@ pm_serialize_metadata(pm_parser_t *parser, pm_buffer_t *buffer) { <%- end -%> pm_serialize_magic_comment_list(&parser->magic_comment_list, buffer); pm_serialize_data_loc(parser, buffer); - pm_serialize_diagnostic_list(parser, &parser->error_list, buffer); - pm_serialize_diagnostic_list(parser, &parser->warning_list, buffer); + pm_serialize_diagnostic_list(&parser->error_list, buffer); + pm_serialize_diagnostic_list(&parser->warning_list, buffer); } #line <%= __LINE__ + 1 %> "prism/templates/src/<%= File.basename(__FILE__) %>" From d645e868261fd0c3b57b152aa7ea47b2743f912e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 15:28:09 -0500 Subject: [PATCH 07/22] Add locals from slices --- src/prism.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/prism.c b/src/prism.c index 445aadf217..761a96bffc 100644 --- a/src/prism.c +++ b/src/prism.c @@ -7090,6 +7090,14 @@ pm_parser_local_add_location(pm_parser_t *parser, const uint8_t *start, const ui return constant_id; } +/** + * Add a local variable from a slice to the current scope. + */ +static pm_constant_id_t +pm_parser_local_add_slice(pm_parser_t *parser, pm_slice_t *slice, uint32_t reads) { + return pm_parser_local_add_location(parser, parser->start + slice->start, parser->start + slice->start + slice->length, reads); +} + /** * Add a local variable from a token to the current scope. */ @@ -12724,7 +12732,7 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p .end = parser->start + call->message_loc.start + call->message_loc.length }; - pm_constant_id_t name = pm_parser_local_add_location(parser, message_loc.start, message_loc.end, 0); + pm_constant_id_t name = pm_parser_local_add_slice(parser, &call->message_loc, 0); pm_node_destroy(parser, target); return UP(pm_local_variable_target_node_create(parser, &message_loc, name, 0)); @@ -12912,7 +12920,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod .end = parser->start + call->message_loc.start + call->message_loc.length }; - pm_parser_local_add_location(parser, message.start, message.end, 0); + pm_parser_local_add_slice(parser, &call->message_loc, 0); pm_node_destroy(parser, target); pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, message.start, message.end); @@ -20399,7 +20407,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // is parsed because it could be referenced in the value. pm_call_node_t *call_node = (pm_call_node_t *) node; if (PM_NODE_FLAG_P(call_node, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_parser_local_add_location(parser, parser->start + call_node->message_loc.start, parser->start + call_node->message_loc.start + call_node->message_loc.length, 0); + pm_parser_local_add_slice(parser, &call_node->message_loc, 0); } } PRISM_FALLTHROUGH @@ -20539,7 +20547,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t }; pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); - pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc.start, message_loc.end, 1); + pm_constant_id_t constant_id = pm_parser_local_add_slice(parser, &cast->message_loc, 1); parser_lex(parser); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1)); @@ -20676,7 +20684,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t }; pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); - pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc.start, message_loc.end, 1); + pm_constant_id_t constant_id = pm_parser_local_add_slice(parser, &cast->message_loc, 1); parser_lex(parser); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1)); @@ -20824,8 +20832,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t }; pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); - - pm_constant_id_t constant_id = pm_parser_local_add_location(parser, message_loc.start, message_loc.end, 1); + pm_constant_id_t constant_id = pm_parser_local_add_slice(parser, &cast->message_loc, 1); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1)); pm_node_t *result = UP(pm_local_variable_operator_write_node_create(parser, UP(cast), &token, value, constant_id, 0)); From 8267e8c1c371cee098affd5466248e7f226ed554 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 15:33:53 -0500 Subject: [PATCH 08/22] Switch comments over to using slices --- ext/prism/extension.c | 18 ++++++------------ include/prism.h | 3 +-- include/prism/parser.h | 2 +- src/prism.c | 9 +++++---- templates/src/serialize.c.erb | 10 +++++----- 5 files changed, 18 insertions(+), 24 deletions(-) diff --git a/ext/prism/extension.c b/ext/prism/extension.c index 96bfa36b20..8a1f165bdd 100644 --- a/ext/prism/extension.c +++ b/ext/prism/extension.c @@ -463,21 +463,15 @@ parser_location(VALUE source, bool freeze, uint32_t start, uint32_t length) { /** * Create a new Location instance from the given parser and slice. */ -#define PARSER_LOCATION_SLICE(parser, source, freeze, slice) \ +#define PARSER_LOCATION_SLICE(source, freeze, slice) \ parser_location(source, freeze, slice.start, slice.length) -/** - * Create a new Location instance from the given parser and location. - */ -#define PARSER_LOCATION_LOC(parser, source, freeze, loc) \ - parser_location(source, freeze, (uint32_t) (loc.start - parser->start), (uint32_t) (loc.end - loc.start)) - /** * Build a new Comment instance from the given parser and comment. */ static inline VALUE -parser_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_comment_t *comment) { - VALUE argv[] = { PARSER_LOCATION_LOC(parser, source, freeze, comment->location) }; +parser_comment(VALUE source, bool freeze, const pm_comment_t *comment) { + VALUE argv[] = { PARSER_LOCATION_SLICE(source, freeze, comment->location) }; VALUE type = (comment->type == PM_COMMENT_EMBDOC) ? rb_cPrismEmbDocComment : rb_cPrismInlineComment; return rb_class_new_instance_freeze(1, argv, type, freeze); } @@ -494,7 +488,7 @@ parser_comments(const pm_parser_t *parser, VALUE source, bool freeze) { comment != NULL; comment = (const pm_comment_t *) comment->node.next ) { - VALUE value = parser_comment(parser, source, freeze, comment); + VALUE value = parser_comment(source, freeze, comment); rb_ary_push(comments, value); } @@ -560,7 +554,7 @@ parser_errors(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bo ) { VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(error->diag_id))); VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(error->message, encoding)); - VALUE location = PARSER_LOCATION_SLICE(parser, source, freeze, error->location); + VALUE location = PARSER_LOCATION_SLICE(source, freeze, error->location); VALUE level = Qnil; switch (error->level) { @@ -600,7 +594,7 @@ parser_warnings(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, ) { VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(warning->diag_id))); VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(warning->message, encoding)); - VALUE location = PARSER_LOCATION_SLICE(parser, source, freeze, warning->location); + VALUE location = PARSER_LOCATION_SLICE(source, freeze, warning->location); VALUE level = Qnil; switch (warning->level) { diff --git a/include/prism.h b/include/prism.h index dc31f26e78..0747f64365 100644 --- a/include/prism.h +++ b/include/prism.h @@ -143,11 +143,10 @@ PRISM_EXPORTED_FUNCTION void pm_serialize_parse_stream(pm_buffer_t *buffer, void /** * Serialize the given list of comments to the given buffer. * - * @param parser The parser to serialize. * @param list The list of comments to serialize. * @param buffer The buffer to serialize to. */ -void pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer); +void pm_serialize_comment_list(pm_list_t *list, pm_buffer_t *buffer); /** * Serialize the name of the encoding to the buffer. diff --git a/include/prism/parser.h b/include/prism/parser.h index c4edd128f0..4e5d380ff0 100644 --- a/include/prism/parser.h +++ b/include/prism/parser.h @@ -463,7 +463,7 @@ typedef struct pm_comment { pm_list_node_t node; /** The location of the comment in the source. */ - pm_location_t location; + pm_slice_t location; /** The type of comment that we've found. */ pm_comment_type_t type; diff --git a/src/prism.c b/src/prism.c index 761a96bffc..1518c13b5f 100644 --- a/src/prism.c +++ b/src/prism.c @@ -9277,7 +9277,7 @@ parser_comment(pm_parser_t *parser, pm_comment_type_t type) { *comment = (pm_comment_t) { .type = type, - .location = { parser->current.start, parser->current.end } + .location = TOKEN2SLICE(parser, &parser->current) }; return comment; @@ -9304,6 +9304,7 @@ lex_embdoc(pm_parser_t *parser) { parser_lex_callback(parser); // Now, create a comment that is going to be attached to the parser. + const uint8_t *comment_start = parser->current.start; pm_comment_t *comment = parser_comment(parser, PM_COMMENT_EMBDOC); if (comment == NULL) return PM_TOKEN_EOF; @@ -9336,7 +9337,7 @@ lex_embdoc(pm_parser_t *parser) { parser->current.type = PM_TOKEN_EMBDOC_END; parser_lex_callback(parser); - comment->location.end = parser->current.end; + comment->location.length = (uint32_t) (parser->current.end - comment_start); pm_list_append(&parser->comment_list, (pm_list_node_t *) comment); return PM_TOKEN_EMBDOC_END; @@ -9359,7 +9360,7 @@ lex_embdoc(pm_parser_t *parser) { pm_parser_err_current(parser, PM_ERR_EMBDOC_TERM); - comment->location.end = parser->current.end; + comment->location.length = (uint32_t) (parser->current.end - comment_start); pm_list_append(&parser->comment_list, (pm_list_node_t *) comment); return PM_TOKEN_EOF; @@ -22288,7 +22289,7 @@ pm_serialize_parse_comments(pm_buffer_t *buffer, const uint8_t *source, size_t s pm_serialize_header(buffer); pm_serialize_encoding(parser.encoding, buffer); pm_buffer_append_varsint(buffer, parser.start_line); - pm_serialize_comment_list(&parser, &parser.comment_list, buffer); + pm_serialize_comment_list(&parser.comment_list, buffer); pm_node_destroy(&parser, node); pm_parser_free(&parser); diff --git a/templates/src/serialize.c.erb b/templates/src/serialize.c.erb index ac79b643dd..142f15070e 100644 --- a/templates/src/serialize.c.erb +++ b/templates/src/serialize.c.erb @@ -188,24 +188,24 @@ pm_serialize_newline_list(pm_newline_list_t *list, pm_buffer_t *buffer) { } static void -pm_serialize_comment(pm_parser_t *parser, pm_comment_t *comment, pm_buffer_t *buffer) { +pm_serialize_comment(pm_comment_t *comment, pm_buffer_t *buffer) { // serialize type pm_buffer_append_byte(buffer, (uint8_t) comment->type); // serialize location - pm_serialize_location(parser, &comment->location, buffer); + pm_serialize_slice(&comment->location, buffer); } /** * Serialize the given list of comments to the given buffer. */ void -pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) { +pm_serialize_comment_list(pm_list_t *list, pm_buffer_t *buffer) { pm_buffer_append_varuint(buffer, pm_sizet_to_u32(pm_list_size(list))); pm_comment_t *comment; for (comment = (pm_comment_t *) list->head; comment != NULL; comment = (pm_comment_t *) comment->node.next) { - pm_serialize_comment(parser, comment, buffer); + pm_serialize_comment(comment, buffer); } } @@ -282,7 +282,7 @@ pm_serialize_metadata(pm_parser_t *parser, pm_buffer_t *buffer) { pm_buffer_append_varsint(buffer, parser->start_line); pm_serialize_newline_list(&parser->newline_list, buffer); <%- unless Prism::Template::SERIALIZE_ONLY_SEMANTICS_FIELDS -%> - pm_serialize_comment_list(parser, &parser->comment_list, buffer); + pm_serialize_comment_list(&parser->comment_list, buffer); <%- end -%> pm_serialize_magic_comment_list(&parser->magic_comment_list, buffer); pm_serialize_data_loc(parser, buffer); From df06a2c919507b49239490adb0c74e8b3cc8c484 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 15:48:55 -0500 Subject: [PATCH 09/22] Switch arguments over to using slices --- src/prism.c | 113 +++++++++++++++++++++++++++------------------------- 1 file changed, 59 insertions(+), 54 deletions(-) diff --git a/src/prism.c b/src/prism.c index 1518c13b5f..78cb19bb66 100644 --- a/src/prism.c +++ b/src/prism.c @@ -1563,13 +1563,13 @@ pm_conditional_predicate(pm_parser_t *parser, pm_node_t *node, pm_conditional_pr */ typedef struct { /** The optional location of the opening parenthesis or bracket. */ - pm_location_t opening_loc; + pm_slice_t opening_loc; /** The lazily-allocated optional arguments node. */ pm_arguments_node_t *arguments; /** The optional location of the closing parenthesis or bracket. */ - pm_location_t closing_loc; + pm_slice_t closing_loc; /** The optional block attached to the call. */ pm_node_t *block; @@ -1582,21 +1582,26 @@ typedef struct { * Retrieve the end location of a `pm_arguments_t` object. */ static inline const uint8_t * -pm_arguments_end(pm_arguments_t *arguments) { +pm_arguments_end(const pm_parser_t *parser, pm_arguments_t *arguments) { if (arguments->block != NULL) { const uint8_t *end = arguments->block->location.end; - if (arguments->closing_loc.start != NULL && arguments->closing_loc.end > end) { - end = arguments->closing_loc.end; + + if (arguments->closing_loc.length > 0) { + const uint8_t *arguments_end = parser->start + arguments->closing_loc.start + arguments->closing_loc.length; + + if (arguments_end > end) { + end = arguments_end; + } } return end; } - if (arguments->closing_loc.start != NULL) { - return arguments->closing_loc.end; + if (arguments->closing_loc.length > 0) { + return parser->start + arguments->closing_loc.start + arguments->closing_loc.length; } if (arguments->arguments != NULL) { return arguments->arguments->base.location.end; } - return arguments->closing_loc.end; + return NULL; } /** @@ -1607,7 +1612,7 @@ static void pm_arguments_validate_block(pm_parser_t *parser, pm_arguments_t *arguments, pm_block_node_t *block) { // First, check that we have arguments and that we don't have a closing // location for them. - if (arguments->arguments == NULL || arguments->closing_loc.start != NULL) { + if (arguments->arguments == NULL || arguments->closing_loc.length > 0) { return; } @@ -2589,15 +2594,15 @@ pm_call_node_aref_create(pm_parser_t *parser, pm_node_t *receiver, pm_arguments_ pm_call_node_t *node = pm_call_node_create(parser, flags); node->base.location.start = receiver->location.start; - node->base.location.end = pm_arguments_end(arguments); + node->base.location.end = pm_arguments_end(parser, arguments); node->receiver = receiver; - node->message_loc.start = (arguments->opening_loc.start == NULL ? 0 : ((uint32_t) (arguments->opening_loc.start - parser->start))); - node->message_loc.length = (arguments->closing_loc.end == NULL ? 0 : ((uint32_t) (arguments->closing_loc.end - arguments->opening_loc.start))); + node->message_loc.start = arguments->opening_loc.start; + node->message_loc.length = (arguments->closing_loc.start + arguments->closing_loc.length) - arguments->opening_loc.start; - node->opening_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc); + node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; - node->closing_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc); + node->closing_loc = arguments->closing_loc; node->block = arguments->block; node->name = pm_parser_constant_id_constant(parser, "[]", 2); @@ -2640,7 +2645,7 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); node->base.location.start = receiver->location.start; - const uint8_t *end = pm_arguments_end(arguments); + const uint8_t *end = pm_arguments_end(parser, arguments); if (end == NULL) { end = message->end; } @@ -2649,9 +2654,9 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o node->receiver = receiver; node->call_operator_loc = TOKEN2SLICE(parser, operator); node->message_loc = TOKEN2SLICE(parser, message); - node->opening_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc); + node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; - node->closing_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc); + node->closing_loc = arguments->closing_loc; node->block = arguments->block; if (operator->type == PM_TOKEN_AMPERSAND_DOT) { @@ -2691,12 +2696,12 @@ pm_call_node_fcall_create(pm_parser_t *parser, pm_token_t *message, pm_arguments pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); node->base.location.start = message->start; - node->base.location.end = pm_arguments_end(arguments); + node->base.location.end = pm_arguments_end(parser, arguments); node->message_loc = TOKEN2SLICE(parser, message); - node->opening_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc); + node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; - node->closing_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc); + node->closing_loc = arguments->closing_loc; node->block = arguments->block; node->name = pm_parser_constant_id_token(parser, message); @@ -2729,8 +2734,8 @@ pm_call_node_not_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *me pm_call_node_t *node = pm_call_node_create(parser, receiver == NULL ? 0 : pm_call_node_ignore_visibility_flag(receiver)); node->base.location.start = message->start; - if (arguments->closing_loc.start != NULL) { - node->base.location.end = arguments->closing_loc.end; + if (arguments->closing_loc.length > 0) { + node->base.location.end = parser->start + arguments->closing_loc.start + arguments->closing_loc.length; } else { assert(receiver != NULL); node->base.location.end = receiver->location.end; @@ -2738,9 +2743,9 @@ pm_call_node_not_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *me node->receiver = receiver; node->message_loc = TOKEN2SLICE(parser, message); - node->opening_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc); + node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; - node->closing_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc); + node->closing_loc = arguments->closing_loc; node->name = pm_parser_constant_id_constant(parser, "!", 1); return node; @@ -2756,13 +2761,13 @@ pm_call_node_shorthand_create(pm_parser_t *parser, pm_node_t *receiver, pm_token pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); node->base.location.start = receiver->location.start; - node->base.location.end = pm_arguments_end(arguments); + node->base.location.end = pm_arguments_end(parser, arguments); node->receiver = receiver; node->call_operator_loc = TOKEN2SLICE(parser, operator); - node->opening_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc); + node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; - node->closing_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc); + node->closing_loc = arguments->closing_loc; node->block = arguments->block; if (operator->type == PM_TOKEN_AMPERSAND_DOT) { @@ -6311,7 +6316,7 @@ pm_super_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument assert(keyword->type == PM_TOKEN_KEYWORD_SUPER); pm_super_node_t *node = PM_NODE_ALLOC(parser, pm_super_node_t); - const uint8_t *end = pm_arguments_end(arguments); + const uint8_t *end = pm_arguments_end(parser, arguments); if (end == NULL) { assert(false && "unreachable"); } @@ -6319,9 +6324,9 @@ pm_super_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument *node = (pm_super_node_t) { .base = PM_NODE_INIT(parser, PM_SUPER_NODE, 0, keyword->start, end), .keyword_loc = TOKEN2SLICE(parser, keyword), - .lparen_loc = arguments->opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->opening_loc), + .lparen_loc = arguments->opening_loc, .arguments = arguments->arguments, - .rparen_loc = arguments->closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments->closing_loc), + .rparen_loc = arguments->closing_loc, .block = arguments->block }; @@ -7017,16 +7022,16 @@ pm_xstring_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_ * Allocate a new YieldNode node. */ static pm_yield_node_t * -pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_location_t *lparen_loc, pm_arguments_node_t *arguments, const pm_location_t *rparen_loc) { +pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_slice_t *lparen_loc, pm_arguments_node_t *arguments, const pm_slice_t *rparen_loc) { pm_yield_node_t *node = PM_NODE_ALLOC(parser, pm_yield_node_t); const uint8_t *end; - if (rparen_loc != NULL) { - end = rparen_loc->end; + if (rparen_loc->length > 0) { + end = parser->start + rparen_loc->start + rparen_loc->length; } else if (arguments != NULL) { end = arguments->base.location.end; - } else if (lparen_loc != NULL) { - end = lparen_loc->end; + } else if (lparen_loc->length > 0) { + end = parser->start + lparen_loc->start + lparen_loc->length; } else { end = keyword->end; } @@ -7034,9 +7039,9 @@ pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_lo *node = (pm_yield_node_t) { .base = PM_NODE_INIT(parser, PM_YIELD_NODE, 0, keyword->start, end), .keyword_loc = TOKEN2SLICE(parser, keyword), - .lparen_loc = MAYBELOCATION2SLICE(parser, lparen_loc), + .lparen_loc = *lparen_loc, .arguments = arguments, - .rparen_loc = MAYBELOCATION2SLICE(parser, rparen_loc) + .rparen_loc = *rparen_loc }; return node; @@ -14730,10 +14735,10 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept if (accept1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { found |= true; - arguments->opening_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments->opening_loc = TOKEN2SLICE(parser, &parser->previous); if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - arguments->closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments->closing_loc = TOKEN2SLICE(parser, &parser->previous); } else { pm_accepts_block_stack_push(parser, true); parse_arguments(parser, arguments, accepts_block, PM_TOKEN_PARENTHESIS_RIGHT, (uint16_t) (depth + 1)); @@ -14745,7 +14750,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept } pm_accepts_block_stack_pop(parser); - arguments->closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments->closing_loc = TOKEN2SLICE(parser, &parser->previous); } } else if (accepts_command_call && (token_begins_expression_p(parser->current.type) || match3(parser, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR, PM_TOKEN_UAMPERSAND)) && !match1(parser, PM_TOKEN_BRACE_LEFT)) { found |= true; @@ -16298,7 +16303,7 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca .end = parser->start + value_slice->start + value_slice->length }; const pm_location_t *value_loc = &value_location; - + pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); int depth = -1; @@ -17823,12 +17828,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // variable call bit in the flags. pm_node_flag_unset(UP(call), PM_CALL_NODE_FLAGS_VARIABLE_CALL); - call->opening_loc = arguments.opening_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments.opening_loc); + call->opening_loc = arguments.opening_loc; call->arguments = arguments.arguments; - call->closing_loc = arguments.closing_loc.start == NULL ? ((pm_slice_t) { 0 }) : LOCATION2SLICE(parser, &arguments.closing_loc); + call->closing_loc = arguments.closing_loc; call->block = arguments.block; - const uint8_t *end = pm_arguments_end(&arguments); + const uint8_t *end = pm_arguments_end(parser, &arguments); if (!end) { end = parser->start + call->message_loc.start + call->message_loc.length; } @@ -18405,7 +18410,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parse_arguments_list(parser, &arguments, true, accepts_command_call, (uint16_t) (depth + 1)); if ( - arguments.opening_loc.start == NULL && + arguments.opening_loc.length == 0 && arguments.arguments == NULL && ((arguments.block == NULL) || PM_NODE_TYPE_P(arguments.block, PM_BLOCK_NODE)) ) { @@ -18431,7 +18436,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b arguments.block = NULL; } - pm_node_t *node = UP(pm_yield_node_create(parser, &keyword, arguments.opening_loc.start == NULL ? NULL : &arguments.opening_loc, arguments.arguments, arguments.closing_loc.start == NULL ? NULL : &arguments.closing_loc)); + pm_node_t *node = UP(pm_yield_node_create(parser, &keyword, &arguments.opening_loc, arguments.arguments, &arguments.closing_loc)); if (!parser->parsing_eval && !parser->partial_script) parse_yield(parser, node); return node; @@ -19053,13 +19058,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { receiver = UP(pm_parentheses_node_create(parser, &lparen, NULL, &parser->previous, 0)); } else { - arguments.opening_loc = PM_LOCATION_TOKEN_VALUE(&lparen); + arguments.opening_loc = TOKEN2SLICE(parser, &lparen); receiver = parse_expression(parser, PM_BINDING_POWER_COMPOSITION, true, false, PM_ERR_NOT_EXPRESSION, (uint16_t) (depth + 1)); if (!parser->recovering) { accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN); - arguments.closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments.closing_loc = TOKEN2SLICE(parser, &parser->previous); } } } else { @@ -20681,7 +20686,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { pm_location_t message_loc = (pm_location_t) { .start = parser->start + cast->message_loc.start, - .end = parser->start + cast->message_loc.start + cast->message_loc.length + .end = parser->start + cast->message_loc.start + cast->message_loc.length }; pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); @@ -20829,9 +20834,9 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { pm_location_t message_loc = (pm_location_t) { .start = parser->start + cast->message_loc.start, - .end = parser->start + cast->message_loc.start + cast->message_loc.length + .end = parser->start + cast->message_loc.start + cast->message_loc.length }; - + pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); pm_constant_id_t constant_id = pm_parser_local_add_slice(parser, &cast->message_loc, 1); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1)); @@ -21081,7 +21086,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t if ( (previous_binding_power == PM_BINDING_POWER_STATEMENT) && arguments.arguments == NULL && - arguments.opening_loc.start == NULL && + arguments.opening_loc.length == 0 && match1(parser, PM_TOKEN_COMMA) ) { return parse_targets_validate(parser, UP(call), PM_BINDING_POWER_INDEX, (uint16_t) (depth + 1)); @@ -21253,7 +21258,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t parser_lex(parser); pm_arguments_t arguments = { 0 }; - arguments.opening_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments.opening_loc = TOKEN2SLICE(parser, &parser->previous); if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { pm_accepts_block_stack_push(parser, true); @@ -21262,7 +21267,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_EXPECT_RBRACKET); } - arguments.closing_loc = PM_LOCATION_TOKEN_VALUE(&parser->previous); + arguments.closing_loc = TOKEN2SLICE(parser, &parser->previous); // If we have a comma after the closing bracket then this is a multiple // assignment and we should parse the targets. From 0f1ae8cb0bf91c5e3b6fa5944d7863d71050f92b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 15:50:31 -0500 Subject: [PATCH 10/22] Remove unnecessary macros --- src/prism.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/prism.c b/src/prism.c index 78cb19bb66..1f4113f494 100644 --- a/src/prism.c +++ b/src/prism.c @@ -1958,7 +1958,6 @@ pm_missing_node_create(pm_parser_t *parser, const uint8_t *start, const uint8_t #define MAYBETOKEN2SLICE(parser_, token_) ((token_) == NULL ? ((pm_slice_t) { 0 }) : TOKEN2SLICE(parser_, token_)) #define MAYBETOKENPTR(token_) ((token_).start == NULL ? NULL : &(token_)) #define LOCATION2SLICE TOKEN2SLICE -#define MAYBELOCATION2SLICE MAYBETOKEN2SLICE /** * Allocate and initialize a new AliasGlobalVariableNode node. @@ -2716,7 +2715,7 @@ static pm_call_node_t * pm_call_node_fcall_synthesized_create(pm_parser_t *parser, pm_arguments_node_t *arguments, pm_constant_id_t name) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location = PM_LOCATION_NULL_VALUE(parser); + node->base.location = ((pm_location_t) { .start = parser->start, .end = parser->start }); node->arguments = arguments; node->name = name; From a472de2191756dbb228e8dd5ec5db9ca051c8b08 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 15:56:45 -0500 Subject: [PATCH 11/22] Switch locals over to using slices --- include/prism/parser.h | 2 +- src/prism.c | 44 ++++++++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/include/prism/parser.h b/include/prism/parser.h index 4e5d380ff0..be78015bd6 100644 --- a/include/prism/parser.h +++ b/include/prism/parser.h @@ -531,7 +531,7 @@ typedef struct { pm_constant_id_t name; /** The location of the local variable in the source. */ - pm_location_t location; + pm_slice_t location; /** The index of the local variable in the local table. */ uint32_t index; diff --git a/src/prism.c b/src/prism.c index 1f4113f494..a7bc0bfa2c 100644 --- a/src/prism.c +++ b/src/prism.c @@ -530,17 +530,25 @@ pm_parser_warn_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id } /** - * Append a warning to the list of warnings on the parser using a format string. + * Append a warning to the list of warnings on the parser using a format string + * and the given location. */ -#define PM_PARSER_WARN_FORMAT(parser_, start_, end_, diag_id_, ...) \ +#define PM_PARSER_WARN_FORMAT_LOCATION(parser_, start_, end_, diag_id_, ...) \ pm_diagnostic_list_append_format(&(parser_)->warning_list, (uint32_t) ((start_) - (parser_)->start), (uint32_t) ((end_) - (start_)), diag_id_, __VA_ARGS__) +/** + * Append a warning to the list of warnings on the parser using a format string + * and the given slice. + */ +#define PM_PARSER_WARN_FORMAT_SLICE(parser_, slice_, diag_id_, ...) \ + pm_diagnostic_list_append_format(&(parser_)->warning_list, (slice_)->start, (slice_)->length, diag_id_, __VA_ARGS__) + /** * Append a warning to the list of warnings on the parser using the location of * the given token and a format string. */ #define PM_PARSER_WARN_TOKEN_FORMAT(parser, token, diag_id, ...) \ - PM_PARSER_WARN_FORMAT(parser, (token).start, (token).end, diag_id, __VA_ARGS__) + PM_PARSER_WARN_FORMAT_LOCATION(parser, (token).start, (token).end, diag_id, __VA_ARGS__) /** * Append a warning to the list of warnings on the parser using the location of @@ -554,7 +562,7 @@ pm_parser_warn_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id * the given node and a format string. */ #define PM_PARSER_WARN_NODE_FORMAT(parser, node, diag_id, ...) \ - PM_PARSER_WARN_FORMAT(parser, (node)->location.start, (node)->location.end, diag_id, __VA_ARGS__) + PM_PARSER_WARN_FORMAT_LOCATION(parser, (node)->location.start, (node)->location.end, diag_id, __VA_ARGS__) /** * Add an error for an expected heredoc terminator. This is a special function @@ -828,7 +836,7 @@ pm_locals_resize(pm_locals_t *locals) { * @return True if the local was added, and false if the local already exists. */ static bool -pm_locals_write(pm_locals_t *locals, pm_constant_id_t name, const uint8_t *start, const uint8_t *end, uint32_t reads) { +pm_locals_write(pm_locals_t *locals, pm_constant_id_t name, uint32_t start, uint32_t length, uint32_t reads) { if (locals->size >= (locals->capacity / 4 * 3)) { pm_locals_resize(locals); } @@ -840,7 +848,7 @@ pm_locals_write(pm_locals_t *locals, pm_constant_id_t name, const uint8_t *start if (local->name == PM_CONSTANT_ID_UNSET) { *local = (pm_local_t) { .name = name, - .location = { .start = start, .end = end }, + .location = { .start = start, .length = length }, .index = locals->size++, .reads = reads, .hash = 0 @@ -861,7 +869,7 @@ pm_locals_write(pm_locals_t *locals, pm_constant_id_t name, const uint8_t *start if (local->name == PM_CONSTANT_ID_UNSET) { *local = (pm_local_t) { .name = name, - .location = { .start = start, .end = end }, + .location = { .start = start, .length = length }, .index = locals->size++, .reads = reads, .hash = initial_hash @@ -979,14 +987,13 @@ pm_locals_order(PRISM_ATTRIBUTE_UNUSED pm_parser_t *parser, pm_locals_t *locals, if (local->name != PM_CONSTANT_ID_UNSET) { pm_constant_id_list_insert(list, (size_t) local->index, local->name); - if (warn_unused && local->reads == 0 && ((parser->start_line >= 0) || (pm_newline_list_line(&parser->newline_list, local->location.start, parser->start_line) >= 0))) { + if (warn_unused && local->reads == 0 && ((parser->start_line >= 0) || (pm_newline_list_line(&parser->newline_list, parser->start + local->location.start, parser->start_line) >= 0))) { pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, local->name); if (constant->length >= 1 && *constant->start != '_') { - PM_PARSER_WARN_FORMAT( + PM_PARSER_WARN_FORMAT_SLICE( parser, - local->location.start, - local->location.end, + &local->location, PM_WARN_UNUSED_LOCAL_VARIABLE, (int) constant->length, (const char *) constant->start @@ -5547,7 +5554,7 @@ pm_numbered_reference_read_node_number(pm_parser_t *parser, const pm_token_t *to xfree(digits); if ((errno == ERANGE) || (value > NTH_REF_MAX)) { - PM_PARSER_WARN_FORMAT(parser, start, end, PM_WARN_INVALID_NUMBERED_REFERENCE, (int) (length + 1), (const char *) token->start); + PM_PARSER_WARN_FORMAT_LOCATION(parser, start, end, PM_WARN_INVALID_NUMBERED_REFERENCE, (int) (length + 1), (const char *) token->start); value = 0; } @@ -7081,7 +7088,7 @@ pm_parser_local_depth(pm_parser_t *parser, pm_token_t *token) { */ static inline void pm_parser_local_add(pm_parser_t *parser, pm_constant_id_t constant_id, const uint8_t *start, const uint8_t *end, uint32_t reads) { - pm_locals_write(&parser->current_scope->locals, constant_id, start, end, reads); + pm_locals_write(&parser->current_scope->locals, constant_id, (uint32_t) (start - parser->start), (uint32_t) (end - start), reads); } /** @@ -14245,7 +14252,7 @@ parser_warn_indentation_mismatch(pm_parser_t *parser, size_t opening_newline_ind if (allow_indent && (closing_column > opening_column)) return; // Otherwise, add a warning. - PM_PARSER_WARN_FORMAT( + PM_PARSER_WARN_FORMAT_LOCATION( parser, closing_token->start, closing_token->end, @@ -16207,18 +16214,17 @@ parse_pattern_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) { // will check for that here. If they do, then we'll add it to the local // table since this pattern will cause it to become a local variable. if (accept1(parser, PM_TOKEN_IDENTIFIER)) { - pm_token_t identifier = parser->previous; - pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &identifier); + pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous); int depth; if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { - pm_parser_local_add(parser, constant_id, identifier.start, identifier.end, 0); + pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&identifier)); + parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); name = UP(pm_local_variable_target_node_create( parser, - &PM_LOCATION_TOKEN_VALUE(&identifier), + &PM_LOCATION_TOKEN_VALUE(&parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) )); From d5064ac252eac6c0d8f2b9aa9d178c8e5c68d2ef Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 22:05:41 -0500 Subject: [PATCH 12/22] Switch the config back referencing locations --- config.yml | 456 ++++++++++---------- rbi/prism/reflection.rbi | 6 - rust/ruby-prism-sys/tests/parser_tests.rs | 8 +- rust/ruby-prism/build.rs | 23 - rust/ruby-prism/src/lib.rs | 42 +- sig/prism/reflection.rbs | 6 - templates/ext/prism/api_node.c.erb | 6 - templates/include/prism/ast.h.erb | 3 +- templates/java/org/prism/Loader.java.erb | 4 +- templates/java/org/prism/Nodes.java.erb | 2 +- templates/javascript/src/deserialize.js.erb | 4 +- templates/javascript/src/nodes.js.erb | 4 +- templates/lib/prism/dot_visitor.rb.erb | 4 +- templates/lib/prism/dsl.rb.erb | 4 +- templates/lib/prism/inspect_visitor.rb.erb | 2 +- templates/lib/prism/node.rb.erb | 14 +- templates/lib/prism/reflection.rb.erb | 15 - templates/lib/prism/serialize.rb.erb | 8 +- templates/rbi/prism/dsl.rbi.erb | 4 +- templates/rbi/prism/node.rbi.erb | 4 +- templates/sig/prism/node.rbs.erb | 6 +- templates/src/node.c.erb | 12 +- templates/src/prettyprint.c.erb | 18 - templates/src/serialize.c.erb | 13 - templates/template.rb | 42 -- 25 files changed, 292 insertions(+), 418 deletions(-) diff --git a/config.yml b/config.yml index 9b7360e31a..f88b98679d 100644 --- a/config.yml +++ b/config.yml @@ -815,7 +815,7 @@ nodes: alias $foo $bar ^^^^ - name: keyword_loc - type: slice + type: location comment: | The location of the `alias` keyword. @@ -863,7 +863,7 @@ nodes: alias :"#{foo}" :"#{bar}" ^^^^^^^^^ - name: keyword_loc - type: slice + type: location comment: | Represents the location of the `alias` keyword. @@ -893,7 +893,7 @@ nodes: foo => bar | baz ^^^ - name: operator_loc - type: slice + type: location comment: | Represents the alternation operator location. @@ -929,7 +929,7 @@ nodes: 1 and 2 ^ - name: operator_loc - type: slice + type: location comment: | The location of the `and` keyword or the `&&` operator. @@ -964,7 +964,7 @@ nodes: kind: non-void expression comment: Represent the list of zero or more [non-void expressions](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression) within the array. - name: opening_loc - type: slice? + type: location? comment: | Represents the optional source location for the opening token. @@ -973,7 +973,7 @@ nodes: %I(apple orange banana) # "%I(" foo = 1, 2, 3 # nil - name: closing_loc - type: slice? + type: location? comment: | Represents the optional source location for the closing token. @@ -1029,14 +1029,14 @@ nodes: foo in *bar, baz ^^^ - name: opening_loc - type: slice? + type: location? comment: | Represents the opening location of the array pattern. foo in [1, 2] ^ - name: closing_loc - type: slice? + type: location? comment: | Represents the closing location of the array pattern. @@ -1087,7 +1087,7 @@ nodes: { x: 1 } ^ - name: operator_loc - type: slice? + type: location? comment: | The location of the `=>` operator, if present. @@ -1109,7 +1109,7 @@ nodes: { **foo } ^^^ - name: operator_loc - type: slice + type: location comment: | The location of the `**` operator. @@ -1138,7 +1138,7 @@ nodes: - name: BeginNode fields: - name: begin_keyword_loc - type: slice? + type: location? comment: | Represents the location of the `begin` keyword. @@ -1177,7 +1177,7 @@ nodes: begin x; ensure y; end ^^^^^^^^ - name: end_keyword_loc - type: slice? + type: location? comment: | Represents the location of the `end` keyword. @@ -1202,7 +1202,7 @@ nodes: foo(&args) ^^^^^ - name: operator_loc - type: slice + type: location comment: | Represents the location of the `&` operator. @@ -1263,14 +1263,14 @@ nodes: [1, 2, 3].each { |i| puts x } ^^^^^^ - name: opening_loc - type: slice + type: location comment: | Represents the location of the opening `|`. [1, 2, 3].each { |i| puts x } ^ - name: closing_loc - type: slice + type: location comment: | Represents the location of the closing `|`. @@ -1293,14 +1293,14 @@ nodes: ^ end - name: name_loc - type: slice? + type: location? comment: | Represents the location of the block parameter name. def a(&b) ^ - name: operator_loc - type: slice + type: location comment: | Represents the location of the `&` operator. @@ -1340,7 +1340,7 @@ nodes: ^^^^^ end - name: opening_loc - type: slice? + type: location? comment: | Represents the opening location of the block parameters. @@ -1351,7 +1351,7 @@ nodes: ^ end - name: closing_loc - type: slice? + type: location? comment: | Represents the closing location of the block parameters. @@ -1381,7 +1381,7 @@ nodes: break foo ^^^ - name: keyword_loc - type: slice + type: location comment: | The location of the `break` keyword. @@ -1404,14 +1404,14 @@ nodes: foo.bar &&= value ^^^ - name: call_operator_loc - type: slice? + type: location? comment: | Represents the location of the call operator. foo.bar &&= value ^ - name: message_loc - type: slice? + type: location? comment: | Represents the location of the message. @@ -1432,7 +1432,7 @@ nodes: foo.bar &&= value # write_name `:bar=` ^^^ - name: operator_loc - type: slice + type: location comment: | Represents the location of the operator. @@ -1469,7 +1469,7 @@ nodes: foo + bar ^^^ - name: call_operator_loc - type: slice? + type: location? comment: | Represents the location of the call operator. @@ -1486,14 +1486,14 @@ nodes: foo.bar # name `:foo` ^^^ - name: message_loc - type: slice? + type: location? comment: | Represents the location of the message. foo.bar ^^^ - name: opening_loc - type: slice? + type: location? comment: | Represents the location of the left parenthesis. foo(bar) @@ -1507,14 +1507,14 @@ nodes: foo(bar) ^^^ - name: closing_loc - type: slice? + type: location? comment: | Represents the location of the right parenthesis. foo(bar) ^ - name: equal_loc - type: slice? + type: location? comment: | Represents the location of the equal sign, in the case that this is an attribute write. @@ -1565,14 +1565,14 @@ nodes: foo.bar += value ^^^ - name: call_operator_loc - type: slice? + type: location? comment: | Represents the location of the call operator. foo.bar += value ^ - name: message_loc - type: slice? + type: location? comment: | Represents the location of the message. @@ -1600,7 +1600,7 @@ nodes: foo.bar += value # binary_operator `:+` ^ - name: binary_operator_loc - type: slice + type: location comment: | Represents the location of the binary operator. @@ -1631,14 +1631,14 @@ nodes: foo.bar ||= value ^^^ - name: call_operator_loc - type: slice? + type: location? comment: | Represents the location of the call operator. foo.bar ||= value ^ - name: message_loc - type: slice? + type: location? comment: | Represents the location of the message. @@ -1659,7 +1659,7 @@ nodes: foo.bar ||= value # write_name `:bar=` ^^^ - name: operator_loc - type: slice + type: location comment: | Represents the location of the operator. @@ -1690,7 +1690,7 @@ nodes: foo.bar = 1 ^^^ - name: call_operator_loc - type: slice + type: location comment: | Represents the location of the call operator. @@ -1704,7 +1704,7 @@ nodes: foo.bar = 1 # name `:foo` ^^^ - name: message_loc - type: slice + type: location comment: | Represents the location of the message. @@ -1742,7 +1742,7 @@ nodes: foo => bar ^^^ - name: operator_loc - type: slice + type: location comment: | Represents the location of the `=>` operator. @@ -1780,14 +1780,14 @@ nodes: case true; in false; else; end ^^^^ - name: case_keyword_loc - type: slice + type: location comment: | Represents the location of the `case` keyword. case true; in false; end ^^^^ - name: end_keyword_loc - type: slice + type: location comment: | Represents the location of the `end` keyword. @@ -1827,14 +1827,14 @@ nodes: case true; when false; else; end ^^^^ - name: case_keyword_loc - type: slice + type: location comment: | Represents the location of the `case` keyword. case true; when false; end ^^^^ - name: end_keyword_loc - type: slice + type: location comment: | Represents the location of the `end` keyword. @@ -1852,7 +1852,7 @@ nodes: - name: locals type: constant[] - name: class_keyword_loc - type: slice + type: location comment: | Represents the location of the `class` keyword. @@ -1865,7 +1865,7 @@ nodes: - ConstantPathNode - on error: CallNode # class 0.X end - name: inheritance_operator_loc - type: slice? + type: location? comment: | Represents the location of the `<` operator. @@ -1891,7 +1891,7 @@ nodes: foo ^^^ - name: end_keyword_loc - type: slice + type: location comment: | Represents the location of the `end` keyword. @@ -1918,14 +1918,14 @@ nodes: @@target &&= value # name `:@@target` ^^^^^^^^ - name: name_loc - type: slice + type: location comment: | Represents the location of the variable name. @@target &&= value ^^^^^^^^ - name: operator_loc - type: slice + type: location comment: | Represents the location of the `&&=` operator. @@ -1949,9 +1949,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: binary_operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -1967,9 +1967,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -2013,7 +2013,7 @@ nodes: @@_test = :test # name `@@_test` - name: name_loc - type: slice + type: location comment: | The location of the variable name. @@ -2031,7 +2031,7 @@ nodes: @@_xyz = 123 ^^^ - name: operator_loc - type: slice + type: location comment: | The location of the `=` operator. @@ -2047,9 +2047,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -2063,9 +2063,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: binary_operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -2081,9 +2081,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -2098,7 +2098,7 @@ nodes: type: node kind: ConstantPathNode - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -2127,7 +2127,7 @@ nodes: type: constant? comment: The name of the constant being accessed. This could be `nil` in the event of a syntax error. - name: delimiter_loc - type: slice + type: location comment: | The location of the `::` delimiter. @@ -2137,7 +2137,7 @@ nodes: One::Two ^^ - name: name_loc - type: slice + type: location comment: | The location of the name of the constant. @@ -2157,7 +2157,7 @@ nodes: type: node kind: ConstantPathNode - name: binary_operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -2174,7 +2174,7 @@ nodes: type: node kind: ConstantPathNode - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -2191,9 +2191,9 @@ nodes: - name: name type: constant? - name: delimiter_loc - type: slice + type: location - name: name_loc - type: slice + type: location comment: | Represents writing to a constant path in a context that doesn't have an explicit value. @@ -2213,7 +2213,7 @@ nodes: ::Foo = :abc ^^^^^ - name: operator_loc - type: slice + type: location comment: | The location of the `=` operator. @@ -2273,7 +2273,7 @@ nodes: XYZ = 1 # name `:XYZ` - name: name_loc - type: slice + type: location comment: | The location of the constant name. @@ -2291,7 +2291,7 @@ nodes: MyClass = Class.new ^^^^^^^^^ - name: operator_loc - type: slice + type: location comment: | The location of the `=` operator. @@ -2307,7 +2307,7 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: receiver type: node? kind: non-void expression @@ -2322,17 +2322,17 @@ nodes: - name: locals type: constant[] - name: def_keyword_loc - type: slice + type: location - name: operator_loc - type: slice? + type: location? - name: lparen_loc - type: slice? + type: location? - name: rparen_loc - type: slice? + type: location? - name: equal_loc - type: slice? + type: location? - name: end_keyword_loc - type: slice? + type: location? comment: | Represents a method definition. @@ -2342,14 +2342,14 @@ nodes: - name: DefinedNode fields: - name: lparen_loc - type: slice? + type: location? - name: value type: node kind: Node # More than non-void expression as defined?(return) is allowed, yet defined?(BEGIN{}) is SyntaxError - name: rparen_loc - type: slice? + type: location? - name: keyword_loc - type: slice + type: location comment: | Represents the use of the `defined?` keyword. @@ -2358,12 +2358,12 @@ nodes: - name: ElseNode fields: - name: else_keyword_loc - type: slice + type: location - name: statements type: node? kind: StatementsNode - name: end_keyword_loc - type: slice? + type: location? comment: | Represents an `else` clause in a `case`, `if`, or `unless` statement. @@ -2372,12 +2372,12 @@ nodes: - name: EmbeddedStatementsNode fields: - name: opening_loc - type: slice + type: location - name: statements type: node? kind: StatementsNode - name: closing_loc - type: slice + type: location comment: | Represents an interpolated set of statements. @@ -2386,7 +2386,7 @@ nodes: - name: EmbeddedVariableNode fields: - name: operator_loc - type: slice + type: location - name: variable type: node kind: @@ -2403,12 +2403,12 @@ nodes: - name: EnsureNode fields: - name: ensure_keyword_loc - type: slice + type: location - name: statements type: node? kind: StatementsNode - name: end_keyword_loc - type: slice + type: location comment: | Represents an `ensure` clause in a `begin` statement. @@ -2472,7 +2472,7 @@ nodes: foo in Foo(*bar, baz, *qux) ^^^^ - name: opening_loc - type: slice? + type: location? comment: | The location of the opening brace. @@ -2482,7 +2482,7 @@ nodes: foo in Foo(*bar, baz, *qux) ^ - name: closing_loc - type: slice? + type: location? comment: | The location of the closing brace. @@ -2515,7 +2515,7 @@ nodes: type: node? kind: non-void expression - name: operator_loc - type: slice + type: location comment: | Represents the use of the `..` or `...` operators to create flip flops. @@ -2572,28 +2572,28 @@ nodes: ^^^^^^ end - name: for_keyword_loc - type: slice + type: location comment: | The location of the `for` keyword. for i in a end ^^^ - name: in_keyword_loc - type: slice + type: location comment: | The location of the `in` keyword. for i in a end ^^ - name: do_keyword_loc - type: slice? + type: location? comment: | The location of the `do` keyword, if present. for i in a do end ^^ - name: end_keyword_loc - type: slice + type: location comment: | The location of the `end` keyword. @@ -2641,9 +2641,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -2657,9 +2657,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: binary_operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -2675,9 +2675,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -2721,7 +2721,7 @@ nodes: $_Test = 123 # name `:$_Test` - name: name_loc - type: slice + type: location comment: | The location of the global variable's name. @@ -2739,7 +2739,7 @@ nodes: $-xyz = 123 ^^^ - name: operator_loc - type: slice + type: location comment: | The location of the `=` operator. @@ -2753,7 +2753,7 @@ nodes: - name: HashNode fields: - name: opening_loc - type: slice + type: location comment: | The location of the opening brace. @@ -2773,7 +2773,7 @@ nodes: { **foo } ^^^^^ - name: closing_loc - type: slice + type: location comment: | The location of the closing brace. @@ -2824,7 +2824,7 @@ nodes: foo => { a: 1, b:, **nil } ^^^^^ - name: opening_loc - type: slice? + type: location? comment: | The location of the opening brace. @@ -2834,7 +2834,7 @@ nodes: foo => Bar[a: 1] ^ - name: closing_loc - type: slice? + type: location? comment: | The location of the closing brace. @@ -2860,7 +2860,7 @@ nodes: - name: IfNode fields: - name: if_keyword_loc - type: slice? + type: location? comment: | The location of the `if` keyword if present. @@ -2885,7 +2885,7 @@ nodes: foo ? bar : baz ^^^ - name: then_keyword_loc - type: slice? + type: location? comment: | The location of the `then` keyword (if present) or the `?` in a ternary expression, `nil` otherwise. @@ -2926,7 +2926,7 @@ nodes: if foo then bar else baz end ^^^^^^^^^^^^ - name: end_keyword_loc - type: slice? + type: location? comment: | The location of the `end` keyword if present, `nil` otherwise. @@ -3003,9 +3003,9 @@ nodes: type: node? kind: StatementsNode - name: in_loc - type: slice + type: location - name: then_loc - type: slice? + type: location? comment: | Represents the use of the `in` keyword in a case statement. @@ -3018,19 +3018,19 @@ nodes: type: node? kind: non-void expression - name: call_operator_loc - type: slice? + type: location? - name: opening_loc - type: slice + type: location - name: arguments type: node? kind: ArgumentsNode - name: closing_loc - type: slice + type: location - name: block type: node? kind: BlockArgumentNode # foo[&b] &&= value, only valid on Ruby < 3.4 - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -3046,21 +3046,21 @@ nodes: type: node? kind: non-void expression - name: call_operator_loc - type: slice? + type: location? - name: opening_loc - type: slice + type: location - name: arguments type: node? kind: ArgumentsNode - name: closing_loc - type: slice + type: location - name: block type: node? kind: BlockArgumentNode # foo[&b] += value, only valid on Ruby < 3.4 - name: binary_operator type: constant - name: binary_operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -3076,19 +3076,19 @@ nodes: type: node? kind: non-void expression - name: call_operator_loc - type: slice? + type: location? - name: opening_loc - type: slice + type: location - name: arguments type: node? kind: ArgumentsNode - name: closing_loc - type: slice + type: location - name: block type: node? kind: BlockArgumentNode # foo[&b] ||= value, only valid on Ruby < 3.4 - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -3104,12 +3104,12 @@ nodes: type: node kind: non-void expression - name: opening_loc - type: slice + type: location - name: arguments type: node? kind: ArgumentsNode - name: closing_loc - type: slice + type: location - name: block type: node? kind: BlockArgumentNode # foo[&b], = 1, only valid on Ruby < 3.4 @@ -3131,9 +3131,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -3147,9 +3147,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: binary_operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -3165,9 +3165,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -3211,7 +3211,7 @@ nodes: @_foo = "bar" # name `@_foo` - name: name_loc - type: slice + type: location comment: | The location of the variable name. @@ -3229,7 +3229,7 @@ nodes: @_x = 1234 ^^^^ - name: operator_loc - type: slice + type: location comment: | The location of the `=` operator. @@ -3255,7 +3255,7 @@ nodes: flags: RegularExpressionFlags fields: - name: opening_loc - type: slice + type: location - name: parts type: node[] kind: @@ -3263,7 +3263,7 @@ nodes: - EmbeddedStatementsNode - EmbeddedVariableNode - name: closing_loc - type: slice + type: location newline: parts comment: | Represents a regular expression literal that contains interpolation that is being used in the predicate of a conditional to implicitly match against the last line read by an IO object. @@ -3274,7 +3274,7 @@ nodes: flags: RegularExpressionFlags fields: - name: opening_loc - type: slice + type: location - name: parts type: node[] kind: @@ -3282,7 +3282,7 @@ nodes: - EmbeddedStatementsNode - EmbeddedVariableNode - name: closing_loc - type: slice + type: location newline: parts comment: | Represents a regular expression literal that contains interpolation. @@ -3293,7 +3293,7 @@ nodes: flags: InterpolatedStringNodeFlags fields: - name: opening_loc - type: slice? + type: location? - name: parts type: node[] kind: @@ -3306,7 +3306,7 @@ nodes: - on error: SymbolNode - on error: InterpolatedSymbolNode - name: closing_loc - type: slice? + type: location? newline: parts comment: | Represents a string literal that contains interpolation. @@ -3316,7 +3316,7 @@ nodes: - name: InterpolatedSymbolNode fields: - name: opening_loc - type: slice? + type: location? - name: parts type: node[] kind: @@ -3324,7 +3324,7 @@ nodes: - EmbeddedStatementsNode - EmbeddedVariableNode - name: closing_loc - type: slice? + type: location? newline: parts comment: | Represents a symbol literal that contains interpolation. @@ -3334,7 +3334,7 @@ nodes: - name: InterpolatedXStringNode fields: - name: opening_loc - type: slice + type: location - name: parts type: node[] kind: @@ -3342,7 +3342,7 @@ nodes: - EmbeddedStatementsNode - EmbeddedVariableNode - name: closing_loc - type: slice + type: location newline: parts comment: | Represents an xstring literal that contains interpolation. @@ -3380,9 +3380,9 @@ nodes: - name: name type: constant? - name: name_loc - type: slice? + type: location? - name: operator_loc - type: slice + type: location comment: | Represents a keyword rest parameter to a method, block, or lambda definition. @@ -3394,11 +3394,11 @@ nodes: - name: locals type: constant[] - name: operator_loc - type: slice + type: location - name: opening_loc - type: slice + type: location - name: closing_loc - type: slice + type: location - name: parameters type: node? kind: @@ -3418,9 +3418,9 @@ nodes: - name: LocalVariableAndWriteNode fields: - name: name_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -3436,9 +3436,9 @@ nodes: - name: LocalVariableOperatorWriteNode fields: - name: name_loc - type: slice + type: location - name: binary_operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -3456,9 +3456,9 @@ nodes: - name: LocalVariableOrWriteNode fields: - name: name_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -3536,7 +3536,7 @@ nodes: The specific rules for calculating the depth may differ from individual Ruby implementations, as they are not specified by the language. For more information, see [the Prism documentation](https://github.com/ruby/prism/blob/main/docs/local_variable_depth.md). - name: name_loc - type: slice + type: location comment: | The location of the variable name. @@ -3558,7 +3558,7 @@ nodes: foo = foo - name: operator_loc - type: slice + type: location comment: | The location of the `=` operator. @@ -3573,11 +3573,11 @@ nodes: flags: RegularExpressionFlags fields: - name: opening_loc - type: slice + type: location - name: content_loc - type: slice + type: location - name: closing_loc - type: slice + type: location - name: unescaped type: string comment: | @@ -3594,7 +3594,7 @@ nodes: type: node kind: pattern expression - name: operator_loc - type: slice + type: location comment: | Represents the use of the modifier `in` operator. @@ -3658,7 +3658,7 @@ nodes: foo => CONST - name: operator_loc - type: slice + type: location comment: | The location of the operator. @@ -3690,7 +3690,7 @@ nodes: - name: locals type: constant[] - name: module_keyword_loc - type: slice + type: location - name: constant_path type: node kind: @@ -3703,7 +3703,7 @@ nodes: - StatementsNode - BeginNode - name: end_keyword_loc - type: slice + type: location - name: name type: constant comment: | @@ -3779,14 +3779,14 @@ nodes: a, (*, b, c) = 1, 2, 3, 4, 5 ^^^^ - name: lparen_loc - type: slice? + type: location? comment: | The location of the opening parenthesis. a, (b, c) = 1, 2, 3 ^ - name: rparen_loc - type: slice? + type: location? comment: | The location of the closing parenthesis. @@ -3868,21 +3868,21 @@ nodes: a, *, b, c = 1, 2, 3, 4, 5 ^^^^ - name: lparen_loc - type: slice? + type: location? comment: | The location of the opening parenthesis. (a, b, c) = 1, 2, 3 ^ - name: rparen_loc - type: slice? + type: location? comment: | The location of the closing parenthesis. (a, b, c) = 1, 2, 3 ^ - name: operator_loc - type: slice + type: location comment: | The location of the operator. @@ -3907,7 +3907,7 @@ nodes: type: node? kind: ArgumentsNode - name: keyword_loc - type: slice + type: location comment: | Represents the use of the `next` keyword. @@ -3922,9 +3922,9 @@ nodes: - name: NoKeywordsParameterNode fields: - name: operator_loc - type: slice + type: location - name: keyword_loc - type: slice + type: location comment: | Represents the use of `**nil` inside method arguments. @@ -3963,7 +3963,7 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -3979,9 +3979,9 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: value type: node kind: non-void expression @@ -4016,7 +4016,7 @@ nodes: 1 or 2 ^ - name: operator_loc - type: slice + type: location comment: | The location of the `or` keyword or the `||` operator. @@ -4079,9 +4079,9 @@ nodes: type: node? kind: non-void expression # Usually a StatementsNode but not always e.g. `1 in (..10)` - name: opening_loc - type: slice + type: location - name: closing_loc - type: slice + type: location newline: false comment: | Represents a parenthesized expression @@ -4099,21 +4099,21 @@ nodes: foo in ^(bar) ^^^ - name: operator_loc - type: slice + type: location comment: | The location of the `^` operator foo in ^(bar) ^ - name: lparen_loc - type: slice + type: location comment: | The location of the opening parenthesis. foo in ^(bar) ^ - name: rparen_loc - type: slice + type: location comment: | The location of the closing parenthesis. @@ -4143,7 +4143,7 @@ nodes: foo in ^bar ^^^ - name: operator_loc - type: slice + type: location comment: | The location of the `^` operator @@ -4160,11 +4160,11 @@ nodes: type: node? kind: StatementsNode - name: keyword_loc - type: slice + type: location - name: opening_loc - type: slice + type: location - name: closing_loc - type: slice + type: location comment: | Represents the use of the `END` keyword. @@ -4176,11 +4176,11 @@ nodes: type: node? kind: StatementsNode - name: keyword_loc - type: slice + type: location - name: opening_loc - type: slice + type: location - name: closing_loc - type: slice + type: location comment: | Represents the use of the `BEGIN` keyword. @@ -4221,7 +4221,7 @@ nodes: ^^^ If neither right-hand or left-hand side was included, this will be a MissingNode. - name: operator_loc - type: slice + type: location comment: | The location of the `..` or `...` operator. comment: | @@ -4262,11 +4262,11 @@ nodes: flags: RegularExpressionFlags fields: - name: opening_loc - type: slice + type: location - name: content_loc - type: slice + type: location - name: closing_loc - type: slice + type: location - name: unescaped type: string comment: | @@ -4280,7 +4280,7 @@ nodes: - name: name type: constant - name: name_loc - type: slice + type: location comment: | Represents a required keyword parameter to a method, block, or lambda definition. @@ -4304,7 +4304,7 @@ nodes: type: node kind: Node - name: keyword_loc - type: slice + type: location - name: rescue_expression type: node kind: Node @@ -4317,12 +4317,12 @@ nodes: - name: RescueNode fields: - name: keyword_loc - type: slice + type: location - name: exceptions type: node[] kind: non-void expression - name: operator_loc - type: slice? + type: location? - name: reference type: node? kind: @@ -4338,7 +4338,7 @@ nodes: - on error: NumberedReferenceReadNode # => begin; rescue => $1; end - on error: MissingNode # begin; rescue =>; end - name: then_keyword_loc - type: slice? + type: location? - name: statements type: node? kind: StatementsNode @@ -4361,9 +4361,9 @@ nodes: - name: name type: constant? - name: name_loc - type: slice? + type: location? - name: operator_loc - type: slice + type: location comment: | Represents a rest parameter to a method, block, or lambda definition. @@ -4379,7 +4379,7 @@ nodes: - name: ReturnNode fields: - name: keyword_loc - type: slice + type: location - name: arguments type: node? kind: ArgumentsNode @@ -4420,9 +4420,9 @@ nodes: - name: locals type: constant[] - name: class_keyword_loc - type: slice + type: location - name: operator_loc - type: slice + type: location - name: expression type: node kind: non-void expression @@ -4432,7 +4432,7 @@ nodes: - StatementsNode - BeginNode - name: end_keyword_loc - type: slice + type: location comment: | Represents a singleton class declaration involving the `class` keyword. @@ -4464,7 +4464,7 @@ nodes: - name: SplatNode fields: - name: operator_loc - type: slice + type: location - name: expression type: node? kind: non-void expression @@ -4487,11 +4487,11 @@ nodes: flags: StringFlags fields: - name: opening_loc - type: slice? + type: location? - name: content_loc - type: slice + type: location - name: closing_loc - type: slice? + type: location? - name: unescaped type: string comment: | @@ -4508,15 +4508,15 @@ nodes: - name: SuperNode fields: - name: keyword_loc - type: slice + type: location - name: lparen_loc - type: slice? + type: location? - name: arguments type: node? kind: ArgumentsNode comment: "Can be only `nil` when there are empty parentheses, like `super()`." - name: rparen_loc - type: slice? + type: location? - name: block type: node? kind: @@ -4536,11 +4536,11 @@ nodes: flags: SymbolFlags fields: - name: opening_loc - type: slice? + type: location? - name: value_loc - type: slice? + type: location? - name: closing_loc - type: slice? + type: location? - name: unescaped type: string comment: | @@ -4565,7 +4565,7 @@ nodes: - SymbolNode - InterpolatedSymbolNode - name: keyword_loc - type: slice + type: location comment: | Represents the use of the `undef` keyword. @@ -4574,7 +4574,7 @@ nodes: - name: UnlessNode fields: - name: keyword_loc - type: slice + type: location comment: | The location of the `unless` keyword. @@ -4595,7 +4595,7 @@ nodes: bar unless cond ^^^^ - name: then_keyword_loc - type: slice? + type: location? comment: | The location of the `then` keyword, if present. @@ -4619,7 +4619,7 @@ nodes: unless cond then bar else baz end ^^^^^^^^ - name: end_keyword_loc - type: slice? + type: location? comment: | The location of the `end` keyword, if present. @@ -4638,11 +4638,11 @@ nodes: flags: LoopFlags fields: - name: keyword_loc - type: slice + type: location - name: do_keyword_loc - type: slice? + type: location? - name: closing_loc - type: slice? + type: location? - name: predicate type: node kind: non-void expression @@ -4661,12 +4661,12 @@ nodes: - name: WhenNode fields: - name: keyword_loc - type: slice + type: location - name: conditions type: node[] kind: non-void expression - name: then_keyword_loc - type: slice? + type: location? - name: statements type: node? kind: StatementsNode @@ -4681,11 +4681,11 @@ nodes: flags: LoopFlags fields: - name: keyword_loc - type: slice + type: location - name: do_keyword_loc - type: slice? + type: location? - name: closing_loc - type: slice? + type: location? - name: predicate type: node kind: non-void expression @@ -4705,11 +4705,11 @@ nodes: flags: EncodingFlags fields: - name: opening_loc - type: slice + type: location - name: content_loc - type: slice + type: location - name: closing_loc - type: slice + type: location - name: unescaped type: string comment: | @@ -4720,14 +4720,14 @@ nodes: - name: YieldNode fields: - name: keyword_loc - type: slice + type: location - name: lparen_loc - type: slice? + type: location? - name: arguments type: node? kind: ArgumentsNode - name: rparen_loc - type: slice? + type: location? comment: | Represents the use of the `yield` keyword. diff --git a/rbi/prism/reflection.rbi b/rbi/prism/reflection.rbi index 0cbce6c9d3..b21e9b5def 100644 --- a/rbi/prism/reflection.rbi +++ b/rbi/prism/reflection.rbi @@ -35,15 +35,9 @@ end class Prism::Reflection::LocationField < Prism::Reflection::Field end -class Prism::Reflection::SliceField < Prism::Reflection::Field -end - class Prism::Reflection::OptionalLocationField < Prism::Reflection::Field end -class Prism::Reflection::OptionalSliceField < Prism::Reflection::Field -end - class Prism::Reflection::IntegerField < Prism::Reflection::Field end diff --git a/rust/ruby-prism-sys/tests/parser_tests.rs b/rust/ruby-prism-sys/tests/parser_tests.rs index cd5308dd54..6ce3a3b980 100644 --- a/rust/ruby-prism-sys/tests/parser_tests.rs +++ b/rust/ruby-prism-sys/tests/parser_tests.rs @@ -52,8 +52,8 @@ fn comments_test() { assert_eq!((*comment).type_, pm_comment_type_t::PM_COMMENT_INLINE); let location = { - let start = (*comment).location.start.offset_from(parser.start); - let end = (*comment).location.end.offset_from(parser.start); + let start = (*comment).location.start; + let end = (*comment).location.start + (*comment).location.length; start..end }; assert_eq!(location, 0..7); @@ -89,8 +89,8 @@ fn diagnostics_test() { ); let location = { - let start = (*error).location.start.offset_from(parser.start); - let end = (*error).location.end.offset_from(parser.start); + let start = (*error).location.start; + let end = (*error).location.start + (*error).location.length; start..end }; assert_eq!(location, 10..10); diff --git a/rust/ruby-prism/build.rs b/rust/ruby-prism/build.rs index 4f6c468fea..19a495f927 100644 --- a/rust/ruby-prism/build.rs +++ b/rust/ruby-prism/build.rs @@ -31,15 +31,9 @@ enum NodeFieldType { #[serde(rename = "location")] Location, - #[serde(rename = "slice")] - Slice, - #[serde(rename = "location?")] OptionalLocation, - #[serde(rename = "slice?")] - OptionalSlice, - #[serde(rename = "uint8")] UInt8, @@ -370,29 +364,12 @@ fn write_node(file: &mut File, flags: &[Flags], node: &Node) -> Result<(), Box { - writeln!(file, " pub fn {}(&self) -> Location<'pr> {{", field.name)?; - writeln!(file, " let pointer: *mut pm_location_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; - writeln!(file, " Location::new(self.parser, unsafe {{ &(*pointer) }})")?; - writeln!(file, " }}")?; - }, - NodeFieldType::Slice => { writeln!(file, " pub fn {}(&self) -> Slice<'pr> {{", field.name)?; writeln!(file, " let pointer: *mut pm_slice_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; writeln!(file, " Slice::new(self.parser, unsafe {{ &(*pointer) }})")?; writeln!(file, " }}")?; }, NodeFieldType::OptionalLocation => { - writeln!(file, " pub fn {}(&self) -> Option> {{", field.name)?; - writeln!(file, " let pointer: *mut pm_location_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; - writeln!(file, " let start = unsafe {{ (*pointer).start }};")?; - writeln!(file, " if start.is_null() {{")?; - writeln!(file, " None")?; - writeln!(file, " }} else {{")?; - writeln!(file, " Some(Location::new(self.parser, unsafe {{ &(*pointer) }}))")?; - writeln!(file, " }}")?; - writeln!(file, " }}")?; - }, - NodeFieldType::OptionalSlice => { writeln!(file, " pub fn {}(&self) -> Option> {{", field.name)?; writeln!(file, " let pointer: *mut pm_slice_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; writeln!(file, " let length = unsafe {{ (*pointer).length }};")?; diff --git a/rust/ruby-prism/src/lib.rs b/rust/ruby-prism/src/lib.rs index f5a6ed749c..c427c84067 100644 --- a/rust/ruby-prism/src/lib.rs +++ b/rust/ruby-prism/src/lib.rs @@ -397,8 +397,8 @@ impl<'pr> Diagnostic<'pr> { /// The location of the diagnostic in the source. #[must_use] - pub const fn location(&self) -> Location<'pr> { - Location::new(self.parser, unsafe { &self.diag.as_ref().location }) + pub const fn location(&self) -> Slice<'pr> { + Slice::new(self.parser, unsafe { &self.diag.as_ref().location }) } } @@ -442,14 +442,15 @@ impl<'pr> Comment<'pr> { /// The location of the comment in the source. #[must_use] - pub const fn location(&self) -> Location<'pr> { - Location::new(self.parser, unsafe { &self.content.as_ref().location }) + pub const fn location(&self) -> Slice<'pr> { + Slice::new(self.parser, unsafe { &self.content.as_ref().location }) } } /// A magic comment that was found during parsing. #[derive(Debug)] pub struct MagicComment<'pr> { + parser: NonNull, comment: NonNull, marker: PhantomData<&'pr pm_magic_comment_t>, } @@ -459,8 +460,8 @@ impl MagicComment<'_> { #[must_use] pub const fn key(&self) -> &[u8] { unsafe { - let start = self.comment.as_ref().key_start; - let len = self.comment.as_ref().key_length as usize; + let start = self.parser.as_ref().start.add(self.comment.as_ref().key.start as usize); + let len = self.comment.as_ref().key.length as usize; std::slice::from_raw_parts(start, len) } } @@ -469,8 +470,8 @@ impl MagicComment<'_> { #[must_use] pub const fn value(&self) -> &[u8] { unsafe { - let start = self.comment.as_ref().value_start; - let len = self.comment.as_ref().value_length as usize; + let start = self.parser.as_ref().start.add(self.comment.as_ref().value.start as usize); + let len = self.comment.as_ref().value.length as usize; std::slice::from_raw_parts(start, len) } } @@ -531,6 +532,7 @@ impl<'pr> Iterator for Comments<'pr> { /// A struct created by the `magic_comments` method on `ParseResult`. It can be used /// to iterate over the magic comments in the parse result. pub struct MagicComments<'pr> { + parser: NonNull, comment: *mut pm_magic_comment_t, marker: PhantomData<&'pr pm_magic_comment_t>, } @@ -540,7 +542,7 @@ impl<'pr> Iterator for MagicComments<'pr> { fn next(&mut self) -> Option { if let Some(comment) = NonNull::new(self.comment) { - let current = MagicComment { comment, marker: PhantomData }; + let current = MagicComment { parser: self.parser, comment, marker: PhantomData }; self.comment = unsafe { comment.as_ref().node.next.cast::() }; Some(current) } else { @@ -576,7 +578,7 @@ impl<'pr> ParseResult<'pr> { /// # Panics /// Panics if start offset or end offset are not valid offsets from the root. #[must_use] - pub fn as_slice(&self, location: &Location<'pr>) -> &'pr [u8] { + pub fn as_location_slice(&self, location: &Location<'pr>) -> &'pr [u8] { let root = self.source.as_ptr(); let start = usize::try_from(unsafe { location.start.offset_from(root) }).expect("start should point to memory after root"); @@ -585,6 +587,15 @@ impl<'pr> ParseResult<'pr> { &self.source[start..end] } + /// Returns a slice of the source string that was parsed using the given + /// slice range. + #[must_use] + pub fn as_slice(&self, slice: &Slice<'pr>) -> &'pr [u8] { + let start = slice.start as usize; + let end = start + slice.length as usize; + &self.source[start..end] + } + /// Returns an iterator that can be used to iterate over the errors in the /// parse result. #[must_use] @@ -634,6 +645,7 @@ impl<'pr> ParseResult<'pr> { unsafe { let list = &mut (*self.parser.as_ptr()).magic_comment_list; MagicComments { + parser: self.parser, comment: list.head.cast::(), marker: PhantomData, } @@ -642,12 +654,12 @@ impl<'pr> ParseResult<'pr> { /// Returns an optional location of the __END__ marker and the rest of the content of the file. #[must_use] - pub fn data_loc(&self) -> Option> { + pub fn data_loc(&self) -> Option> { let location = unsafe { &(*self.parser.as_ptr()).data_loc }; - if location.start.is_null() { + if location.length == 0 { None } else { - Some(Location::new(self.parser, location)) + Some(Slice::new(self.parser, location)) } } @@ -760,7 +772,7 @@ mod tests { let node = plus.arguments().unwrap().arguments().iter().next().unwrap(); let location = node.as_integer_node().unwrap().location(); - let slice = std::str::from_utf8(result.as_slice(&location)).unwrap(); + let slice = std::str::from_utf8(result.as_location_slice(&location)).unwrap(); assert_eq!(slice, "222"); assert_eq!(6, location.start_offset()); @@ -794,7 +806,7 @@ mod tests { } let location = node.location(); - let slice = std::str::from_utf8(result.as_slice(&location)).unwrap(); + let slice = std::str::from_utf8(result.as_location_slice(&location)).unwrap(); assert_eq!(slice, "222"); diff --git a/sig/prism/reflection.rbs b/sig/prism/reflection.rbs index 7cd9123154..047ea32a50 100644 --- a/sig/prism/reflection.rbs +++ b/sig/prism/reflection.rbs @@ -30,15 +30,9 @@ module Prism class LocationField < Field end - class SliceField < Field - end - class OptionalLocationField < Field end - class OptionalSliceField < Field - end - class IntegerField < Field end diff --git a/templates/ext/prism/api_node.c.erb b/templates/ext/prism/api_node.c.erb index f56d16fbed..31c611c416 100644 --- a/templates/ext/prism/api_node.c.erb +++ b/templates/ext/prism/api_node.c.erb @@ -252,15 +252,9 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi } if (freeze) rb_obj_freeze(argv[<%= index %>]); <%- when Prism::Template::LocationField -%> -#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" - argv[<%= index %>] = pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze); - <%- when Prism::Template::SliceField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" argv[<%= index %>] = pm_slice_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::OptionalLocationField -%> -#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" - argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : pm_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source, freeze); - <%- when Prism::Template::OptionalSliceField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" argv[<%= index %>] = cast-><%= field.name %>.length == 0 ? Qnil : pm_slice_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::UInt8Field -%> diff --git a/templates/include/prism/ast.h.erb b/templates/include/prism/ast.h.erb index 47faf7cfd8..85c941011c 100644 --- a/templates/include/prism/ast.h.erb +++ b/templates/include/prism/ast.h.erb @@ -216,8 +216,7 @@ typedef struct pm_<%= node.human %> { when Prism::Template::ConstantField, Prism::Template::OptionalConstantField then "pm_constant_id_t #{field.name}" when Prism::Template::ConstantListField then "pm_constant_id_list_t #{field.name}" when Prism::Template::StringField then "pm_string_t #{field.name}" - when Prism::Template::LocationField, Prism::Template::OptionalLocationField then "pm_location_t #{field.name}" - when Prism::Template::SliceField, Prism::Template::OptionalSliceField then "pm_slice_t #{field.name}" + when Prism::Template::LocationField, Prism::Template::OptionalLocationField then "pm_slice_t #{field.name}" when Prism::Template::UInt8Field then "uint8_t #{field.name}" when Prism::Template::UInt32Field then "uint32_t #{field.name}" when Prism::Template::IntegerField then "pm_integer_t #{field.name}" diff --git a/templates/java/org/prism/Loader.java.erb b/templates/java/org/prism/Loader.java.erb index 63ff51a557..6c52d36feb 100644 --- a/templates/java/org/prism/Loader.java.erb +++ b/templates/java/org/prism/Loader.java.erb @@ -386,8 +386,8 @@ public class Loader { when Prism::Template::ConstantField then "loadConstant()" when Prism::Template::OptionalConstantField then "loadOptionalConstant()" when Prism::Template::ConstantListField then "loadConstants()" - when Prism::Template::LocationField, Prism::Template::SliceField then "loadLocation()" - when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "loadOptionalLocation()" + when Prism::Template::LocationField then "loadLocation()" + when Prism::Template::OptionalLocationField then "loadOptionalLocation()" when Prism::Template::UInt8Field then "buffer.get()" when Prism::Template::UInt32Field then "loadVarUInt()" when Prism::Template::IntegerField then "loadInteger()" diff --git a/templates/java/org/prism/Nodes.java.erb b/templates/java/org/prism/Nodes.java.erb index edf264318d..8b0a9ce20d 100644 --- a/templates/java/org/prism/Nodes.java.erb +++ b/templates/java/org/prism/Nodes.java.erb @@ -342,7 +342,7 @@ public abstract class Nodes { <%- if node.fields.any?(Prism::Template::NodeListField) or node.fields.any?(Prism::Template::ConstantListField) -%> String nextNextIndent = nextIndent + " "; <%- end -%> - <%- [*node.flags, *node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::SliceField).grep_v(Prism::Template::OptionalLocationField).grep_v(Prism::Template::OptionalSliceField)].each do |field| -%> + <%- [*node.flags, *node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::OptionalLocationField)].each do |field| -%> builder.append(nextIndent); builder.append("<%= field.name %>: "); <%- case field -%> diff --git a/templates/javascript/src/deserialize.js.erb b/templates/javascript/src/deserialize.js.erb index 054312a0a7..a058d6cfeb 100644 --- a/templates/javascript/src/deserialize.js.erb +++ b/templates/javascript/src/deserialize.js.erb @@ -379,8 +379,8 @@ export function deserialize(source, array) { when Prism::Template::ConstantField then "readRequiredConstant()" when Prism::Template::OptionalConstantField then "readOptionalConstant()" when Prism::Template::ConstantListField then "Array.from({ length: buffer.readVarInt() }, readRequiredConstant)" - when Prism::Template::LocationField, Prism::Template::SliceField then "buffer.readLocation()" - when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "buffer.readOptionalLocation()" + when Prism::Template::LocationField then "buffer.readLocation()" + when Prism::Template::OptionalLocationField then "buffer.readOptionalLocation()" when Prism::Template::UInt8Field then "buffer.readByte()" when Prism::Template::UInt32Field then "buffer.readVarInt()" when Prism::Template::IntegerField then "readInteger()" diff --git a/templates/javascript/src/nodes.js.erb b/templates/javascript/src/nodes.js.erb index 092d78bdc5..d955d61303 100644 --- a/templates/javascript/src/nodes.js.erb +++ b/templates/javascript/src/nodes.js.erb @@ -17,8 +17,8 @@ def jstype(field) when Prism::Template::ConstantField then "string" when Prism::Template::OptionalConstantField then "string | null" when Prism::Template::ConstantListField then "string[]" - when Prism::Template::LocationField, Prism::Template::SliceField then "Location" - when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "Location | null" + when Prism::Template::LocationField then "Location" + when Prism::Template::OptionalLocationField then "Location | null" when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField, Prism::Template::DoubleField then "number" else raise end diff --git a/templates/lib/prism/dot_visitor.rb.erb b/templates/lib/prism/dot_visitor.rb.erb index 8a1b6f5b71..cd2998fe61 100644 --- a/templates/lib/prism/dot_visitor.rb.erb +++ b/templates/lib/prism/dot_visitor.rb.erb @@ -141,9 +141,9 @@ module Prism end <%- when Prism::Template::StringField, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantListField, Prism::Template::IntegerField, Prism::Template::DoubleField -%> table.field("<%= field.name %>", node.<%= field.name %>.inspect) - <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> + <%- when Prism::Template::LocationField -%> table.field("<%= field.name %>", location_inspect(node.<%= field.name %>)) - <%- when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> + <%- when Prism::Template::OptionalLocationField -%> unless (<%= field.name %> = node.<%= field.name %>).nil? table.field("<%= field.name %>", location_inspect(<%= field.name %>)) end diff --git a/templates/lib/prism/dsl.rb.erb b/templates/lib/prism/dsl.rb.erb index cd9b401989..e16ebb7110 100644 --- a/templates/lib/prism/dsl.rb.erb +++ b/templates/lib/prism/dsl.rb.erb @@ -78,13 +78,13 @@ module Prism end when Prism::Template::ConstantField "#{field.name}: :\"\"" - when Prism::Template::OptionalNodeField, Prism::Template::OptionalConstantField, Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField + when Prism::Template::OptionalNodeField, Prism::Template::OptionalConstantField, Prism::Template::OptionalLocationField "#{field.name}: nil" when Prism::Template::NodeListField, Prism::Template::ConstantListField "#{field.name}: []" when Prism::Template::StringField "#{field.name}: \"\"" - when Prism::Template::LocationField, Prism::Template::SliceField + when Prism::Template::LocationField "#{field.name}: location" when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField "#{field.name}: 0" diff --git a/templates/lib/prism/inspect_visitor.rb.erb b/templates/lib/prism/inspect_visitor.rb.erb index 226c28b148..3cfe615d85 100644 --- a/templates/lib/prism/inspect_visitor.rb.erb +++ b/templates/lib/prism/inspect_visitor.rb.erb @@ -104,7 +104,7 @@ module Prism else commands << ["<%= pointer %><%= field.name %>: #{<%= field.name %>.inspect}\n", indent] end - <%- when Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> + <%- when Prism::Template::LocationField, Prism::Template::OptionalLocationField -%> commands << ["<%= pointer %><%= field.name %>: #{inspect_location(node.<%= field.name %>)}\n", indent] <%- end -%> <%- end -%> diff --git a/templates/lib/prism/node.rb.erb b/templates/lib/prism/node.rb.erb index 1b14335b57..ceee2b0ffe 100644 --- a/templates/lib/prism/node.rb.erb +++ b/templates/lib/prism/node.rb.erb @@ -364,8 +364,8 @@ module Prism def comment_targets [<%= node.fields.map { |field| case field - when Prism::Template::NodeField, Prism::Template::LocationField, Prism::Template::SliceField then field.name - when Prism::Template::OptionalNodeField, Prism::Template::NodeListField, Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "*#{field.name}" + when Prism::Template::NodeField, Prism::Template::LocationField then field.name + when Prism::Template::OptionalNodeField, Prism::Template::NodeListField, Prism::Template::OptionalLocationField then "*#{field.name}" end }.compact.join(", ") %>] #: Array[Prism::node | Location] end @@ -401,7 +401,7 @@ module Prism <%- end -%> <%- end -%> <%- case field -%> - <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> + <%- when Prism::Template::LocationField -%> def <%= field.name %> location = @<%= field.name %> return location if location.is_a?(Location) @@ -413,7 +413,7 @@ module Prism def save_<%= field.name %>(repository) repository.enter(node_id, :<%= field.name %>) end - <%- when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> + <%- when Prism::Template::OptionalLocationField -%> def <%= field.name %> location = @<%= field.name %> case location @@ -437,7 +437,7 @@ module Prism <%- end -%> <%- node.fields.each do |field| -%> <%- case field -%> - <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> + <%- when Prism::Template::LocationField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> @@ -445,7 +445,7 @@ module Prism def <%= field.name.delete_suffix("_loc") %> <%= field.name %>.slice end - <%- when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> + <%- when Prism::Template::OptionalLocationField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> @@ -476,7 +476,7 @@ module Prism def ===(other) other.is_a?(<%= node.name %>)<%= " &&" if (fields = [*node.flags, *node.fields]).any? %> <%- fields.each_with_index do |field, index| -%> - <%- if field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::SliceField) || field.is_a?(Prism::Template::OptionalLocationField) || field.is_a?(Prism::Template::OptionalSliceField) -%> + <%- if field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::OptionalLocationField) -%> (<%= field.name %>.nil? == other.<%= field.name %>.nil?)<%= " &&" if index != fields.length - 1 %> <%- elsif field.is_a?(Prism::Template::NodeListField) || field.is_a?(Prism::Template::ConstantListField) -%> (<%= field.name %>.length == other.<%= field.name %>.length) && diff --git a/templates/lib/prism/reflection.rb.erb b/templates/lib/prism/reflection.rb.erb index 89c0bc6ce2..6c8b2f4d25 100644 --- a/templates/lib/prism/reflection.rb.erb +++ b/templates/lib/prism/reflection.rb.erb @@ -59,23 +59,12 @@ module Prism class LocationField < Field end - # A slice field represents a slice of the source code. It resolves to a - # Prism::Location in Ruby. - class SliceField < Field - end - # An optional location field represents the location of some part of the # node in the source code that may or may not be present. It resolves to # either a Prism::Location or nil in Ruby. class OptionalLocationField < Field end - # An optional slice field represents a slice of the source code that may or - # may not be present. It resolves to either a Prism::Location or nil in - # Ruby. - class OptionalSliceField < Field - end - # An integer field represents an integer value. It is used to represent the # value of an integer literal, the depth of local variables, and the number # of a numbered reference. It resolves to an Integer in Ruby. @@ -126,12 +115,8 @@ module Prism "StringField.new(:#{field.name})" when Prism::Template::LocationField "LocationField.new(:#{field.name})" - when Prism::Template::SliceField - "SliceField.new(:#{field.name})" when Prism::Template::OptionalLocationField "OptionalLocationField.new(:#{field.name})" - when Prism::Template::OptionalSliceField - "OptionalSliceField.new(:#{field.name})" when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField "IntegerField.new(:#{field.name})" when Prism::Template::DoubleField diff --git a/templates/lib/prism/serialize.rb.erb b/templates/lib/prism/serialize.rb.erb index b9003d276c..526be67431 100644 --- a/templates/lib/prism/serialize.rb.erb +++ b/templates/lib/prism/serialize.rb.erb @@ -529,8 +529,8 @@ module Prism when Prism::Template::ConstantField then "load_constant(constant_pool, encoding)" when Prism::Template::OptionalConstantField then "load_optional_constant(constant_pool, encoding)" when Prism::Template::ConstantListField then "Array.new(load_varuint) { load_constant(constant_pool, encoding) }.tap { |constants| constants.freeze if freeze }" - when Prism::Template::LocationField, Prism::Template::SliceField then "load_location(freeze)" - when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "load_optional_location(freeze)" + when Prism::Template::LocationField then "load_location(freeze)" + when Prism::Template::OptionalLocationField then "load_optional_location(freeze)" when Prism::Template::UInt8Field then "io.getbyte" when Prism::Template::UInt32Field then "load_varuint" when Prism::Template::IntegerField then "load_integer" @@ -568,8 +568,8 @@ module Prism when Prism::Template::ConstantField then "load_constant(constant_pool, encoding)" when Prism::Template::OptionalConstantField then "load_optional_constant(constant_pool, encoding)" when Prism::Template::ConstantListField then "Array.new(load_varuint) { load_constant(constant_pool, encoding) }" - when Prism::Template::LocationField, Prism::Template::SliceField then "load_location(freeze)" - when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField then "load_optional_location(freeze)" + when Prism::Template::LocationField then "load_location(freeze)" + when Prism::Template::OptionalLocationField then "load_optional_location(freeze)" when Prism::Template::UInt8Field then "io.getbyte" when Prism::Template::UInt32Field then "load_varuint" when Prism::Template::IntegerField then "load_integer" diff --git a/templates/rbi/prism/dsl.rbi.erb b/templates/rbi/prism/dsl.rbi.erb index 7fcae1fada..76a39c5051 100644 --- a/templates/rbi/prism/dsl.rbi.erb +++ b/templates/rbi/prism/dsl.rbi.erb @@ -32,9 +32,9 @@ module Prism::DSL [field.name, "[]", field.rbi_class] when Prism::Template::StringField [field.name, "\"\"", field.rbi_class] - when Prism::Template::LocationField, Prism::Template::SliceField + when Prism::Template::LocationField [field.name, "location", field.rbi_class] - when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField + when Prism::Template::OptionalLocationField [field.name, "nil", field.rbi_class] when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField [field.name, "0", field.rbi_class] diff --git a/templates/rbi/prism/node.rbi.erb b/templates/rbi/prism/node.rbi.erb index 3cc7436900..c0474bd208 100644 --- a/templates/rbi/prism/node.rbi.erb +++ b/templates/rbi/prism/node.rbi.erb @@ -116,13 +116,13 @@ class Prism::<%= node.name -%> < Prism::Node def deconstruct_keys(keys); end <%- node.fields.each do |field| -%> <%- case field -%> - <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> + <%- when Prism::Template::LocationField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> sig { returns(String) } def <%= field.name.delete_suffix("_loc") %>; end - <%- when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> + <%- when Prism::Template::OptionalLocationField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> diff --git a/templates/sig/prism/node.rbs.erb b/templates/sig/prism/node.rbs.erb index 693428bd0d..6d6fe67336 100644 --- a/templates/sig/prism/node.rbs.erb +++ b/templates/sig/prism/node.rbs.erb @@ -86,7 +86,7 @@ module Prism <%- node.fields.each do |field| -%> attr_reader <%= field.name %>: <%= field.rbs_class %> <%- end -%> - <%- if (locations = node.fields.select { |field| field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::SliceField) || field.is_a?(Prism::Template::OptionalLocationField) || field.is_a?(Prism::Template::OptionalSliceField) }) -%> + <%- if (locations = node.fields.select { |field| field.is_a?(Prism::Template::LocationField) || field.is_a?(Prism::Template::OptionalLocationField) }) -%> <%- locations.each do |field| -%> def save_<%= field.name %>: (_Repository repository) -> void @@ -98,11 +98,11 @@ module Prism def deconstruct_keys: (Array[Symbol] keys) -> { <%= (["node_id: Integer", "location: Location"] + node.fields.map { |field| "#{field.name}: #{field.rbs_class}" }).join(", ") %> } <%- node.fields.each do |field| -%> <%- case field -%> - <%- when Prism::Template::LocationField, Prism::Template::SliceField -%> + <%- when Prism::Template::LocationField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> def <%= field.name.delete_suffix("_loc") %>: () -> String - <%- when Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField -%> + <%- when Prism::Template::OptionalLocationField -%> <%- raise unless field.name.end_with?("_loc") -%> <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> def <%= field.name.delete_suffix("_loc") %>: () -> String? diff --git a/templates/src/node.c.erb b/templates/src/node.c.erb index fba7a11d7c..a9cffb99a6 100644 --- a/templates/src/node.c.erb +++ b/templates/src/node.c.erb @@ -108,12 +108,12 @@ pm_node_destroy(pm_parser_t *parser, pm_node_t *node) { <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "prism/templates/src/<%= File.basename(__FILE__) %>" case <%= node.type %>: { - <%- if node.fields.any? { |field| ![Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField].include?(field.class) } -%> + <%- if node.fields.any? { |field| ![Prism::Template::LocationField, Prism::Template::OptionalLocationField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField].include?(field.class) } -%> pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node; <%- end -%> <%- node.fields.each do |field| -%> <%- case field -%> - <%- when Prism::Template::LocationField, Prism::Template::SliceField, Prism::Template::OptionalLocationField, Prism::Template::OptionalSliceField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField -%> + <%- when Prism::Template::LocationField, Prism::Template::OptionalLocationField, Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::ConstantField, Prism::Template::OptionalConstantField, Prism::Template::DoubleField -%> <%- when Prism::Template::NodeField -%> pm_node_destroy(parser, (pm_node_t *)cast-><%= field.name %>); <%- when Prism::Template::OptionalNodeField -%> @@ -295,16 +295,8 @@ pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *no } pm_buffer_append_byte(buffer, ']'); <%- when Prism::Template::LocationField -%> - pm_dump_json_location(buffer, parser, &cast-><%= field.name %>); - <%- when Prism::Template::SliceField -%> pm_dump_json_slice(buffer, &cast-><%= field.name %>); <%- when Prism::Template::OptionalLocationField -%> - if (cast-><%= field.name %>.start != NULL) { - pm_dump_json_location(buffer, parser, &cast-><%= field.name %>); - } else { - pm_buffer_append_string(buffer, "null", 4); - } - <%- when Prism::Template::OptionalSliceField -%> if (cast-><%= field.name %>.length != 0) { pm_dump_json_slice(buffer, &cast-><%= field.name %>); } else { diff --git a/templates/src/prettyprint.c.erb b/templates/src/prettyprint.c.erb index 2d4e11f9f9..ac4e2b5465 100644 --- a/templates/src/prettyprint.c.erb +++ b/templates/src/prettyprint.c.erb @@ -109,13 +109,6 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm } pm_buffer_append_string(output_buffer, "]\n", 2); <%- when Prism::Template::LocationField -%> - pm_location_t *location = &cast-><%= field.name %>; - pm_buffer_append_byte(output_buffer, ' '); - prettyprint_location(output_buffer, parser, location); - pm_buffer_append_string(output_buffer, " = \"", 4); - pm_buffer_append_source(output_buffer, location->start, (size_t) (location->end - location->start), PM_BUFFER_ESCAPING_RUBY); - pm_buffer_append_string(output_buffer, "\"\n", 2); - <%- when Prism::Template::SliceField -%> pm_slice_t *slice = &cast-><%= field.name %>; pm_buffer_append_byte(output_buffer, ' '); prettyprint_slice(output_buffer, parser, slice); @@ -123,17 +116,6 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm pm_buffer_append_source(output_buffer, parser->start + slice->start, (size_t) slice->length, PM_BUFFER_ESCAPING_RUBY); pm_buffer_append_string(output_buffer, "\"\n", 2); <%- when Prism::Template::OptionalLocationField -%> - pm_location_t *location = &cast-><%= field.name %>; - if (location->start == NULL) { - pm_buffer_append_string(output_buffer, " nil\n", 5); - } else { - pm_buffer_append_byte(output_buffer, ' '); - prettyprint_location(output_buffer, parser, location); - pm_buffer_append_string(output_buffer, " = \"", 4); - pm_buffer_append_source(output_buffer, location->start, (size_t) (location->end - location->start), PM_BUFFER_ESCAPING_RUBY); - pm_buffer_append_string(output_buffer, "\"\n", 2); - } - <%- when Prism::Template::OptionalSliceField -%> pm_slice_t *slice = &cast-><%= field.name %>; if (slice->length == 0) { pm_buffer_append_string(output_buffer, " nil\n", 5); diff --git a/templates/src/serialize.c.erb b/templates/src/serialize.c.erb index 142f15070e..4cae0c5c2d 100644 --- a/templates/src/serialize.c.erb +++ b/templates/src/serialize.c.erb @@ -129,23 +129,10 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { } <%- when Prism::Template::LocationField -%> <%- if field.should_be_serialized? -%> - pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); - <%- end -%> - <%- when Prism::Template::SliceField -%> - <%- if field.should_be_serialized? -%> pm_serialize_slice(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); <%- end -%> <%- when Prism::Template::OptionalLocationField -%> <%- if field.should_be_serialized? -%> - if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) { - pm_buffer_append_byte(buffer, 0); - } else { - pm_buffer_append_byte(buffer, 1); - pm_serialize_location(parser, &((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); - } - <%- end -%> - <%- when Prism::Template::OptionalSliceField -%> - <%- if field.should_be_serialized? -%> if (((pm_<%= node.human %>_t *)node)-><%= field.name %>.length == 0) { pm_buffer_append_byte(buffer, 0); } else { diff --git a/templates/template.rb b/templates/template.rb index 1ea2f3d3d5..6c3efd7e6c 100755 --- a/templates/template.rb +++ b/templates/template.rb @@ -317,27 +317,6 @@ def java_type end end - # This represents the same information as a location field, except it is - # represented in the source using two 32-bit integers, as opposed to two - # pointers. - class SliceField < Field - def semantic_field? - false - end - - def rbs_class - "Location" - end - - def rbi_class - "Prism::Location" - end - - def java_type - "Location" - end - end - # This represents a field on a node that is a location that is optional. class OptionalLocationField < Field def semantic_field? @@ -357,25 +336,6 @@ def java_type end end - # This represents a field on a node that is a slice that is optional. - class OptionalSliceField < Field - def semantic_field? - false - end - - def rbs_class - "Location?" - end - - def rbi_class - "T.nilable(Prism::Location)" - end - - def java_type - "Location" - end - end - # This represents an integer field. class UInt8Field < Field def rbs_class @@ -533,8 +493,6 @@ def field_type_for(name) when "uint32" then UInt32Field when "integer" then IntegerField when "double" then DoubleField - when "slice" then SliceField - when "slice?" then OptionalSliceField else raise("Unknown field type: #{name.inspect}") end end From 1cbdca20670ba1638f1920ea1a5e06aa0a96a685 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 1 Dec 2025 22:34:21 -0500 Subject: [PATCH 13/22] Fix up gcc warnings --- src/prism.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/prism.c b/src/prism.c index a7bc0bfa2c..1fa4eabfdb 100644 --- a/src/prism.c +++ b/src/prism.c @@ -2828,7 +2828,7 @@ pm_call_node_writable_p(const pm_parser_t *parser, const pm_call_node_t *node) { (node->message_loc.length > 0) && (parser->start[node->message_loc.start + node->message_loc.length - 1] != '!') && (parser->start[node->message_loc.start + node->message_loc.length - 1] != '?') && - char_is_identifier_start(parser, parser->start + node->message_loc.start, node->message_loc.length) && + char_is_identifier_start(parser, parser->start + node->message_loc.start, (ptrdiff_t) node->message_loc.length) && (node->opening_loc.length == 0) && (node->arguments == NULL) && (node->block == NULL) @@ -12750,7 +12750,7 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p return UP(pm_local_variable_target_node_create(parser, &message_loc, name, 0)); } - if (parser->start[call->message_loc.start] == '_' || parser->encoding->alnum_char(parser->start + call->message_loc.start, call->message_loc.length)) { + if (parser->start[call->message_loc.start] == '_' || parser->encoding->alnum_char(parser->start + call->message_loc.start, (ptrdiff_t) call->message_loc.length)) { if (multiple && PM_NODE_FLAG_P(call, PM_CALL_NODE_FLAGS_SAFE_NAVIGATION)) { pm_parser_err_node(parser, (const pm_node_t *) call, PM_ERR_UNEXPECTED_SAFE_NAVIGATION); } @@ -12942,7 +12942,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod return target; } - if (char_is_identifier_start(parser, parser->start + call->message_loc.start, call->message_loc.length)) { + if (char_is_identifier_start(parser, parser->start + call->message_loc.start, (ptrdiff_t) call->message_loc.length)) { // When we get here, we have a method call, because it was // previously marked as a method call but now we have an =. This // looks like: From 8b3f295229946e6c7feac2481111990d692ee5c1 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 2 Dec 2025 13:01:42 -0500 Subject: [PATCH 14/22] Switch newline list to use 32-bit offsets --- include/prism/static_literals.h | 6 ++- include/prism/util/pm_char.h | 4 +- include/prism/util/pm_newline_list.h | 23 +++++------ src/prism.c | 60 ++++++++++++++-------------- src/static_literals.c | 20 +++++++--- src/util/pm_char.c | 10 ++--- src/util/pm_newline_list.c | 42 +++++++------------ templates/src/prettyprint.c.erb | 8 ++-- 8 files changed, 85 insertions(+), 88 deletions(-) diff --git a/include/prism/static_literals.h b/include/prism/static_literals.h index bd29761899..0f8eb43bfa 100644 --- a/include/prism/static_literals.h +++ b/include/prism/static_literals.h @@ -92,13 +92,14 @@ typedef struct { * Add a node to the set of static literals. * * @param newline_list The list of newline offsets to use to calculate lines. + * @param start The start of the source being parsed. * @param start_line The line number that the parser starts on. * @param literals The set of static literals to add the node to. * @param node The node to add to the set. * @param replace Whether to replace the previous node if one already exists. * @return A pointer to the node that is being overwritten, if there is one. */ -pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace); +pm_node_t * pm_static_literals_add(const pm_newline_list_t *newline_list, const uint8_t *start, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace); /** * Free the internal memory associated with the given static literals set. @@ -112,10 +113,11 @@ void pm_static_literals_free(pm_static_literals_t *literals); * * @param buffer The buffer to write the string to. * @param newline_list The list of newline offsets to use to calculate lines. + * @param start The start of the source being parsed. * @param start_line The line number that the parser starts on. * @param encoding_name The name of the encoding of the source being parsed. * @param node The node to create a string representation of. */ -void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node); +void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, const uint8_t *start, int32_t start_line, const char *encoding_name, const pm_node_t *node); #endif diff --git a/include/prism/util/pm_char.h b/include/prism/util/pm_char.h index deeafd6321..b213e8edee 100644 --- a/include/prism/util/pm_char.h +++ b/include/prism/util/pm_char.h @@ -31,10 +31,12 @@ size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length); * @param string The string to search. * @param length The maximum number of characters to search. * @param newline_list The list of newlines to populate. + * @param start_offset The offset at which the string occurs in the source, for + * the purpose of tracking newlines. * @return The number of characters at the start of the string that are * whitespace. */ -size_t pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list); +size_t pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list, uint32_t start_offset); /** * Returns the number of characters at the start of the string that are inline diff --git a/include/prism/util/pm_newline_list.h b/include/prism/util/pm_newline_list.h index 406abe8ba5..b27c031de8 100644 --- a/include/prism/util/pm_newline_list.h +++ b/include/prism/util/pm_newline_list.h @@ -26,9 +26,6 @@ * sorted/inserted in ascending order. */ typedef struct { - /** A pointer to the start of the source string. */ - const uint8_t *start; - /** The number of offsets in the list. */ size_t size; @@ -36,7 +33,7 @@ typedef struct { size_t capacity; /** The list of offsets. */ - size_t *offsets; + uint32_t *offsets; } pm_newline_list_t; /** @@ -55,41 +52,39 @@ typedef struct { * allocation of the offsets succeeds, otherwise returns false. * * @param list The list to initialize. - * @param start A pointer to the start of the source string. * @param capacity The initial capacity of the list. * @return True if the allocation of the offsets succeeds, otherwise false. */ -bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity); +bool pm_newline_list_init(pm_newline_list_t *list, size_t capacity); /** * Clear out the newlines that have been appended to the list. * * @param list The list to clear. */ -void -pm_newline_list_clear(pm_newline_list_t *list); +void pm_newline_list_clear(pm_newline_list_t *list); /** * Append a new offset to the newline list. Returns true if the reallocation of * the offsets succeeds (if one was necessary), otherwise returns false. * * @param list The list to append to. - * @param cursor A pointer to the offset to append. + * @param cursor The offset to append. * @return True if the reallocation of the offsets succeeds (if one was * necessary), otherwise false. */ -bool pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor); +bool pm_newline_list_append(pm_newline_list_t *list, uint32_t cursor); /** * Returns the line of the given offset. If the offset is not in the list, the * line of the closest offset less than the given offset is returned. * * @param list The list to search. - * @param cursor A pointer to the offset to search for. + * @param cursor The offset to search for. * @param start_line The line to start counting from. * @return The line of the given offset. */ -int32_t pm_newline_list_line(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line); +int32_t pm_newline_list_line(const pm_newline_list_t *list, uint32_t cursor, int32_t start_line); /** * Returns the line and column of the given offset. If the offset is not in the @@ -97,11 +92,11 @@ int32_t pm_newline_list_line(const pm_newline_list_t *list, const uint8_t *curso * are returned. * * @param list The list to search. - * @param cursor A pointer to the offset to search for. + * @param cursor The offset to search for. * @param start_line The line to start counting from. * @return The line and column of the given offset. */ -pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line); +pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, uint32_t cursor, int32_t start_line); /** * Free the internal memory allocated for the newline list. diff --git a/src/prism.c b/src/prism.c index 1fa4eabfdb..16db1b4bf6 100644 --- a/src/prism.c +++ b/src/prism.c @@ -987,7 +987,7 @@ pm_locals_order(PRISM_ATTRIBUTE_UNUSED pm_parser_t *parser, pm_locals_t *locals, if (local->name != PM_CONSTANT_ID_UNSET) { pm_constant_id_list_insert(list, (size_t) local->index, local->name); - if (warn_unused && local->reads == 0 && ((parser->start_line >= 0) || (pm_newline_list_line(&parser->newline_list, parser->start + local->location.start, parser->start_line) >= 0))) { + if (warn_unused && local->reads == 0 && ((parser->start_line >= 0) || (pm_newline_list_line(&parser->newline_list, local->location.start, parser->start_line) >= 0))) { pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, local->name); if (constant->length >= 1 && *constant->start != '_') { @@ -9307,7 +9307,7 @@ lex_embdoc(pm_parser_t *parser) { if (newline == NULL) { parser->current.end = parser->end; } else { - pm_newline_list_append(&parser->newline_list, newline); + pm_newline_list_append(&parser->newline_list, (uint32_t) (newline - parser->start + 1)); parser->current.end = newline + 1; } @@ -9341,7 +9341,7 @@ lex_embdoc(pm_parser_t *parser) { if (newline == NULL) { parser->current.end = parser->end; } else { - pm_newline_list_append(&parser->newline_list, newline); + pm_newline_list_append(&parser->newline_list, (uint32_t) (newline - parser->start + 1)); parser->current.end = newline + 1; } @@ -9361,7 +9361,7 @@ lex_embdoc(pm_parser_t *parser) { if (newline == NULL) { parser->current.end = parser->end; } else { - pm_newline_list_append(&parser->newline_list, newline); + pm_newline_list_append(&parser->newline_list, (uint32_t) (newline - parser->start + 1)); parser->current.end = newline + 1; } @@ -9675,7 +9675,7 @@ pm_lex_percent_delimiter(pm_parser_t *parser) { parser_flush_heredoc_end(parser); } else { // Otherwise, we'll add the newline to the list of newlines. - pm_newline_list_append(&parser->newline_list, parser->current.end + eol_length - 1); + pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end + eol_length - parser->start)); } uint8_t delimiter = *parser->current.end; @@ -9773,7 +9773,7 @@ parser_lex(pm_parser_t *parser) { parser->heredoc_end = NULL; } else { parser->current.end += eol_length + 1; - pm_newline_list_append(&parser->newline_list, parser->current.end - 1); + pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start)); space_seen = true; } } else if (pm_char_is_inline_whitespace(*parser->current.end)) { @@ -9867,7 +9867,7 @@ parser_lex(pm_parser_t *parser) { } if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, parser->current.end - 1); + pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start)); } } @@ -10376,7 +10376,7 @@ parser_lex(pm_parser_t *parser) { } else { // Otherwise, we want to indicate that the body of the // heredoc starts on the character after the next newline. - pm_newline_list_append(&parser->newline_list, body_start); + pm_newline_list_append(&parser->newline_list, (uint32_t) (body_start - parser->start + 1)); body_start++; } @@ -11017,7 +11017,7 @@ parser_lex(pm_parser_t *parser) { // correct column information for it. const uint8_t *cursor = parser->current.end; while ((cursor = next_newline(cursor, parser->end - cursor)) != NULL) { - pm_newline_list_append(&parser->newline_list, cursor++); + pm_newline_list_append(&parser->newline_list, (uint32_t) (++cursor - parser->start)); } parser->current.end = parser->end; @@ -11078,7 +11078,7 @@ parser_lex(pm_parser_t *parser) { whitespace += 1; } } else { - whitespace = pm_strspn_whitespace_newlines(parser->current.end, parser->end - parser->current.end, &parser->newline_list); + whitespace = pm_strspn_whitespace_newlines(parser->current.end, parser->end - parser->current.end, &parser->newline_list, (uint32_t) (parser->current.end - parser->start)); } if (whitespace > 0) { @@ -11193,7 +11193,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } else { // ... else track the newline. - pm_newline_list_append(&parser->newline_list, parser->current.end); + pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start + 1)); } parser->current.end++; @@ -11331,7 +11331,7 @@ parser_lex(pm_parser_t *parser) { // would have already have added the newline to the // list. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, parser->current.end - 1); + pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start)); } } else { parser->current.end = breakpoint + 1; @@ -11378,7 +11378,7 @@ parser_lex(pm_parser_t *parser) { // If we've hit a newline, then we need to track that in // the list of newlines. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, breakpoint); + pm_newline_list_append(&parser->newline_list, (uint32_t) (breakpoint - parser->start + 1)); parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); break; @@ -11426,7 +11426,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } else { // ... else track the newline. - pm_newline_list_append(&parser->newline_list, parser->current.end); + pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start + 1)); } parser->current.end++; @@ -11591,7 +11591,7 @@ parser_lex(pm_parser_t *parser) { // would have already have added the newline to the // list. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, parser->current.end - 1); + pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start)); } } else { parser->current.end = breakpoint + 1; @@ -11636,7 +11636,7 @@ parser_lex(pm_parser_t *parser) { // for the terminator in case the terminator is a // newline character. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, breakpoint); + pm_newline_list_append(&parser->newline_list, (uint32_t) (breakpoint - parser->start + 1)); parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); break; @@ -11690,7 +11690,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } else { // ... else track the newline. - pm_newline_list_append(&parser->newline_list, parser->current.end); + pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start + 1)); } parser->current.end++; @@ -11819,7 +11819,7 @@ parser_lex(pm_parser_t *parser) { (memcmp(terminator_start, ident_start, ident_length) == 0) ) { if (newline != NULL) { - pm_newline_list_append(&parser->newline_list, newline); + pm_newline_list_append(&parser->newline_list, (uint32_t) (newline - parser->start + 1)); } parser->current.end = terminator_end; @@ -11891,7 +11891,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } - pm_newline_list_append(&parser->newline_list, breakpoint); + pm_newline_list_append(&parser->newline_list, (uint32_t) (breakpoint - parser->start + 1)); // If we have a - or ~ heredoc, then we can match after // some leading whitespace. @@ -12009,7 +12009,7 @@ parser_lex(pm_parser_t *parser) { // string content. if (heredoc_lex_mode->indent == PM_HEREDOC_INDENT_TILDE) { const uint8_t *end = parser->current.end; - pm_newline_list_append(&parser->newline_list, end); + pm_newline_list_append(&parser->newline_list, (uint32_t) (end - parser->start + 1)); // Here we want the buffer to only // include up to the backslash. @@ -13220,11 +13220,11 @@ parse_statements(pm_parser_t *parser, pm_context_t context, uint16_t depth) { */ static void pm_hash_key_static_literals_add(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { - const pm_node_t *duplicated = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node, true); + const pm_node_t *duplicated = pm_static_literals_add(&parser->newline_list, parser->start, parser->start_line, literals, node, true); if (duplicated != NULL) { pm_buffer_t buffer = { 0 }; - pm_static_literal_inspect(&buffer, &parser->newline_list, parser->start_line, parser->encoding->name, duplicated); + pm_static_literal_inspect(&buffer, &parser->newline_list, parser->start, parser->start_line, parser->encoding->name, duplicated); pm_diagnostic_list_append_format( &parser->warning_list, @@ -13233,7 +13233,7 @@ pm_hash_key_static_literals_add(pm_parser_t *parser, pm_static_literals_t *liter PM_WARN_DUPLICATED_HASH_KEY, (int) pm_buffer_length(&buffer), pm_buffer_value(&buffer), - pm_newline_list_line_column(&parser->newline_list, node->location.start, parser->start_line).line + pm_newline_list_line_column(&parser->newline_list, (uint32_t) (node->location.start - parser->start), parser->start_line).line ); pm_buffer_free(&buffer); @@ -13248,14 +13248,14 @@ static void pm_when_clause_static_literals_add(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { pm_node_t *previous; - if ((previous = pm_static_literals_add(&parser->newline_list, parser->start_line, literals, node, false)) != NULL) { + if ((previous = pm_static_literals_add(&parser->newline_list, parser->start, parser->start_line, literals, node, false)) != NULL) { pm_diagnostic_list_append_format( &parser->warning_list, (uint32_t) (node->location.start - parser->start), (uint32_t) (node->location.end - node->location.start), PM_WARN_DUPLICATED_WHEN_CLAUSE, - pm_newline_list_line_column(&parser->newline_list, node->location.start, parser->start_line).line, - pm_newline_list_line_column(&parser->newline_list, previous->location.start, parser->start_line).line + pm_newline_list_line_column(&parser->newline_list, (uint32_t) (node->location.start - parser->start), parser->start_line).line, + pm_newline_list_line_column(&parser->newline_list, (uint32_t) (previous->location.start - parser->start), parser->start_line).line ); } } @@ -14180,7 +14180,7 @@ token_newline_index(const pm_parser_t *parser) { // start of a heredoc, so we cannot rely on looking at the previous // offset of the newline list, and instead must go through the whole // process of a binary search for the line number. - return (size_t) pm_newline_list_line(&parser->newline_list, parser->current.start, 0); + return (size_t) pm_newline_list_line(&parser->newline_list, (uint32_t) (parser->current.start - parser->start), 0); } } @@ -16343,7 +16343,7 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca */ static void parse_pattern_hash_key(pm_parser_t *parser, pm_static_literals_t *keys, pm_node_t *node) { - if (pm_static_literals_add(&parser->newline_list, parser->start_line, keys, node, true) != NULL) { + if (pm_static_literals_add(&parser->newline_list, parser->start, parser->start_line, keys, node, true) != NULL) { pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_KEY_DUPLICATE); } } @@ -21834,7 +21834,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm // guess at the number of newlines that we'll need based on the size of the // input. size_t newline_size = size / 22; - pm_newline_list_init(&parser->newline_list, source, newline_size < 4 ? 4 : newline_size); + pm_newline_list_init(&parser->newline_list, newline_size < 4 ? 4 : newline_size); // If options were provided to this parse, establish them here. if (options != NULL) { @@ -21973,7 +21973,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm const uint8_t *newline = next_newline(cursor, parser->end - cursor); while (newline != NULL) { - pm_newline_list_append(&parser->newline_list, newline); + pm_newline_list_append(&parser->newline_list, (uint32_t) (newline - parser->start + 1)); cursor = newline + 1; newline = next_newline(cursor, parser->end - cursor); diff --git a/src/static_literals.c b/src/static_literals.c index 9fa37b999a..35110b9535 100644 --- a/src/static_literals.c +++ b/src/static_literals.c @@ -9,6 +9,9 @@ typedef struct { /** The list of newline offsets to use to calculate line numbers. */ const pm_newline_list_t *newline_list; + /** The start of the source being parsed. */ + const uint8_t *start; + /** The line number that the parser starts on. */ int32_t start_line; @@ -92,7 +95,7 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node) } case PM_SOURCE_LINE_NODE: { // Source lines hash their line number. - const pm_line_column_t line_column = pm_newline_list_line_column(metadata->newline_list, node->location.start, metadata->start_line); + const pm_line_column_t line_column = pm_newline_list_line_column(metadata->newline_list, (uint32_t) (node->location.start - metadata->start), metadata->start_line); const int32_t *value = &line_column.line; return murmur_hash((const uint8_t *) value, sizeof(int32_t)); } @@ -240,7 +243,7 @@ pm_int64_value(const pm_static_literals_metadata_t *metadata, const pm_node_t *n return integer->negative ? -value : value; } case PM_SOURCE_LINE_NODE: - return (int64_t) pm_newline_list_line_column(metadata->newline_list, node->location.start, metadata->start_line).line; + return (int64_t) pm_newline_list_line_column(metadata->newline_list, (uint32_t) (node->location.start - metadata->start), metadata->start_line).line; default: assert(false && "unreachable"); return 0; @@ -353,7 +356,7 @@ pm_compare_regular_expression_nodes(PRISM_ATTRIBUTE_UNUSED const pm_static_liter * Add a node to the set of static literals. */ pm_node_t * -pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace) { +pm_static_literals_add(const pm_newline_list_t *newline_list, const uint8_t *start, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace) { switch (PM_NODE_TYPE(node)) { case PM_INTEGER_NODE: case PM_SOURCE_LINE_NODE: @@ -361,6 +364,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->integer_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -373,6 +377,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->float_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -386,6 +391,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->number_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -399,6 +405,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->string_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -411,6 +418,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->regexp_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -423,6 +431,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line &literals->symbol_nodes, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = NULL }, @@ -576,7 +585,7 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met break; } case PM_SOURCE_LINE_NODE: - pm_buffer_append_format(buffer, "%d", pm_newline_list_line_column(metadata->newline_list, node->location.start, metadata->start_line).line); + pm_buffer_append_format(buffer, "%d", pm_newline_list_line_column(metadata->newline_list, (uint32_t) (node->location.start - metadata->start), metadata->start_line).line); break; case PM_STRING_NODE: { const pm_string_t *unescaped = &((const pm_string_node_t *) node)->unescaped; @@ -604,11 +613,12 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met * Create a string-based representation of the given static literal. */ void -pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node) { +pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, const uint8_t *start, int32_t start_line, const char *encoding_name, const pm_node_t *node) { pm_static_literal_inspect_node( buffer, &(pm_static_literals_metadata_t) { .newline_list = newline_list, + .start = start, .start_line = start_line, .encoding_name = encoding_name }, diff --git a/src/util/pm_char.c b/src/util/pm_char.c index a51dc11645..05fadf9353 100644 --- a/src/util/pm_char.c +++ b/src/util/pm_char.c @@ -83,15 +83,15 @@ pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length) { * searching past the given maximum number of characters. */ size_t -pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list) { - if (length <= 0) return 0; +pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list, uint32_t start_offset) { + if (length <= 0 || length > UINT32_MAX) return 0; - size_t size = 0; - size_t maximum = (size_t) length; + uint32_t size = 0; + uint32_t maximum = (uint32_t) length; while (size < maximum && (pm_byte_table[string[size]] & PRISM_CHAR_BIT_WHITESPACE)) { if (string[size] == '\n') { - pm_newline_list_append(newline_list, string + size); + pm_newline_list_append(newline_list, start_offset + size + 1); } size++; diff --git a/src/util/pm_newline_list.c b/src/util/pm_newline_list.c index 8331618f54..89c294a6d7 100644 --- a/src/util/pm_newline_list.c +++ b/src/util/pm_newline_list.c @@ -5,12 +5,10 @@ * allocation of the offsets succeeds, otherwise returns false. */ bool -pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity) { - list->offsets = (size_t *) xcalloc(capacity, sizeof(size_t)); +pm_newline_list_init(pm_newline_list_t *list, size_t capacity) { + list->offsets = (uint32_t *) xcalloc(capacity, sizeof(uint32_t)); if (list->offsets == NULL) return false; - list->start = start; - // This is 1 instead of 0 because we want to include the first line of the // file as having offset 0, which is set because of calloc. list->size = 1; @@ -32,24 +30,20 @@ pm_newline_list_clear(pm_newline_list_t *list) { * the offsets succeeds (if one was necessary), otherwise returns false. */ bool -pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor) { +pm_newline_list_append(pm_newline_list_t *list, uint32_t cursor) { if (list->size == list->capacity) { - size_t *original_offsets = list->offsets; + uint32_t *original_offsets = list->offsets; list->capacity = (list->capacity * 3) / 2; - list->offsets = (size_t *) xcalloc(list->capacity, sizeof(size_t)); + list->offsets = (uint32_t *) xcalloc(list->capacity, sizeof(uint32_t)); if (list->offsets == NULL) return false; - memcpy(list->offsets, original_offsets, list->size * sizeof(size_t)); + memcpy(list->offsets, original_offsets, list->size * sizeof(uint32_t)); xfree(original_offsets); } - assert(*cursor == '\n'); - assert(cursor >= list->start); - size_t newline_offset = (size_t) (cursor - list->start + 1); - - assert(list->size == 0 || newline_offset > list->offsets[list->size - 1]); - list->offsets[list->size++] = newline_offset; + assert(list->size == 0 || cursor > list->offsets[list->size - 1]); + list->offsets[list->size++] = cursor; return true; } @@ -59,21 +53,18 @@ pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor) { * line of the closest offset less than the given offset is returned. */ int32_t -pm_newline_list_line(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line) { - assert(cursor >= list->start); - size_t offset = (size_t) (cursor - list->start); - +pm_newline_list_line(const pm_newline_list_t *list, uint32_t cursor, int32_t start_line) { size_t left = 0; size_t right = list->size - 1; while (left <= right) { size_t mid = left + (right - left) / 2; - if (list->offsets[mid] == offset) { + if (list->offsets[mid] == cursor) { return ((int32_t) mid) + start_line; } - if (list->offsets[mid] < offset) { + if (list->offsets[mid] < cursor) { left = mid + 1; } else { right = mid - 1; @@ -89,21 +80,18 @@ pm_newline_list_line(const pm_newline_list_t *list, const uint8_t *cursor, int32 * are returned. */ pm_line_column_t -pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line) { - assert(cursor >= list->start); - size_t offset = (size_t) (cursor - list->start); - +pm_newline_list_line_column(const pm_newline_list_t *list, uint32_t cursor, int32_t start_line) { size_t left = 0; size_t right = list->size - 1; while (left <= right) { size_t mid = left + (right - left) / 2; - if (list->offsets[mid] == offset) { + if (list->offsets[mid] == cursor) { return ((pm_line_column_t) { ((int32_t) mid) + start_line, 0 }); } - if (list->offsets[mid] < offset) { + if (list->offsets[mid] < cursor) { left = mid + 1; } else { right = mid - 1; @@ -112,7 +100,7 @@ pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor return ((pm_line_column_t) { .line = ((int32_t) left) + start_line - 1, - .column = (uint32_t) (offset - list->offsets[left - 1]) + .column = cursor - list->offsets[left - 1] }); } diff --git a/templates/src/prettyprint.c.erb b/templates/src/prettyprint.c.erb index ac4e2b5465..aa904e80bc 100644 --- a/templates/src/prettyprint.c.erb +++ b/templates/src/prettyprint.c.erb @@ -12,15 +12,15 @@ void pm_prettyprint(void) {} static inline void prettyprint_location(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_location_t *location) { - pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, location->start, parser->start_line); - pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, location->end, parser->start_line); + pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, (uint32_t) (location->start - parser->start), parser->start_line); + pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, (uint32_t) (location->end - parser->start), parser->start_line); pm_buffer_append_format(output_buffer, "(%" PRIi32 ",%" PRIu32 ")-(%" PRIi32 ",%" PRIu32 ")", start.line, start.column, end.line, end.column); } static inline void prettyprint_slice(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_slice_t *slice) { - pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, parser->start + slice->start, parser->start_line); - pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, parser->start + slice->start + slice->length, parser->start_line); + pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, slice->start, parser->start_line); + pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, slice->start + slice->length, parser->start_line); pm_buffer_append_format(output_buffer, "(%" PRIi32 ",%" PRIu32 ")-(%" PRIi32 ",%" PRIu32 ")", start.line, start.column, end.line, end.column); } From 53e9f0a3330edcdbccdc050286984d48260fc5c1 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 2 Dec 2025 17:52:59 -0500 Subject: [PATCH 15/22] Rename some of the macros --- docs/configuration.md | 4 +- src/prism.c | 537 +++++++++++++++++++++--------------------- 2 files changed, 269 insertions(+), 272 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2e3ff02579..1e5297297f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -60,8 +60,8 @@ The available values for `type` are: * `string` - A field that is a string. For example, this is used as the name of the method in a call node, since it cannot directly reference the source string (as in `@-` or `foo=`). This is a `pm_string_t` in C. * `constant` - A field that is an integer that represents an index in the constant pool. This is a `pm_constant_id_t` in C. * `constant[]` - A field that is an array of constants. This is a `pm_constant_id_list_t` in C. -* `location` - A field that is a location. This is a `pm_location_t` in C. -* `location?` - A field that is a location that is optionally present. This is a `pm_location_t` in C, but if the value is not present then the `start` and `end` fields will be `NULL`. +* `location` - A field that is a location. This is a `pm_slice_t` in C. +* `location?` - A field that is a location that is optionally present. This is a `pm_slice_t` in C, but if the value is not present then the `length` field will be `0`. * `uint8` - A field that is an 8-bit unsigned integer. This is a `uint8_t` in C. * `uint32` - A field that is a 32-bit unsigned integer. This is a `uint32_t` in C. diff --git a/src/prism.c b/src/prism.c index 16db1b4bf6..1076b17ee6 100644 --- a/src/prism.c +++ b/src/prism.c @@ -19,7 +19,7 @@ pm_version(void) { #define MAX(a,b) (((a)>(b))?(a):(b)) /******************************************************************************/ -/* Helpful AST-related macros */ +/* Helpful AST-related macros */ /******************************************************************************/ #define FL PM_NODE_FLAGS @@ -31,10 +31,12 @@ pm_version(void) { #define PM_NODE_START(node_) (UP(node_)->location.start) #define PM_NODE_END(node_) (UP(node_)->location.end) -#define PM_LOCATION_NULL_VALUE(parser_) ((pm_location_t) { .start = (parser_)->start, .end = (parser_)->start }) #define PM_LOCATION_TOKEN_VALUE(token_) ((pm_location_t) { .start = PM_TOKEN_START(token_), .end = PM_TOKEN_END(token_) }) -#define PM_LOCATION_NODE_VALUE(node_) ((pm_location_t) { .start = PM_NODE_START(node_), .end = PM_NODE_END(node_) }) -#define PM_OPTIONAL_LOCATION_TOKEN_VALUE(token) ((token)->type == PM_TOKEN_NOT_PROVIDED ? ((pm_location_t) { 0 }) : PM_LOCATION_TOKEN_VALUE(token)) + +#define TOK2SLICE(parser_, token_) ((pm_slice_t) { .start = (uint32_t) ((token_)->start - (parser_)->start), .length = (uint32_t) ((token_)->end - (token_)->start) }) +#define NTOK2SLICE(parser_, token_) ((token_) == NULL ? ((pm_slice_t) { 0 }) : TOK2SLICE(parser_, token_)) +#define NTOK2PTR(token_) ((token_).start == NULL ? NULL : &(token_)) +#define LOC2SLICE TOK2SLICE /******************************************************************************/ /* Lex mode manipulations */ @@ -1961,11 +1963,6 @@ pm_missing_node_create(pm_parser_t *parser, const uint8_t *start, const uint8_t return node; } -#define TOKEN2SLICE(parser_, token_) ((pm_slice_t) { .start = (uint32_t) ((token_)->start - (parser_)->start), .length = (uint32_t) ((token_)->end - (token_)->start) }) -#define MAYBETOKEN2SLICE(parser_, token_) ((token_) == NULL ? ((pm_slice_t) { 0 }) : TOKEN2SLICE(parser_, token_)) -#define MAYBETOKENPTR(token_) ((token_).start == NULL ? NULL : &(token_)) -#define LOCATION2SLICE TOKEN2SLICE - /** * Allocate and initialize a new AliasGlobalVariableNode node. */ @@ -1978,7 +1975,7 @@ pm_alias_global_variable_node_create(pm_parser_t *parser, const pm_token_t *keyw .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_ALIAS_GLOBAL_VARIABLE_NODE, 0, keyword, old_name), .new_name = new_name, .old_name = old_name, - .keyword_loc = TOKEN2SLICE(parser, keyword) + .keyword_loc = TOK2SLICE(parser, keyword) }; return node; @@ -1996,7 +1993,7 @@ pm_alias_method_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_n .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_ALIAS_METHOD_NODE, 0, keyword, old_name), .new_name = new_name, .old_name = old_name, - .keyword_loc = TOKEN2SLICE(parser, keyword) + .keyword_loc = TOK2SLICE(parser, keyword) }; return node; @@ -2013,7 +2010,7 @@ pm_alternation_pattern_node_create(pm_parser_t *parser, pm_node_t *left, pm_node .base = PM_NODE_INIT_NODES(parser, PM_ALTERNATION_PATTERN_NODE, 0, left, right), .left = left, .right = right, - .operator_loc = TOKEN2SLICE(parser, operator) + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -2031,7 +2028,7 @@ pm_and_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *opera *node = (pm_and_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_AND_NODE, 0, left, right), .left = left, - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .right = right }; @@ -2099,8 +2096,8 @@ pm_array_node_create(pm_parser_t *parser, const pm_token_t *opening) { } else { *node = (pm_array_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_ARRAY_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, opening), + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, opening), .elements = { 0 } }; } @@ -2138,7 +2135,7 @@ static void pm_array_node_close_set(const pm_parser_t *parser, pm_array_node_t *node, const pm_token_t *closing) { assert(closing->type == PM_TOKEN_BRACKET_RIGHT || closing->type == PM_TOKEN_STRING_END || closing->type == 0); node->base.location.end = closing->end; - node->closing_loc = TOKEN2SLICE(parser, closing); + node->closing_loc = TOK2SLICE(parser, closing); } /** @@ -2210,8 +2207,8 @@ pm_array_pattern_node_constant_create(pm_parser_t *parser, pm_node_t *constant, .base = PM_NODE_INIT_NODE_TOKEN(parser, PM_ARRAY_PATTERN_NODE, 0, constant, closing), .constant = constant, .rest = NULL, - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, closing), + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, closing), .requireds = { 0 }, .posts = { 0 } }; @@ -2231,8 +2228,8 @@ pm_array_pattern_node_empty_create(pm_parser_t *parser, const pm_token_t *openin .base = PM_NODE_INIT_TOKENS(parser, PM_ARRAY_PATTERN_NODE, 0, opening, closing), .constant = NULL, .rest = NULL, - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, closing), + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, closing), .requireds = { 0 }, .posts = { 0 } }; @@ -2281,7 +2278,7 @@ pm_assoc_node_create(pm_parser_t *parser, pm_node_t *key, const pm_token_t *oper *node = (pm_assoc_node_t) { .base = PM_NODE_INIT(parser, PM_ASSOC_NODE, flags, key->location.start, end), .key = key, - .operator_loc = MAYBETOKEN2SLICE(parser, operator), + .operator_loc = NTOK2SLICE(parser, operator), .value = value }; @@ -2303,7 +2300,7 @@ pm_assoc_splat_node_create(pm_parser_t *parser, pm_node_t *value, const pm_token : PM_NODE_INIT_TOKEN_NODE(parser, PM_ASSOC_SPLAT_NODE, 0, operator, value) ), .value = value, - .operator_loc = TOKEN2SLICE(parser, operator) + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -2337,7 +2334,7 @@ pm_begin_node_create(pm_parser_t *parser, const pm_token_t *begin_keyword, pm_st *node = (pm_begin_node_t) { .base = PM_NODE_INIT(parser, PM_BEGIN_NODE, 0, start, end), - .begin_keyword_loc = MAYBETOKEN2SLICE(parser, begin_keyword), + .begin_keyword_loc = NTOK2SLICE(parser, begin_keyword), .statements = statements, .end_keyword_loc = { 0 } }; @@ -2383,7 +2380,7 @@ static void pm_begin_node_end_keyword_set(const pm_parser_t *parser, pm_begin_node_t *node, const pm_token_t *end_keyword) { assert(end_keyword->type == PM_TOKEN_KEYWORD_END || end_keyword->type == 0); node->base.location.end = end_keyword->end; - node->end_keyword_loc = TOKEN2SLICE(parser, end_keyword); + node->end_keyword_loc = TOK2SLICE(parser, end_keyword); } /** @@ -2401,7 +2398,7 @@ pm_block_argument_node_create(pm_parser_t *parser, const pm_token_t *operator, p : PM_NODE_INIT_TOKEN_NODE(parser, PM_BLOCK_ARGUMENT_NODE, 0, operator, expression) ), .expression = expression, - .operator_loc = TOKEN2SLICE(parser, operator) + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -2419,8 +2416,8 @@ pm_block_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const p .locals = *locals, .parameters = parameters, .body = body, - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, closing) + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, closing) }; return node; @@ -2441,8 +2438,8 @@ pm_block_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, cons : PM_NODE_INIT_TOKENS(parser, PM_BLOCK_PARAMETER_NODE, 0, operator, name) ), .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), - .name_loc = MAYBETOKEN2SLICE(parser, name), - .operator_loc = TOKEN2SLICE(parser, operator) + .name_loc = NTOK2SLICE(parser, name), + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -2476,7 +2473,7 @@ pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *param *node = (pm_block_parameters_node_t) { .base = PM_NODE_INIT(parser, PM_BLOCK_PARAMETERS_NODE, 0, start, end), .parameters = parameters, - .opening_loc = MAYBETOKEN2SLICE(parser, opening), + .opening_loc = NTOK2SLICE(parser, opening), .closing_loc = { 0 }, .locals = { 0 } }; @@ -2491,7 +2488,7 @@ static void pm_block_parameters_node_closing_set(const pm_parser_t *parser, pm_block_parameters_node_t *node, const pm_token_t *closing) { assert(closing->type == PM_TOKEN_PIPE || closing->type == PM_TOKEN_PARENTHESIS_RIGHT || closing->type == 0); node->base.location.end = closing->end; - node->closing_loc = TOKEN2SLICE(parser, closing); + node->closing_loc = TOK2SLICE(parser, closing); } /** @@ -2535,7 +2532,7 @@ pm_break_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument : PM_NODE_INIT_TOKEN_NODE(parser, PM_BREAK_NODE, 0, keyword, arguments) ), .arguments = arguments, - .keyword_loc = TOKEN2SLICE(parser, keyword) + .keyword_loc = TOK2SLICE(parser, keyword) }; return node; @@ -2629,7 +2626,7 @@ pm_call_node_binary_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t node->base.location.end = MAX(receiver->location.end, argument->location.end); node->receiver = receiver; - node->message_loc = TOKEN2SLICE(parser, operator); + node->message_loc = TOK2SLICE(parser, operator); pm_arguments_node_t *arguments = pm_arguments_node_create(parser); pm_arguments_node_arguments_append(arguments, argument); @@ -2658,8 +2655,8 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o node->base.location.end = end; node->receiver = receiver; - node->call_operator_loc = TOKEN2SLICE(parser, operator); - node->message_loc = TOKEN2SLICE(parser, message); + node->call_operator_loc = TOK2SLICE(parser, operator); + node->message_loc = TOK2SLICE(parser, message); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2704,7 +2701,7 @@ pm_call_node_fcall_create(pm_parser_t *parser, pm_token_t *message, pm_arguments node->base.location.start = message->start; node->base.location.end = pm_arguments_end(parser, arguments); - node->message_loc = TOKEN2SLICE(parser, message); + node->message_loc = TOK2SLICE(parser, message); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2748,7 +2745,7 @@ pm_call_node_not_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *me } node->receiver = receiver; - node->message_loc = TOKEN2SLICE(parser, message); + node->message_loc = TOK2SLICE(parser, message); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2770,7 +2767,7 @@ pm_call_node_shorthand_create(pm_parser_t *parser, pm_node_t *receiver, pm_token node->base.location.end = pm_arguments_end(parser, arguments); node->receiver = receiver; - node->call_operator_loc = TOKEN2SLICE(parser, operator); + node->call_operator_loc = TOK2SLICE(parser, operator); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2797,7 +2794,7 @@ pm_call_node_unary_create(pm_parser_t *parser, pm_token_t *operator, pm_node_t * node->base.location.end = receiver->location.end; node->receiver = receiver; - node->message_loc = TOKEN2SLICE(parser, operator); + node->message_loc = TOK2SLICE(parser, operator); node->name = pm_parser_constant_id_constant(parser, name, strlen(name)); return node; @@ -2812,7 +2809,7 @@ pm_call_node_variable_call_create(pm_parser_t *parser, pm_token_t *message) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); node->base.location = PM_LOCATION_TOKEN_VALUE(message); - node->message_loc = TOKEN2SLICE(parser, message); + node->message_loc = TOK2SLICE(parser, message); node->name = pm_parser_constant_id_token(parser, message); return node; @@ -2871,7 +2868,7 @@ pm_call_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -2927,7 +2924,7 @@ pm_index_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, cons .arguments = target->arguments, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -2955,7 +2952,7 @@ pm_call_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .read_name = 0, .write_name = target->name, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .binary_operator_loc = TOKEN2SLICE(parser, operator), + .binary_operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -2988,7 +2985,7 @@ pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .binary_operator_loc = TOKEN2SLICE(parser, operator), + .binary_operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3016,7 +3013,7 @@ pm_call_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3049,7 +3046,7 @@ pm_index_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const .arguments = target->arguments, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3124,7 +3121,7 @@ pm_capture_pattern_node_create(pm_parser_t *parser, pm_node_t *value, pm_local_v .base = PM_NODE_INIT_NODES(parser, PM_CAPTURE_PATTERN_NODE, 0, value, target), .value = value, .target = target, - .operator_loc = TOKEN2SLICE(parser, operator) + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -3141,8 +3138,8 @@ pm_case_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node .base = PM_NODE_INIT(parser, PM_CASE_NODE, 0, case_keyword->start, end_keyword == NULL ? case_keyword->end : end_keyword->end), .predicate = predicate, .else_clause = NULL, - .case_keyword_loc = TOKEN2SLICE(parser, case_keyword), - .end_keyword_loc = MAYBETOKEN2SLICE(parser, end_keyword), + .case_keyword_loc = TOK2SLICE(parser, case_keyword), + .end_keyword_loc = NTOK2SLICE(parser, end_keyword), .conditions = { 0 } }; @@ -3175,7 +3172,7 @@ pm_case_node_else_clause_set(pm_case_node_t *node, pm_else_node_t *else_clause) static void pm_case_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_node_t *node, const pm_token_t *end_keyword) { node->base.location.end = end_keyword->end; - node->end_keyword_loc = TOKEN2SLICE(parser, end_keyword); + node->end_keyword_loc = TOK2SLICE(parser, end_keyword); } /** @@ -3189,7 +3186,7 @@ pm_case_match_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, p .base = PM_NODE_INIT_TOKEN(parser, PM_CASE_MATCH_NODE, 0, case_keyword), .predicate = predicate, .else_clause = NULL, - .case_keyword_loc = TOKEN2SLICE(parser, case_keyword), + .case_keyword_loc = TOK2SLICE(parser, case_keyword), .end_keyword_loc = { 0 }, .conditions = { 0 } }; @@ -3223,7 +3220,7 @@ pm_case_match_node_else_clause_set(pm_case_match_node_t *node, pm_else_node_t *e static void pm_case_match_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_match_node_t *node, const pm_token_t *end_keyword) { node->base.location.end = end_keyword->end; - node->end_keyword_loc = TOKEN2SLICE(parser, end_keyword); + node->end_keyword_loc = TOK2SLICE(parser, end_keyword); } /** @@ -3236,12 +3233,12 @@ pm_class_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const p *node = (pm_class_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_CLASS_NODE, 0, class_keyword, end_keyword), .locals = *locals, - .class_keyword_loc = TOKEN2SLICE(parser, class_keyword), + .class_keyword_loc = TOK2SLICE(parser, class_keyword), .constant_path = constant_path, - .inheritance_operator_loc = MAYBETOKEN2SLICE(parser, inheritance_operator), + .inheritance_operator_loc = NTOK2SLICE(parser, inheritance_operator), .superclass = superclass, .body = body, - .end_keyword_loc = TOKEN2SLICE(parser, end_keyword), + .end_keyword_loc = TOK2SLICE(parser, end_keyword), .name = pm_parser_constant_id_token(parser, name) }; @@ -3259,8 +3256,8 @@ pm_class_variable_and_write_node_create(pm_parser_t *parser, pm_class_variable_r *node = (pm_class_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOCATION2SLICE(parser, &target->base.location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->base.location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3277,8 +3274,8 @@ pm_class_variable_operator_write_node_create(pm_parser_t *parser, pm_class_varia *node = (pm_class_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOCATION2SLICE(parser, &target->base.location), - .binary_operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->base.location), + .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -3297,8 +3294,8 @@ pm_class_variable_or_write_node_create(pm_parser_t *parser, pm_class_variable_re *node = (pm_class_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOCATION2SLICE(parser, &target->base.location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->base.location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3346,8 +3343,8 @@ pm_class_variable_write_node_create(pm_parser_t *parser, pm_class_variable_read_ *node = (pm_class_variable_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_WRITE_NODE, flags, read_node, value), .name = read_node->name, - .name_loc = LOCATION2SLICE(parser, &read_node->base.location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &read_node->base.location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3365,7 +3362,7 @@ pm_constant_path_and_write_node_create(pm_parser_t *parser, pm_constant_path_nod *node = (pm_constant_path_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_AND_WRITE_NODE, 0, target, value), .target = target, - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3382,7 +3379,7 @@ pm_constant_path_operator_write_node_create(pm_parser_t *parser, pm_constant_pat *node = (pm_constant_path_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_OPERATOR_WRITE_NODE, 0, target, value), .target = target, - .binary_operator_loc = TOKEN2SLICE(parser, operator), + .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -3401,7 +3398,7 @@ pm_constant_path_or_write_node_create(pm_parser_t *parser, pm_constant_path_node *node = (pm_constant_path_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_OR_WRITE_NODE, 0, target, value), .target = target, - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3429,8 +3426,8 @@ pm_constant_path_node_create(pm_parser_t *parser, pm_node_t *parent, const pm_to ), .parent = parent, .name = name, - .delimiter_loc = TOKEN2SLICE(parser, delimiter), - .name_loc = TOKEN2SLICE(parser, name_token) + .delimiter_loc = TOK2SLICE(parser, delimiter), + .name_loc = TOK2SLICE(parser, name_token) }; return node; @@ -3447,7 +3444,7 @@ pm_constant_path_write_node_create(pm_parser_t *parser, pm_constant_path_node_t *node = (pm_constant_path_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_WRITE_NODE, flags, target, value), .target = target, - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3465,8 +3462,8 @@ pm_constant_and_write_node_create(pm_parser_t *parser, pm_constant_read_node_t * *node = (pm_constant_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_AND_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOCATION2SLICE(parser, &target->base.location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->base.location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3483,8 +3480,8 @@ pm_constant_operator_write_node_create(pm_parser_t *parser, pm_constant_read_nod *node = (pm_constant_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOCATION2SLICE(parser, &target->base.location), - .binary_operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->base.location), + .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -3503,8 +3500,8 @@ pm_constant_or_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *t *node = (pm_constant_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_OR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOCATION2SLICE(parser, &target->base.location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->base.location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3538,8 +3535,8 @@ pm_constant_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *targ *node = (pm_constant_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_WRITE_NODE, flags, target, value), .name = target->name, - .name_loc = LOCATION2SLICE(parser, &target->base.location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->base.location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3622,17 +3619,17 @@ pm_def_node_create( : PM_NODE_INIT_TOKENS(parser, PM_DEF_NODE, 0, def_keyword, end_keyword) ), .name = name, - .name_loc = TOKEN2SLICE(parser, name_loc), + .name_loc = TOK2SLICE(parser, name_loc), .receiver = receiver, .parameters = parameters, .body = body, .locals = *locals, - .def_keyword_loc = TOKEN2SLICE(parser, def_keyword), - .operator_loc = MAYBETOKEN2SLICE(parser, operator), - .lparen_loc = MAYBETOKEN2SLICE(parser, lparen), - .rparen_loc = MAYBETOKEN2SLICE(parser, rparen), - .equal_loc = MAYBETOKEN2SLICE(parser, equal), - .end_keyword_loc = MAYBETOKEN2SLICE(parser, end_keyword) + .def_keyword_loc = TOK2SLICE(parser, def_keyword), + .operator_loc = NTOK2SLICE(parser, operator), + .lparen_loc = NTOK2SLICE(parser, lparen), + .rparen_loc = NTOK2SLICE(parser, rparen), + .equal_loc = NTOK2SLICE(parser, equal), + .end_keyword_loc = NTOK2SLICE(parser, end_keyword) }; return node; @@ -3651,10 +3648,10 @@ pm_defined_node_create(pm_parser_t *parser, const pm_token_t *lparen, pm_node_t ? PM_NODE_INIT_TOKEN_NODE(parser, PM_DEFINED_NODE, 0, keyword, value) : PM_NODE_INIT_TOKENS(parser, PM_DEFINED_NODE, 0, keyword, rparen) ), - .lparen_loc = MAYBETOKEN2SLICE(parser, lparen), + .lparen_loc = NTOK2SLICE(parser, lparen), .value = value, - .rparen_loc = MAYBETOKEN2SLICE(parser, rparen), - .keyword_loc = TOKEN2SLICE(parser, keyword) + .rparen_loc = NTOK2SLICE(parser, rparen), + .keyword_loc = TOK2SLICE(parser, keyword) }; return node; @@ -3673,9 +3670,9 @@ pm_else_node_create(pm_parser_t *parser, const pm_token_t *else_keyword, pm_stat ? PM_NODE_INIT_TOKEN_NODE(parser, PM_ELSE_NODE, 0, else_keyword, statements) : PM_NODE_INIT_TOKENS(parser, PM_ELSE_NODE, 0, else_keyword, end_keyword) ), - .else_keyword_loc = TOKEN2SLICE(parser, else_keyword), + .else_keyword_loc = TOK2SLICE(parser, else_keyword), .statements = statements, - .end_keyword_loc = MAYBETOKEN2SLICE(parser, end_keyword) + .end_keyword_loc = NTOK2SLICE(parser, end_keyword) }; return node; @@ -3690,9 +3687,9 @@ pm_embedded_statements_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_embedded_statements_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_EMBEDDED_STATEMENTS_NODE, 0, opening, closing), - .opening_loc = TOKEN2SLICE(parser, opening), + .opening_loc = TOK2SLICE(parser, opening), .statements = statements, - .closing_loc = TOKEN2SLICE(parser, closing) + .closing_loc = TOK2SLICE(parser, closing) }; return node; @@ -3707,7 +3704,7 @@ pm_embedded_variable_node_create(pm_parser_t *parser, const pm_token_t *operator *node = (pm_embedded_variable_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_EMBEDDED_VARIABLE_NODE, 0, operator, variable), - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .variable = variable }; @@ -3723,9 +3720,9 @@ pm_ensure_node_create(pm_parser_t *parser, const pm_token_t *ensure_keyword, pm_ *node = (pm_ensure_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_ENSURE_NODE, 0, ensure_keyword, end_keyword), - .ensure_keyword_loc = TOKEN2SLICE(parser, ensure_keyword), + .ensure_keyword_loc = TOK2SLICE(parser, ensure_keyword), .statements = statements, - .end_keyword_loc = TOKEN2SLICE(parser, end_keyword) + .end_keyword_loc = TOK2SLICE(parser, end_keyword) }; return node; @@ -3993,10 +3990,10 @@ pm_for_node_create( .index = index, .collection = collection, .statements = statements, - .for_keyword_loc = TOKEN2SLICE(parser, for_keyword), - .in_keyword_loc = TOKEN2SLICE(parser, in_keyword), - .do_keyword_loc = MAYBETOKEN2SLICE(parser, do_keyword), - .end_keyword_loc = TOKEN2SLICE(parser, end_keyword) + .for_keyword_loc = TOK2SLICE(parser, for_keyword), + .in_keyword_loc = TOK2SLICE(parser, in_keyword), + .do_keyword_loc = NTOK2SLICE(parser, do_keyword), + .end_keyword_loc = TOK2SLICE(parser, end_keyword) }; return node; @@ -4069,8 +4066,8 @@ pm_hash_pattern_node_empty_create(pm_parser_t *parser, const pm_token_t *opening *node = (pm_hash_pattern_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_HASH_PATTERN_NODE, 0, opening, closing), .constant = NULL, - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, closing), + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, closing), .elements = { 0 }, .rest = NULL }; @@ -4150,8 +4147,8 @@ pm_global_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, *node = (pm_global_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = LOCATION2SLICE(parser, &target->location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4168,8 +4165,8 @@ pm_global_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *ta *node = (pm_global_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = LOCATION2SLICE(parser, &target->location), - .binary_operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->location), + .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -4188,8 +4185,8 @@ pm_global_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, *node = (pm_global_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = LOCATION2SLICE(parser, &target->location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4237,8 +4234,8 @@ pm_global_variable_write_node_create(pm_parser_t *parser, pm_node_t *target, con *node = (pm_global_variable_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_WRITE_NODE, flags, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = LOCATION2SLICE(parser, &target->location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4273,7 +4270,7 @@ pm_hash_node_create(pm_parser_t *parser, const pm_token_t *opening) { *node = (pm_hash_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_HASH_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = TOKEN2SLICE(parser, opening), + .opening_loc = TOK2SLICE(parser, opening), .closing_loc = { 0 }, .elements = { 0 } }; @@ -4304,7 +4301,7 @@ pm_hash_node_elements_append(pm_hash_node_t *hash, pm_node_t *element) { static inline void pm_hash_node_closing_loc_set(const pm_parser_t *parser, pm_hash_node_t *hash, pm_token_t *token) { hash->base.location.end = token->end; - hash->closing_loc = TOKEN2SLICE(parser, token); + hash->closing_loc = TOK2SLICE(parser, token); } /** @@ -4335,12 +4332,12 @@ pm_if_node_create(pm_parser_t *parser, *node = (pm_if_node_t) { .base = PM_NODE_INIT(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, if_keyword->start, end), - .if_keyword_loc = TOKEN2SLICE(parser, if_keyword), + .if_keyword_loc = TOK2SLICE(parser, if_keyword), .predicate = predicate, - .then_keyword_loc = MAYBETOKEN2SLICE(parser, then_keyword), + .then_keyword_loc = NTOK2SLICE(parser, then_keyword), .statements = statements, .subsequent = subsequent, - .end_keyword_loc = MAYBETOKEN2SLICE(parser, end_keyword) + .end_keyword_loc = NTOK2SLICE(parser, end_keyword) }; return node; @@ -4359,7 +4356,7 @@ pm_if_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const pm_t *node = (pm_if_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, statement, predicate), - .if_keyword_loc = TOKEN2SLICE(parser, if_keyword), + .if_keyword_loc = TOK2SLICE(parser, if_keyword), .predicate = predicate, .then_keyword_loc = { 0 }, .statements = statements, @@ -4391,7 +4388,7 @@ pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_to .base = PM_NODE_INIT_NODES(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, predicate, false_expression), .if_keyword_loc = { 0 }, .predicate = predicate, - .then_keyword_loc = TOKEN2SLICE(parser, qmark), + .then_keyword_loc = TOK2SLICE(parser, qmark), .statements = if_statements, .subsequent = UP(else_node), .end_keyword_loc = { 0 } @@ -4404,13 +4401,13 @@ pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_to static inline void pm_if_node_end_keyword_loc_set(const pm_parser_t *parser, pm_if_node_t *node, const pm_token_t *keyword) { node->base.location.end = keyword->end; - node->end_keyword_loc = TOKEN2SLICE(parser, keyword); + node->end_keyword_loc = TOK2SLICE(parser, keyword); } static inline void pm_else_node_end_keyword_loc_set(const pm_parser_t *parser, pm_else_node_t *node, const pm_token_t *keyword) { node->base.location.end = keyword->end; - node->end_keyword_loc = TOKEN2SLICE(parser, keyword); + node->end_keyword_loc = TOK2SLICE(parser, keyword); } /** @@ -4561,8 +4558,8 @@ pm_in_node_create(pm_parser_t *parser, pm_node_t *pattern, pm_statements_node_t .base = PM_NODE_INIT(parser, PM_IN_NODE, 0, in_keyword->start, end), .pattern = pattern, .statements = statements, - .in_loc = TOKEN2SLICE(parser, in_keyword), - .then_loc = MAYBETOKEN2SLICE(parser, then_keyword) + .in_loc = TOK2SLICE(parser, in_keyword), + .then_loc = NTOK2SLICE(parser, then_keyword) }; return node; @@ -4579,8 +4576,8 @@ pm_instance_variable_and_write_node_create(pm_parser_t *parser, pm_instance_vari *node = (pm_instance_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOCATION2SLICE(parser, &target->base.location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->base.location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4597,8 +4594,8 @@ pm_instance_variable_operator_write_node_create(pm_parser_t *parser, pm_instance *node = (pm_instance_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOCATION2SLICE(parser, &target->base.location), - .binary_operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->base.location), + .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -4617,8 +4614,8 @@ pm_instance_variable_or_write_node_create(pm_parser_t *parser, pm_instance_varia *node = (pm_instance_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOCATION2SLICE(parser, &target->base.location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->base.location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4653,8 +4650,8 @@ pm_instance_variable_write_node_create(pm_parser_t *parser, pm_instance_variable *node = (pm_instance_variable_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_WRITE_NODE, flags, read_node, value), .name = read_node->name, - .name_loc = LOCATION2SLICE(parser, &read_node->base.location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &read_node->base.location), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4714,8 +4711,8 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok *node = (pm_interpolated_regular_expression_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, opening), + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, opening), .parts = { 0 } }; @@ -4736,7 +4733,7 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio static inline void pm_interpolated_regular_expression_node_closing_set(pm_parser_t *parser, pm_interpolated_regular_expression_node_t *node, const pm_token_t *closing) { - node->closing_loc = TOKEN2SLICE(parser, closing); + node->closing_loc = TOK2SLICE(parser, closing); node->base.location.end = closing->end; pm_node_flag_set(UP(node), pm_regular_expression_flags_create(parser, closing)); } @@ -4873,8 +4870,8 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_string_node_t) { .base = PM_NODE_INIT(parser, PM_INTERPOLATED_STRING_NODE, flags, opening == NULL ? NULL : opening->start, closing == NULL ? NULL : closing->end), - .opening_loc = MAYBETOKEN2SLICE(parser, opening), - .closing_loc = MAYBETOKEN2SLICE(parser, closing), + .opening_loc = NTOK2SLICE(parser, opening), + .closing_loc = NTOK2SLICE(parser, closing), .parts = { 0 } }; @@ -4893,7 +4890,7 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin */ static void pm_interpolated_string_node_closing_set(const pm_parser_t *parser, pm_interpolated_string_node_t *node, const pm_token_t *closing) { - node->closing_loc = TOKEN2SLICE(parser, closing); + node->closing_loc = TOK2SLICE(parser, closing); node->base.location.end = closing->end; } @@ -4909,7 +4906,7 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ static void pm_interpolated_symbol_node_closing_loc_set(const pm_parser_t *parser, pm_interpolated_symbol_node_t *node, const pm_token_t *closing) { - node->closing_loc = TOKEN2SLICE(parser, closing); + node->closing_loc = TOK2SLICE(parser, closing); node->base.location.end = closing->end; } @@ -4922,8 +4919,8 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_symbol_node_t) { .base = PM_NODE_INIT(parser, PM_INTERPOLATED_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening == NULL ? NULL : opening->start, closing == NULL ? NULL : closing->end), - .opening_loc = MAYBETOKEN2SLICE(parser, opening), - .closing_loc = MAYBETOKEN2SLICE(parser, closing), + .opening_loc = NTOK2SLICE(parser, opening), + .closing_loc = NTOK2SLICE(parser, closing), .parts = { 0 } }; @@ -4946,8 +4943,8 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi *node = (pm_interpolated_x_string_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_INTERPOLATED_X_STRING_NODE, 0, opening, closing), - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, closing), + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, closing), .parts = { 0 } }; @@ -4962,7 +4959,7 @@ pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_no static inline void pm_interpolated_xstring_node_closing_set(const pm_parser_t *parser, pm_interpolated_x_string_node_t *node, const pm_token_t *closing) { - node->closing_loc = TOKEN2SLICE(parser, closing); + node->closing_loc = TOK2SLICE(parser, closing); node->base.location.end = closing->end; } @@ -5037,7 +5034,7 @@ pm_required_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t *node = (pm_required_keyword_parameter_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_REQUIRED_KEYWORD_PARAMETER_NODE, 0, name), .name = pm_parser_constant_id_location(parser, name->start, name->end - 1), - .name_loc = TOKEN2SLICE(parser, name), + .name_loc = TOK2SLICE(parser, name), }; return node; @@ -5053,7 +5050,7 @@ pm_optional_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t *node = (pm_optional_keyword_parameter_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_OPTIONAL_KEYWORD_PARAMETER_NODE, 0, name, value), .name = pm_parser_constant_id_location(parser, name->start, name->end - 1), - .name_loc = TOKEN2SLICE(parser, name), + .name_loc = TOK2SLICE(parser, name), .value = value }; @@ -5074,8 +5071,8 @@ pm_keyword_rest_parameter_node_create(pm_parser_t *parser, const pm_token_t *ope : PM_NODE_INIT_TOKENS(parser, PM_KEYWORD_REST_PARAMETER_NODE, 0, operator, name) ), .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), - .name_loc = MAYBETOKEN2SLICE(parser, name), - .operator_loc = TOKEN2SLICE(parser, operator) + .name_loc = NTOK2SLICE(parser, name), + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -5099,9 +5096,9 @@ pm_lambda_node_create( *node = (pm_lambda_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_LAMBDA_NODE, 0, operator, closing), .locals = *locals, - .operator_loc = TOKEN2SLICE(parser, operator), - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, closing), + .operator_loc = TOK2SLICE(parser, operator), + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, closing), .parameters = parameters, .body = body }; @@ -5120,8 +5117,8 @@ pm_local_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, *node = (pm_local_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_AND_WRITE_NODE, 0, target, value), - .name_loc = LOCATION2SLICE(parser, &target->location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->location), + .operator_loc = TOK2SLICE(parser, operator), .value = value, .name = name, .depth = depth @@ -5139,8 +5136,8 @@ pm_local_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *tar *node = (pm_local_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), - .name_loc = LOCATION2SLICE(parser, &target->location), - .binary_operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->location), + .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .name = name, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), @@ -5161,8 +5158,8 @@ pm_local_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, c *node = (pm_local_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_OR_WRITE_NODE, 0, target, value), - .name_loc = LOCATION2SLICE(parser, &target->location), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = LOC2SLICE(parser, &target->location), + .operator_loc = TOK2SLICE(parser, operator), .value = value, .name = name, .depth = depth @@ -5221,8 +5218,8 @@ pm_local_variable_write_node_create(pm_parser_t *parser, pm_constant_id_t name, .name = name, .depth = depth, .value = value, - .name_loc = LOCATION2SLICE(parser, name_loc), - .operator_loc = TOKEN2SLICE(parser, operator) + .name_loc = LOC2SLICE(parser, name_loc), + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -5287,7 +5284,7 @@ pm_match_predicate_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t .base = PM_NODE_INIT_NODES(parser, PM_MATCH_PREDICATE_NODE, 0, value, pattern), .value = value, .pattern = pattern, - .operator_loc = TOKEN2SLICE(parser, operator) + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -5306,7 +5303,7 @@ pm_match_required_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t * .base = PM_NODE_INIT_NODES(parser, PM_MATCH_REQUIRED_NODE, 0, value, pattern), .value = value, .pattern = pattern, - .operator_loc = TOKEN2SLICE(parser, operator) + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -5338,10 +5335,10 @@ pm_module_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const *node = (pm_module_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_MODULE_NODE, 0, module_keyword, end_keyword), .locals = (locals == NULL ? ((pm_constant_id_list_t) { .ids = NULL, .size = 0, .capacity = 0 }) : *locals), - .module_keyword_loc = TOKEN2SLICE(parser, module_keyword), + .module_keyword_loc = TOK2SLICE(parser, module_keyword), .constant_path = constant_path, .body = body, - .end_keyword_loc = TOKEN2SLICE(parser, end_keyword), + .end_keyword_loc = TOK2SLICE(parser, end_keyword), .name = pm_parser_constant_id_token(parser, name) }; @@ -5407,7 +5404,7 @@ pm_multi_target_node_targets_append(pm_parser_t *parser, pm_multi_target_node_t static void pm_multi_target_node_opening_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *lparen) { node->base.location.start = lparen->start; - node->lparen_loc = TOKEN2SLICE(parser, lparen); + node->lparen_loc = TOK2SLICE(parser, lparen); } /** @@ -5416,7 +5413,7 @@ pm_multi_target_node_opening_set(const pm_parser_t *parser, pm_multi_target_node static void pm_multi_target_node_closing_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *rparen) { node->base.location.end = rparen->end; - node->rparen_loc = TOKEN2SLICE(parser, rparen); + node->rparen_loc = TOK2SLICE(parser, rparen); } /** @@ -5434,7 +5431,7 @@ pm_multi_write_node_create(pm_parser_t *parser, pm_multi_target_node_t *target, .rights = target->rights, .lparen_loc = target->lparen_loc, .rparen_loc = target->rparen_loc, - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -5459,7 +5456,7 @@ pm_next_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_arguments ? PM_NODE_INIT_TOKEN(parser, PM_NEXT_NODE, 0, keyword) : PM_NODE_INIT_TOKEN_NODE(parser, PM_NEXT_NODE, 0, keyword, arguments) ), - .keyword_loc = TOKEN2SLICE(parser, keyword), + .keyword_loc = TOK2SLICE(parser, keyword), .arguments = arguments }; @@ -5492,8 +5489,8 @@ pm_no_keywords_parameter_node_create(pm_parser_t *parser, const pm_token_t *oper *node = (pm_no_keywords_parameter_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_NO_KEYWORDS_PARAMETER_NODE, 0, operator, keyword), - .operator_loc = TOKEN2SLICE(parser, operator), - .keyword_loc = TOKEN2SLICE(parser, keyword) + .operator_loc = TOK2SLICE(parser, operator), + .keyword_loc = TOK2SLICE(parser, keyword) }; return node; @@ -5589,8 +5586,8 @@ pm_optional_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, c *node = (pm_optional_parameter_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_OPTIONAL_PARAMETER_NODE, 0, name, value), .name = pm_parser_constant_id_token(parser, name), - .name_loc = TOKEN2SLICE(parser, name), - .operator_loc = TOKEN2SLICE(parser, operator), + .name_loc = TOK2SLICE(parser, name), + .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -5610,7 +5607,7 @@ pm_or_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *operat .base = PM_NODE_INIT_NODES(parser, PM_OR_NODE, 0, left, right), .left = left, .right = right, - .operator_loc = TOKEN2SLICE(parser, operator) + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -5746,8 +5743,8 @@ pm_parentheses_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_no *node = (pm_parentheses_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_PARENTHESES_NODE, flags, opening, closing), .body = body, - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, closing) + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, closing) }; return node; @@ -5763,9 +5760,9 @@ pm_pinned_expression_node_create(pm_parser_t *parser, pm_node_t *expression, con *node = (pm_pinned_expression_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_PINNED_EXPRESSION_NODE, 0, operator, rparen), .expression = expression, - .operator_loc = TOKEN2SLICE(parser, operator), - .lparen_loc = TOKEN2SLICE(parser, lparen), - .rparen_loc = TOKEN2SLICE(parser, rparen) + .operator_loc = TOK2SLICE(parser, operator), + .lparen_loc = TOK2SLICE(parser, lparen), + .rparen_loc = TOK2SLICE(parser, rparen) }; return node; @@ -5781,7 +5778,7 @@ pm_pinned_variable_node_create(pm_parser_t *parser, const pm_token_t *operator, *node = (pm_pinned_variable_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_PINNED_VARIABLE_NODE, 0, operator, variable), .variable = variable, - .operator_loc = TOKEN2SLICE(parser, operator) + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -5797,9 +5794,9 @@ pm_post_execution_node_create(pm_parser_t *parser, const pm_token_t *keyword, co *node = (pm_post_execution_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_POST_EXECUTION_NODE, 0, keyword, closing), .statements = statements, - .keyword_loc = TOKEN2SLICE(parser, keyword), - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, closing) + .keyword_loc = TOK2SLICE(parser, keyword), + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, closing) }; return node; @@ -5815,9 +5812,9 @@ pm_pre_execution_node_create(pm_parser_t *parser, const pm_token_t *keyword, con *node = (pm_pre_execution_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_PRE_EXECUTION_NODE, 0, keyword, closing), .statements = statements, - .keyword_loc = TOKEN2SLICE(parser, keyword), - .opening_loc = TOKEN2SLICE(parser, opening), - .closing_loc = TOKEN2SLICE(parser, closing) + .keyword_loc = TOK2SLICE(parser, keyword), + .opening_loc = TOK2SLICE(parser, opening), + .closing_loc = TOK2SLICE(parser, closing) }; return node; @@ -5853,7 +5850,7 @@ pm_range_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *ope .base = PM_NODE_INIT(parser, PM_RANGE_NODE, flags, (left == NULL ? operator->start : left->location.start), (right == NULL ? operator->end : right->location.end)), .left = left, .right = right, - .operator_loc = TOKEN2SLICE(parser, operator) + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -5885,9 +5882,9 @@ pm_regular_expression_node_create_unescaped(pm_parser_t *parser, const pm_token_ *node = (pm_regular_expression_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_REGULAR_EXPRESSION_NODE, flags, opening, closing), - .opening_loc = TOKEN2SLICE(parser, opening), - .content_loc = TOKEN2SLICE(parser, content), - .closing_loc = TOKEN2SLICE(parser, closing), + .opening_loc = TOK2SLICE(parser, opening), + .content_loc = TOK2SLICE(parser, content), + .closing_loc = TOK2SLICE(parser, closing), .unescaped = *unescaped }; @@ -5927,7 +5924,7 @@ pm_rescue_modifier_node_create(pm_parser_t *parser, pm_node_t *expression, const *node = (pm_rescue_modifier_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_RESCUE_MODIFIER_NODE, 0, expression, rescue_expression), .expression = expression, - .keyword_loc = TOKEN2SLICE(parser, keyword), + .keyword_loc = TOK2SLICE(parser, keyword), .rescue_expression = rescue_expression }; @@ -5943,7 +5940,7 @@ pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword) { *node = (pm_rescue_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_RESCUE_NODE, 0, keyword), - .keyword_loc = TOKEN2SLICE(parser, keyword), + .keyword_loc = TOK2SLICE(parser, keyword), .operator_loc = { 0 }, .then_keyword_loc = { 0 }, .reference = NULL, @@ -5957,7 +5954,7 @@ pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword) { static inline void pm_rescue_node_operator_set(const pm_parser_t *parser, pm_rescue_node_t *node, const pm_token_t *operator) { - node->operator_loc = TOKEN2SLICE(parser, operator); + node->operator_loc = TOK2SLICE(parser, operator); } /** @@ -6012,8 +6009,8 @@ pm_rest_parameter_node_create(pm_parser_t *parser, const pm_token_t *operator, c : PM_NODE_INIT_TOKENS(parser, PM_REST_PARAMETER_NODE, 0, operator, name) ), .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), - .name_loc = MAYBETOKEN2SLICE(parser, name), - .operator_loc = TOKEN2SLICE(parser, operator) + .name_loc = NTOK2SLICE(parser, name), + .operator_loc = TOK2SLICE(parser, operator) }; return node; @@ -6047,7 +6044,7 @@ pm_return_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argumen ? PM_NODE_INIT_TOKEN(parser, PM_RETURN_NODE, 0, keyword) : PM_NODE_INIT_TOKEN_NODE(parser, PM_RETURN_NODE, 0, keyword, arguments) ), - .keyword_loc = TOKEN2SLICE(parser, keyword), + .keyword_loc = TOK2SLICE(parser, keyword), .arguments = arguments }; @@ -6094,11 +6091,11 @@ pm_singleton_class_node_create(pm_parser_t *parser, pm_constant_id_list_t *local *node = (pm_singleton_class_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_SINGLETON_CLASS_NODE, 0, class_keyword, end_keyword), .locals = *locals, - .class_keyword_loc = TOKEN2SLICE(parser, class_keyword), - .operator_loc = TOKEN2SLICE(parser, operator), + .class_keyword_loc = TOK2SLICE(parser, class_keyword), + .operator_loc = TOK2SLICE(parser, operator), .expression = expression, .body = body, - .end_keyword_loc = TOKEN2SLICE(parser, end_keyword) + .end_keyword_loc = TOK2SLICE(parser, end_keyword) }; return node; @@ -6174,7 +6171,7 @@ pm_splat_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_node_t ? PM_NODE_INIT_TOKEN(parser, PM_SPLAT_NODE, 0, operator) : PM_NODE_INIT_TOKEN_NODE(parser, PM_SPLAT_NODE, 0, operator, expression) ), - .operator_loc = TOKEN2SLICE(parser, operator), + .operator_loc = TOK2SLICE(parser, operator), .expression = expression }; @@ -6286,9 +6283,9 @@ pm_string_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, *node = (pm_string_node_t) { .base = PM_NODE_INIT(parser, PM_STRING_NODE, flags, start, end), - .opening_loc = MAYBETOKEN2SLICE(parser, opening), - .content_loc = TOKEN2SLICE(parser, content), - .closing_loc = MAYBETOKEN2SLICE(parser, closing), + .opening_loc = NTOK2SLICE(parser, opening), + .content_loc = TOK2SLICE(parser, content), + .closing_loc = NTOK2SLICE(parser, closing), .unescaped = *string }; @@ -6329,7 +6326,7 @@ pm_super_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument *node = (pm_super_node_t) { .base = PM_NODE_INIT(parser, PM_SUPER_NODE, 0, keyword->start, end), - .keyword_loc = TOKEN2SLICE(parser, keyword), + .keyword_loc = TOK2SLICE(parser, keyword), .lparen_loc = arguments->opening_loc, .arguments = arguments->arguments, .rparen_loc = arguments->closing_loc, @@ -6562,9 +6559,9 @@ pm_symbol_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, *node = (pm_symbol_node_t) { .base = PM_NODE_INIT(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL | flags, start, end), - .opening_loc = MAYBETOKEN2SLICE(parser, opening), - .value_loc = MAYBETOKEN2SLICE(parser, value), - .closing_loc = MAYBETOKEN2SLICE(parser, closing), + .opening_loc = NTOK2SLICE(parser, opening), + .value_loc = NTOK2SLICE(parser, value), + .closing_loc = NTOK2SLICE(parser, closing), .unescaped = *unescaped }; @@ -6662,9 +6659,9 @@ pm_string_node_to_symbol_node(pm_parser_t *parser, pm_string_node_t *node, const *new_node = (pm_symbol_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening, closing), - .opening_loc = TOKEN2SLICE(parser, opening), + .opening_loc = TOK2SLICE(parser, opening), .value_loc = node->content_loc, - .closing_loc = TOKEN2SLICE(parser, closing), + .closing_loc = TOK2SLICE(parser, closing), .unescaped = node->unescaped }; @@ -6756,7 +6753,7 @@ pm_undef_node_create(pm_parser_t *parser, const pm_token_t *token) { *node = (pm_undef_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_UNDEF_NODE, 0, token), - .keyword_loc = TOKEN2SLICE(parser, token), + .keyword_loc = TOK2SLICE(parser, token), .names = { 0 } }; @@ -6784,9 +6781,9 @@ pm_unless_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_t *node = (pm_unless_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_UNLESS_NODE, PM_NODE_FLAG_NEWLINE, keyword, end), - .keyword_loc = TOKEN2SLICE(parser, keyword), + .keyword_loc = TOK2SLICE(parser, keyword), .predicate = predicate, - .then_keyword_loc = MAYBETOKEN2SLICE(parser, then_keyword), + .then_keyword_loc = NTOK2SLICE(parser, then_keyword), .statements = statements, .else_clause = NULL, .end_keyword_loc = { 0 } @@ -6808,7 +6805,7 @@ pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const *node = (pm_unless_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_UNLESS_NODE, PM_NODE_FLAG_NEWLINE, statement, predicate), - .keyword_loc = TOKEN2SLICE(parser, unless_keyword), + .keyword_loc = TOK2SLICE(parser, unless_keyword), .predicate = predicate, .then_keyword_loc = { 0 }, .statements = statements, @@ -6821,7 +6818,7 @@ pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const static inline void pm_unless_node_end_keyword_loc_set(const pm_parser_t *parser, pm_unless_node_t *node, const pm_token_t *end_keyword) { - node->end_keyword_loc = TOKEN2SLICE(parser, end_keyword); + node->end_keyword_loc = TOK2SLICE(parser, end_keyword); node->base.location.end = end_keyword->end; } @@ -6858,9 +6855,9 @@ pm_until_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_to *node = (pm_until_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_UNTIL_NODE, flags, keyword, closing), - .keyword_loc = TOKEN2SLICE(parser, keyword), - .do_keyword_loc = MAYBETOKEN2SLICE(parser, do_keyword), - .closing_loc = TOKEN2SLICE(parser, closing), + .keyword_loc = TOK2SLICE(parser, keyword), + .do_keyword_loc = NTOK2SLICE(parser, do_keyword), + .closing_loc = TOK2SLICE(parser, closing), .predicate = predicate, .statements = statements }; @@ -6879,7 +6876,7 @@ pm_until_node_modifier_create(pm_parser_t *parser, const pm_token_t *keyword, pm *node = (pm_until_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_UNTIL_NODE, flags, statements, predicate), - .keyword_loc = TOKEN2SLICE(parser, keyword), + .keyword_loc = TOK2SLICE(parser, keyword), .do_keyword_loc = { 0 }, .closing_loc = { 0 }, .predicate = predicate, @@ -6898,7 +6895,7 @@ pm_when_node_create(pm_parser_t *parser, const pm_token_t *keyword) { *node = (pm_when_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_WHEN_NODE, 0, keyword), - .keyword_loc = TOKEN2SLICE(parser, keyword), + .keyword_loc = TOK2SLICE(parser, keyword), .statements = NULL, .then_keyword_loc = { 0 }, .conditions = { 0 } @@ -6922,7 +6919,7 @@ pm_when_node_conditions_append(pm_when_node_t *node, pm_node_t *condition) { static inline void pm_when_node_then_keyword_loc_set(const pm_parser_t *parser, pm_when_node_t *node, const pm_token_t *then_keyword) { node->base.location.end = then_keyword->end; - node->then_keyword_loc = TOKEN2SLICE(parser, then_keyword); + node->then_keyword_loc = TOK2SLICE(parser, then_keyword); } /** @@ -6947,9 +6944,9 @@ pm_while_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_to *node = (pm_while_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_WHILE_NODE, flags, keyword, closing), - .keyword_loc = TOKEN2SLICE(parser, keyword), - .do_keyword_loc = MAYBETOKEN2SLICE(parser, do_keyword), - .closing_loc = TOKEN2SLICE(parser, closing), + .keyword_loc = TOK2SLICE(parser, keyword), + .do_keyword_loc = NTOK2SLICE(parser, do_keyword), + .closing_loc = TOK2SLICE(parser, closing), .predicate = predicate, .statements = statements }; @@ -6968,7 +6965,7 @@ pm_while_node_modifier_create(pm_parser_t *parser, const pm_token_t *keyword, pm *node = (pm_while_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_WHILE_NODE, flags, statements, predicate), - .keyword_loc = TOKEN2SLICE(parser, keyword), + .keyword_loc = TOK2SLICE(parser, keyword), .do_keyword_loc = { 0 }, .closing_loc = { 0 }, .predicate = predicate, @@ -7007,9 +7004,9 @@ pm_xstring_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, *node = (pm_x_string_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_X_STRING_NODE, PM_STRING_FLAGS_FROZEN, opening, closing), - .opening_loc = TOKEN2SLICE(parser, opening), - .content_loc = TOKEN2SLICE(parser, content), - .closing_loc = TOKEN2SLICE(parser, closing), + .opening_loc = TOK2SLICE(parser, opening), + .content_loc = TOK2SLICE(parser, content), + .closing_loc = TOK2SLICE(parser, closing), .unescaped = *unescaped }; @@ -7044,7 +7041,7 @@ pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_sl *node = (pm_yield_node_t) { .base = PM_NODE_INIT(parser, PM_YIELD_NODE, 0, keyword->start, end), - .keyword_loc = TOKEN2SLICE(parser, keyword), + .keyword_loc = TOK2SLICE(parser, keyword), .lparen_loc = *lparen_loc, .arguments = arguments, .rparen_loc = *rparen_loc @@ -9288,7 +9285,7 @@ parser_comment(pm_parser_t *parser, pm_comment_type_t type) { *comment = (pm_comment_t) { .type = type, - .location = TOKEN2SLICE(parser, &parser->current) + .location = TOK2SLICE(parser, &parser->current) }; return comment; @@ -12958,7 +12955,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod pm_arguments_node_arguments_append(arguments, value); call->base.location.end = arguments->base.location.end; - call->equal_loc = TOKEN2SLICE(parser, operator); + call->equal_loc = TOK2SLICE(parser, operator); parse_write_name(parser, &call->name); pm_node_flag_set(UP(call), PM_CALL_NODE_FLAGS_ATTRIBUTE_WRITE | pm_implicit_array_write_flags(value, PM_CALL_NODE_FLAGS_IMPLICIT_ARRAY)); @@ -12980,7 +12977,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod // Replace the name with "[]=". call->name = pm_parser_constant_id_constant(parser, "[]=", 3); - call->equal_loc = TOKEN2SLICE(parser, operator); + call->equal_loc = TOK2SLICE(parser, operator); // Ensure that the arguments for []= don't contain keywords pm_index_arguments_check(parser, call->arguments, call->block); @@ -13351,7 +13348,7 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod } pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1)); - element = UP(pm_assoc_node_create(parser, key, MAYBETOKENPTR(operator), value)); + element = UP(pm_assoc_node_create(parser, key, NTOK2PTR(operator), value)); break; } } @@ -13557,7 +13554,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for // Finish parsing the one we are part way through. pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1)); - argument = UP(pm_assoc_node_create(parser, argument, MAYBETOKENPTR(operator), value)); + argument = UP(pm_assoc_node_create(parser, argument, NTOK2PTR(operator), value)); pm_keyword_hash_node_elements_append(bare_hash, argument); argument = UP(bare_hash); @@ -13821,7 +13818,7 @@ parse_parameters( parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_BLOCK; } - pm_block_parameter_node_t *param = pm_block_parameter_node_create(parser, MAYBETOKENPTR(name), &operator); + pm_block_parameter_node_t *param = pm_block_parameter_node_create(parser, NTOK2PTR(name), &operator); if (repeated) { pm_node_flag_set_repeated_parameter(UP(param)); } @@ -14053,7 +14050,7 @@ parse_parameters( parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS; } - pm_node_t *param = UP(pm_rest_parameter_node_create(parser, &operator, MAYBETOKENPTR(name))); + pm_node_t *param = UP(pm_rest_parameter_node_create(parser, &operator, NTOK2PTR(name))); if (repeated) { pm_node_flag_set_repeated_parameter(param); } @@ -14094,7 +14091,7 @@ parse_parameters( parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_KEYWORDS; } - param = UP(pm_keyword_rest_parameter_node_create(parser, &operator, MAYBETOKENPTR(name))); + param = UP(pm_keyword_rest_parameter_node_create(parser, &operator, NTOK2PTR(name))); if (repeated) { pm_node_flag_set_repeated_parameter(param); } @@ -14340,11 +14337,11 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) { - rescue->then_keyword_loc = TOKEN2SLICE(parser, &parser->previous); + rescue->then_keyword_loc = TOK2SLICE(parser, &parser->previous); } } else { expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_RESCUE_TERM); - rescue->then_keyword_loc = TOKEN2SLICE(parser, &parser->previous); + rescue->then_keyword_loc = TOK2SLICE(parser, &parser->previous); } if (!match3(parser, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_END)) { @@ -14741,10 +14738,10 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept if (accept1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { found |= true; - arguments->opening_loc = TOKEN2SLICE(parser, &parser->previous); + arguments->opening_loc = TOK2SLICE(parser, &parser->previous); if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - arguments->closing_loc = TOKEN2SLICE(parser, &parser->previous); + arguments->closing_loc = TOK2SLICE(parser, &parser->previous); } else { pm_accepts_block_stack_push(parser, true); parse_arguments(parser, arguments, accepts_block, PM_TOKEN_PARENTHESIS_RIGHT, (uint16_t) (depth + 1)); @@ -14756,7 +14753,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept } pm_accepts_block_stack_pop(parser); - arguments->closing_loc = TOKEN2SLICE(parser, &parser->previous); + arguments->closing_loc = TOK2SLICE(parser, &parser->previous); } } else if (accepts_command_call && (token_begins_expression_p(parser->current.type) || match3(parser, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR, PM_TOKEN_UAMPERSAND)) && !match1(parser, PM_TOKEN_BRACE_LEFT)) { found |= true; @@ -15091,10 +15088,10 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl switch (context) { case PM_CONTEXT_IF: - parent = UP(pm_if_node_create(parser, &keyword, predicate, MAYBETOKENPTR(then_keyword), statements, NULL, NULL)); + parent = UP(pm_if_node_create(parser, &keyword, predicate, NTOK2PTR(then_keyword), statements, NULL, NULL)); break; case PM_CONTEXT_UNLESS: - parent = UP(pm_unless_node_create(parser, &keyword, predicate, MAYBETOKENPTR(then_keyword), statements)); + parent = UP(pm_unless_node_create(parser, &keyword, predicate, NTOK2PTR(then_keyword), statements)); break; default: assert(false && "unreachable"); @@ -15122,7 +15119,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl pm_accepts_block_stack_pop(parser); accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON); - pm_node_t *elsif = UP(pm_if_node_create(parser, &elsif_keyword, predicate, MAYBETOKENPTR(then_keyword), statements, NULL, NULL)); + pm_node_t *elsif = UP(pm_if_node_create(parser, &elsif_keyword, predicate, NTOK2PTR(then_keyword), statements, NULL, NULL)); ((pm_if_node_t *) current)->subsequent = elsif; current = elsif; } @@ -16149,8 +16146,8 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures pattern_node->base.location.end = closing.end; pattern_node->constant = node; - pattern_node->opening_loc = TOKEN2SLICE(parser, &opening); - pattern_node->closing_loc = TOKEN2SLICE(parser, &closing); + pattern_node->opening_loc = TOK2SLICE(parser, &opening); + pattern_node->closing_loc = TOK2SLICE(parser, &closing); return UP(pattern_node); } @@ -16165,8 +16162,8 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures pattern_node->base.location.end = closing.end; pattern_node->constant = node; - pattern_node->opening_loc = TOKEN2SLICE(parser, &opening); - pattern_node->closing_loc = TOKEN2SLICE(parser, &closing); + pattern_node->opening_loc = TOK2SLICE(parser, &opening); + pattern_node->closing_loc = TOK2SLICE(parser, &closing); return UP(pattern_node); } @@ -16181,8 +16178,8 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures pattern_node->base.location.end = closing.end; pattern_node->constant = node; - pattern_node->opening_loc = TOKEN2SLICE(parser, &opening); - pattern_node->closing_loc = TOKEN2SLICE(parser, &closing); + pattern_node->opening_loc = TOK2SLICE(parser, &opening); + pattern_node->closing_loc = TOK2SLICE(parser, &closing); return UP(pattern_node); } @@ -16518,8 +16515,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pattern_node->base.location.start = opening.start; pattern_node->base.location.end = closing.end; - pattern_node->opening_loc = TOKEN2SLICE(parser, &opening); - pattern_node->closing_loc = TOKEN2SLICE(parser, &closing); + pattern_node->opening_loc = TOK2SLICE(parser, &opening); + pattern_node->closing_loc = TOK2SLICE(parser, &closing); return UP(pattern_node); } @@ -16532,8 +16529,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pattern_node->base.location.start = opening.start; pattern_node->base.location.end = closing.end; - pattern_node->opening_loc = TOKEN2SLICE(parser, &opening); - pattern_node->closing_loc = TOKEN2SLICE(parser, &closing); + pattern_node->opening_loc = TOK2SLICE(parser, &opening); + pattern_node->closing_loc = TOK2SLICE(parser, &closing); return UP(pattern_node); } @@ -16592,8 +16589,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm node->base.location.start = opening.start; node->base.location.end = closing.end; - node->opening_loc = TOKEN2SLICE(parser, &opening); - node->closing_loc = TOKEN2SLICE(parser, &closing); + node->opening_loc = TOK2SLICE(parser, &opening); + node->closing_loc = TOK2SLICE(parser, &closing); } parser->pattern_matching_newlines = previous_pattern_matching_newlines; @@ -17401,7 +17398,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } pm_node_t *value = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_HASH_VALUE, (uint16_t) (depth + 1)); - pm_node_t *assoc = UP(pm_assoc_node_create(parser, element, MAYBETOKENPTR(operator), value)); + pm_node_t *assoc = UP(pm_assoc_node_create(parser, element, NTOK2PTR(operator), value)); pm_keyword_hash_node_elements_append(hash, assoc); element = UP(hash); @@ -17515,8 +17512,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_multi_target_node_targets_append(parser, multi_target, statement); } - multi_target->lparen_loc = TOKEN2SLICE(parser, &opening); - multi_target->rparen_loc = TOKEN2SLICE(parser, &parser->previous); + multi_target->lparen_loc = TOK2SLICE(parser, &opening); + multi_target->rparen_loc = TOK2SLICE(parser, &parser->previous); multi_target->base.location.start = opening.start; multi_target->base.location.end = parser->previous.end; @@ -17932,8 +17929,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_flag_set(part, parse_unescaped_encoding(parser)); pm_string_node_t *cast = (pm_string_node_t *) part; - cast->opening_loc = TOKEN2SLICE(parser, &opening); - cast->closing_loc = TOKEN2SLICE(parser, &parser->current); + cast->opening_loc = TOK2SLICE(parser, &opening); + cast->closing_loc = TOK2SLICE(parser, &parser->current); cast->base.location = (pm_location_t) { .start = parser->start + cast->opening_loc.start, .end = parser->start + cast->opening_loc.start + cast->opening_loc.length @@ -18260,7 +18257,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // Now that we have the full pattern and statements, we can // create the node and attach it to the case node. - pm_node_t *condition = UP(pm_in_node_create(parser, pattern, statements, &in_keyword, MAYBETOKENPTR(then_keyword))); + pm_node_t *condition = UP(pm_in_node_create(parser, pattern, statements, &in_keyword, NTOK2PTR(then_keyword))); pm_case_match_node_condition_append(case_node, condition); } @@ -18555,7 +18552,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pop_block_exits(parser, previous_block_exits); pm_node_list_free(¤t_block_exits); - return UP(pm_class_node_create(parser, &locals, &class_keyword, constant_path, &name, MAYBETOKENPTR(inheritance_operator), superclass, statements, &parser->previous)); + return UP(pm_class_node_create(parser, &locals, &class_keyword, constant_path, &name, NTOK2PTR(inheritance_operator), superclass, statements, &parser->previous)); } case PM_TOKEN_KEYWORD_DEF: { pm_node_list_t current_block_exits = { 0 }; @@ -18865,11 +18862,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b statements, &locals, &def_keyword, - MAYBETOKENPTR(operator), - MAYBETOKENPTR(lparen), - MAYBETOKENPTR(rparen), - MAYBETOKENPTR(equal), - MAYBETOKENPTR(end_keyword) + NTOK2PTR(operator), + NTOK2PTR(lparen), + NTOK2PTR(rparen), + NTOK2PTR(equal), + NTOK2PTR(end_keyword) )); } case PM_TOKEN_KEYWORD_DEFINED: { @@ -18905,9 +18902,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b context_pop(parser); return UP(pm_defined_node_create( parser, - MAYBETOKENPTR(lparen), + NTOK2PTR(lparen), expression, - MAYBETOKENPTR(rparen), + NTOK2PTR(rparen), &keyword )); } @@ -18992,7 +18989,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_warn_indentation_mismatch(parser, opening_newline_index, &for_keyword, false, false); expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_FOR_TERM); - return UP(pm_for_node_create(parser, index, collection, statements, &for_keyword, &in_keyword, MAYBETOKENPTR(do_keyword), &parser->previous)); + return UP(pm_for_node_create(parser, index, collection, statements, &for_keyword, &in_keyword, NTOK2PTR(do_keyword), &parser->previous)); } case PM_TOKEN_KEYWORD_IF: if (parser_end_of_line_p(parser)) { @@ -19063,13 +19060,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { receiver = UP(pm_parentheses_node_create(parser, &lparen, NULL, &parser->previous, 0)); } else { - arguments.opening_loc = TOKEN2SLICE(parser, &lparen); + arguments.opening_loc = TOK2SLICE(parser, &lparen); receiver = parse_expression(parser, PM_BINDING_POWER_COMPOSITION, true, false, PM_ERR_NOT_EXPRESSION, (uint16_t) (depth + 1)); if (!parser->recovering) { accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN); - arguments.closing_loc = TOKEN2SLICE(parser, &parser->previous); + arguments.closing_loc = TOK2SLICE(parser, &parser->previous); } } } else { @@ -19208,7 +19205,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false); expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_UNTIL_TERM); - return UP(pm_until_node_create(parser, &keyword, MAYBETOKENPTR(do_keyword), &parser->previous, predicate, statements, 0)); + return UP(pm_until_node_create(parser, &keyword, NTOK2PTR(do_keyword), &parser->previous, predicate, statements, 0)); } case PM_TOKEN_KEYWORD_WHILE: { size_t opening_newline_index = token_newline_index(parser); @@ -19241,7 +19238,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false); expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_WHILE_TERM); - return UP(pm_while_node_create(parser, &keyword, MAYBETOKENPTR(do_keyword), &parser->previous, predicate, statements, 0)); + return UP(pm_while_node_create(parser, &keyword, NTOK2PTR(do_keyword), &parser->previous, predicate, statements, 0)); } case PM_TOKEN_PERCENT_LOWER_I: { parser_lex(parser); @@ -21263,7 +21260,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t parser_lex(parser); pm_arguments_t arguments = { 0 }; - arguments.opening_loc = TOKEN2SLICE(parser, &parser->previous); + arguments.opening_loc = TOK2SLICE(parser, &parser->previous); if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { pm_accepts_block_stack_push(parser, true); @@ -21272,7 +21269,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_EXPECT_RBRACKET); } - arguments.closing_loc = TOKEN2SLICE(parser, &parser->previous); + arguments.closing_loc = TOK2SLICE(parser, &parser->previous); // If we have a comma after the closing bracket then this is a multiple // assignment and we should parse the targets. From 707bde07e1a264bd83dfe1dd339e611a9fc391c0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 2 Dec 2025 21:48:05 -0500 Subject: [PATCH 16/22] Switch numbered parameter checks to use uint32_t --- src/prism.c | 74 ++++++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/src/prism.c b/src/prism.c index 1076b17ee6..0e2aa3c0fd 100644 --- a/src/prism.c +++ b/src/prism.c @@ -22,14 +22,18 @@ pm_version(void) { /* Helpful AST-related macros */ /******************************************************************************/ +#define U32(value_) ((uint32_t) (value_)) + #define FL PM_NODE_FLAGS #define UP PM_NODE_UPCAST #define PM_TOKEN_START(token_) ((token_)->start) #define PM_TOKEN_END(token_) ((token_)->end) +#define PM_TOKEN_LENGTH(token_) U32(PM_TOKEN_END(token_) - PM_TOKEN_START(token_)) #define PM_NODE_START(node_) (UP(node_)->location.start) #define PM_NODE_END(node_) (UP(node_)->location.end) +#define PM_NODE_LENGTH(node_) U32(PM_NODE_END(node_) - PM_NODE_START(node_)) #define PM_LOCATION_TOKEN_VALUE(token_) ((pm_location_t) { .start = PM_TOKEN_START(token_), .end = PM_TOKEN_END(token_) }) @@ -434,6 +438,12 @@ pm_parser_err(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_ #define PM_PARSER_ERR_FORMAT(parser_, start_, end_, diag_id_, ...) \ pm_diagnostic_list_append_format(&(parser_)->error_list, (uint32_t) ((start_) - (parser_)->start), (uint32_t) ((end_) - (start_)), diag_id_, __VA_ARGS__) +/** + * Append an error to the list of errors on the parser using the given slice. + */ +#define PM_PARSER_ERR_FORMAT_SLICE(parser_, start_, length_, diag_id_, ...) \ + pm_diagnostic_list_append_format(&(parser_)->error_list, (start_), (length_), diag_id_, __VA_ARGS__) + /** * Append an error to the list of errors on the parser using the location of the * current token. @@ -5238,8 +5248,13 @@ pm_token_is_it(const uint8_t *start, const uint8_t *end) { * are of the form /^_\d$/). */ static inline bool -pm_token_is_numbered_parameter(const uint8_t *start, const uint8_t *end) { - return (end - start == 2) && (start[0] == '_') && (start[1] != '0') && (pm_char_is_decimal_digit(start[1])); +pm_token_is_numbered_parameter(const pm_parser_t *parser, uint32_t start, uint32_t length) { + return ( + (length == 2) && + (parser->start[start] == '_') && + (parser->start[start + 1] != '0') && + pm_char_is_decimal_digit(parser->start[start + 1]) + ); } /** @@ -5247,9 +5262,9 @@ pm_token_is_numbered_parameter(const uint8_t *start, const uint8_t *end) { * an appropriate error message to the parser. */ static inline void -pm_refute_numbered_parameter(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { - if (pm_token_is_numbered_parameter(start, end)) { - PM_PARSER_ERR_FORMAT(parser, start, end, PM_ERR_PARAMETER_NUMBERED_RESERVED, start); +pm_refute_numbered_parameter(pm_parser_t *parser, uint32_t start, uint32_t length) { + if (pm_token_is_numbered_parameter(parser, start, length)) { + PM_PARSER_ERR_FORMAT_SLICE(parser, start, length, PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + start); } } @@ -5259,7 +5274,7 @@ pm_refute_numbered_parameter(pm_parser_t *parser, const uint8_t *start, const ui */ static pm_local_variable_target_node_t * pm_local_variable_target_node_create(pm_parser_t *parser, const pm_location_t *location, pm_constant_id_t name, uint32_t depth) { - pm_refute_numbered_parameter(parser, location->start, location->end); + pm_refute_numbered_parameter(parser, U32(location->start - parser->start), PM_TOKEN_LENGTH(location)); pm_local_variable_target_node_t *node = PM_NODE_ALLOC(parser, pm_local_variable_target_node_t); *node = (pm_local_variable_target_node_t) { @@ -7145,7 +7160,7 @@ static bool pm_parser_parameter_name_check(pm_parser_t *parser, const pm_token_t *name) { // We want to check whether the parameter name is a numbered parameter or // not. - pm_refute_numbered_parameter(parser, name->start, name->end); + pm_refute_numbered_parameter(parser, (uint32_t) (name->start - parser->start), PM_TOKEN_LENGTH(name)); // Otherwise we'll fetch the constant id for the parameter name and check // whether it's already in the current scope. @@ -11047,7 +11062,7 @@ parser_lex(pm_parser_t *parser) { !(last_state & (PM_LEX_STATE_DOT | PM_LEX_STATE_FNAME)) && (type == PM_TOKEN_IDENTIFIER) && ((pm_parser_local_depth(parser, &parser->current) != -1) || - pm_token_is_numbered_parameter(parser->current.start, parser->current.end)) + pm_token_is_numbered_parameter(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current))) ) { lex_state_set(parser, PM_LEX_STATE_END | PM_LEX_STATE_LABEL); } @@ -12667,7 +12682,7 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p target->type = PM_GLOBAL_VARIABLE_TARGET_NODE; return target; case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(target->location.start, target->location.end)) { + if (pm_token_is_numbered_parameter(parser, U32(target->location.start - parser->start), PM_NODE_LENGTH(target))) { PM_PARSER_ERR_FORMAT(parser, target->location.start, target->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, target->location.start); pm_node_unreference(parser, target); } @@ -12861,7 +12876,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod uint32_t depth = local_read->depth; pm_scope_t *scope = pm_parser_scope_find(parser, depth); - if (pm_token_is_numbered_parameter(target->location.start, target->location.end)) { + if (pm_token_is_numbered_parameter(parser, U32(target->location.start - parser->start), PM_NODE_LENGTH(target))) { pm_diagnostic_id_t diag_id = (scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_FOUND) ? PM_ERR_EXPRESSION_NOT_WRITABLE_NUMBERED : PM_ERR_PARAMETER_NUMBERED_RESERVED; PM_PARSER_ERR_FORMAT(parser, target->location.start, target->location.end, diag_id, target->location.start); pm_node_unreference(parser, target); @@ -12929,13 +12944,13 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod .end = parser->start + call->message_loc.start + call->message_loc.length }; + pm_refute_numbered_parameter(parser, call->message_loc.start, call->message_loc.length); pm_parser_local_add_slice(parser, &call->message_loc, 0); pm_node_destroy(parser, target); pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, message.start, message.end); target = UP(pm_local_variable_write_node_create(parser, constant_id, 0, value, &message, operator)); - pm_refute_numbered_parameter(parser, message.start, message.end); return target; } @@ -14630,7 +14645,7 @@ parse_blocklike_parameters(pm_parser_t *parser, pm_node_t *parameters, const pm_ pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_OUTER_BLOCK); } else if (parser->current_scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_INNER) { pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_INNER_BLOCK); - } else if (pm_token_is_numbered_parameter(node->location.start, node->location.end)) { + } else if (pm_token_is_numbered_parameter(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node))) { numbered_parameter = MAX(numbered_parameter, (uint8_t) (node->location.start[1] - '0')); } else { assert(false && "unreachable"); @@ -15666,7 +15681,7 @@ static pm_node_t * parse_variable(pm_parser_t *parser) { pm_constant_id_t name_id = pm_parser_constant_id_token(parser, &parser->previous); int depth; - bool is_numbered_param = pm_token_is_numbered_parameter(parser->previous.start, parser->previous.end); + bool is_numbered_param = pm_token_is_numbered_parameter(parser, U32(parser->previous.start - parser->start), PM_TOKEN_LENGTH(&parser->previous)); if (!is_numbered_param && ((depth = pm_parser_local_depth_constant_id(parser, name_id)) != -1)) { return UP(pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, (uint32_t) depth, false)); @@ -15736,7 +15751,7 @@ parse_method_definition_name(pm_parser_t *parser) { parser_lex(parser); return parser->previous; case PM_TOKEN_IDENTIFIER: - pm_refute_numbered_parameter(parser, parser->current.start, parser->current.end); + pm_refute_numbered_parameter(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current)); parser_lex(parser); return parser->previous; case PM_CASE_OPERATOR: @@ -17866,7 +17881,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // purposes of warnings. assert(PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)); - if (pm_token_is_numbered_parameter(identifier.start, identifier.end)) { + if (pm_token_is_numbered_parameter(parser, U32(identifier.start - parser->start), PM_TOKEN_LENGTH(&identifier))) { pm_node_unreference(parser, node); } else { pm_local_variable_read_node_t *cast = (pm_local_variable_read_node_t *) node; @@ -18596,7 +18611,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b operator = parser->previous; name = parse_method_definition_name(parser); } else { - pm_refute_numbered_parameter(parser, parser->previous.start, parser->previous.end); + pm_refute_numbered_parameter(parser, U32(parser->previous.start - parser->start), PM_TOKEN_LENGTH(&parser->previous)); pm_parser_scope_push(parser, true); name = parser->previous; @@ -20528,7 +20543,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(node->location.start, node->location.end)) { + if (pm_token_is_numbered_parameter(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node))) { PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, node->location.start); pm_node_unreference(parser, node); } @@ -20549,12 +20564,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // receiver that could have been a local variable) then we // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_location_t message_loc = { - .start = parser->start + cast->message_loc.start, - .end = parser->start + cast->message_loc.start + cast->message_loc.length - }; - - pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); + pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length); pm_constant_id_t constant_id = pm_parser_local_add_slice(parser, &cast->message_loc, 1); parser_lex(parser); @@ -20665,7 +20675,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(node->location.start, node->location.end)) { + if (pm_token_is_numbered_parameter(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node))) { PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, node->location.start); pm_node_unreference(parser, node); } @@ -20686,12 +20696,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // receiver that could have been a local variable) then we // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_location_t message_loc = (pm_location_t) { - .start = parser->start + cast->message_loc.start, - .end = parser->start + cast->message_loc.start + cast->message_loc.length - }; - - pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); + pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length); pm_constant_id_t constant_id = pm_parser_local_add_slice(parser, &cast->message_loc, 1); parser_lex(parser); @@ -20812,7 +20817,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(node->location.start, node->location.end)) { + if (pm_token_is_numbered_parameter(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node))) { PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, node->location.start); pm_node_unreference(parser, node); } @@ -20834,12 +20839,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // receiver that could have been a local variable) then we // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_location_t message_loc = (pm_location_t) { - .start = parser->start + cast->message_loc.start, - .end = parser->start + cast->message_loc.start + cast->message_loc.length - }; - - pm_refute_numbered_parameter(parser, message_loc.start, message_loc.end); + pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length); pm_constant_id_t constant_id = pm_parser_local_add_slice(parser, &cast->message_loc, 1); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1)); pm_node_t *result = UP(pm_local_variable_operator_write_node_create(parser, UP(cast), &token, value, constant_id, 0)); From 750b81947ec1be0054dd5ab7359fc9bb9231006a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 2 Dec 2025 22:08:46 -0500 Subject: [PATCH 17/22] Switch error generation to use uint32 --- src/prism.c | 83 +++++++++++++++++++++++++++-------------------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/src/prism.c b/src/prism.c index 0e2aa3c0fd..aeb2b2c8c4 100644 --- a/src/prism.c +++ b/src/prism.c @@ -428,8 +428,8 @@ debug_lex_state_set(pm_parser_t *parser, pm_lex_state_t state, char const * call * Append an error to the list of errors on the parser. */ static inline void -pm_parser_err(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { - pm_diagnostic_list_append(&parser->error_list, (uint32_t) (start - parser->start), (uint32_t) (end - start), diag_id); +pm_parser_err(pm_parser_t *parser, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id) { + pm_diagnostic_list_append(&parser->error_list, start, length, diag_id); } /** @@ -450,7 +450,7 @@ pm_parser_err(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_ */ static inline void pm_parser_err_current(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, parser->current.start, parser->current.end, diag_id); + pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current), diag_id); } /** @@ -466,7 +466,7 @@ pm_parser_err_current(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { */ static inline void pm_parser_err_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, node->location.start, node->location.end, diag_id); + pm_parser_err(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node), diag_id); } /** @@ -489,7 +489,7 @@ pm_parser_err_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id_ */ static inline void pm_parser_err_previous(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, parser->previous.start, parser->previous.end, diag_id); + pm_parser_err(parser, U32(parser->previous.start - parser->start), PM_TOKEN_LENGTH(&parser->previous), diag_id); } /** @@ -498,7 +498,7 @@ pm_parser_err_previous(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { */ static inline void pm_parser_err_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, token->start, token->end, diag_id); + pm_parser_err(parser, U32(token->start - parser->start), PM_TOKEN_LENGTH(token), diag_id); } /** @@ -5559,7 +5559,7 @@ pm_numbered_reference_read_node_number(pm_parser_t *parser, const pm_token_t *to unsigned long value = strtoul(digits, &endptr, 10); if ((digits == endptr) || (*endptr != '\0')) { - pm_parser_err(parser, start, end, PM_ERR_INVALID_NUMBER_DECIMAL); + pm_parser_err(parser, U32(start - parser->start), U32(length), PM_ERR_INVALID_NUMBER_DECIMAL); value = 0; } @@ -6376,7 +6376,7 @@ parse_symbol_encoding_validate_utf8(pm_parser_t *parser, const pm_token_t *locat size_t width = pm_encoding_utf_8_char_width(cursor, end - cursor); if (width == 0) { - pm_parser_err(parser, location->start, location->end, PM_ERR_INVALID_SYMBOL); + pm_parser_err(parser, U32(location->start - parser->start), PM_TOKEN_LENGTH(location), PM_ERR_INVALID_SYMBOL); break; } @@ -6396,7 +6396,7 @@ parse_symbol_encoding_validate_other(pm_parser_t *parser, const pm_token_t *loca size_t width = encoding->char_width(cursor, end - cursor); if (width == 0) { - pm_parser_err(parser, location->start, location->end, PM_ERR_INVALID_SYMBOL); + pm_parser_err(parser, U32(location->start - parser->start), PM_TOKEN_LENGTH(location), PM_ERR_INVALID_SYMBOL); break; } @@ -7425,7 +7425,7 @@ parser_lex_magic_comment_encoding(pm_parser_t *parser) { // issue because we didn't understand the encoding that the user was // trying to use. In this case we'll keep using the default encoding but // add an error to the parser to indicate an unsuccessful parse. - pm_parser_err(parser, value_start, cursor, PM_ERR_INVALID_ENCODING_MAGIC_COMMENT); + pm_parser_err(parser, U32(value_start - parser->start), U32(cursor - value_start), PM_ERR_INVALID_ENCODING_MAGIC_COMMENT); } } @@ -7912,7 +7912,7 @@ static inline void pm_strspn_number_validate(pm_parser_t *parser, const uint8_t *string, size_t length, const uint8_t *invalid) { if (invalid != NULL) { pm_diagnostic_id_t diag_id = (invalid == (string + length - 1)) ? PM_ERR_INVALID_NUMBER_UNDERSCORE_TRAILING : PM_ERR_INVALID_NUMBER_UNDERSCORE_INNER; - pm_parser_err(parser, invalid, invalid + 1, diag_id); + pm_parser_err(parser, U32(invalid - parser->start), 1, diag_id); } } @@ -8097,7 +8097,7 @@ lex_numeric_prefix(pm_parser_t *parser, bool* seen_e) { const uint8_t *fraction_start = parser->current.end; const uint8_t *fraction_end = parser->current.end + 2; fraction_end += pm_strspn_decimal_digit(fraction_end, parser->end - fraction_end); - pm_parser_err(parser, fraction_start, fraction_end, PM_ERR_INVALID_NUMBER_FRACTION); + pm_parser_err(parser, U32(fraction_start - parser->start), U32(fraction_end - fraction_start), PM_ERR_INVALID_NUMBER_FRACTION); } return type; @@ -8616,7 +8616,7 @@ escape_unicode(pm_parser_t *parser, const uint8_t *string, size_t length) { // Here we're going to verify that the value is actually a valid Unicode // codepoint and not a surrogate pair. if (value >= 0xD800 && value <= 0xDFFF) { - pm_parser_err(parser, string, string + length, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, U32(string - parser->start), U32(length), PM_ERR_ESCAPE_INVALID_UNICODE); return 0xFFFD; } @@ -8650,7 +8650,7 @@ escape_write_unicode(pm_parser_t *parser, pm_buffer_t *buffer, const uint8_t fla } if (!pm_buffer_append_unicode_codepoint(buffer, value)) { - pm_parser_err(parser, start, end, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, U32(start - parser->start), U32(end - start), PM_ERR_ESCAPE_INVALID_UNICODE); pm_buffer_append_byte(buffer, 0xEF); pm_buffer_append_byte(buffer, 0xBF); pm_buffer_append_byte(buffer, 0xBD); @@ -8893,7 +8893,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre if (hexadecimal_length > 6) { // \u{nnnn} character literal allows only 1-6 hexadecimal digits - pm_parser_err(parser, unicode_start, unicode_start + hexadecimal_length, PM_ERR_ESCAPE_INVALID_UNICODE_LONG); + pm_parser_err(parser, U32(unicode_start - parser->start), U32(hexadecimal_length), PM_ERR_ESCAPE_INVALID_UNICODE_LONG); } else if (hexadecimal_length == 0) { // there are not hexadecimal characters @@ -8903,8 +8903,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre // error instead of us. pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); } else { - pm_parser_err(parser, parser->current.end, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE); - pm_parser_err(parser, parser->current.end, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_TERM); + pm_parser_err(parser, U32(parser->current.end - parser->start), 0, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, U32(parser->current.end - parser->start), 0, PM_ERR_ESCAPE_INVALID_UNICODE_TERM); } return; @@ -8925,7 +8925,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre // ?\u{nnnn} character literal should contain only one codepoint // and cannot be like ?\u{nnnn mmmm}. if (flags & PM_ESCAPE_FLAG_SINGLE && codepoints_count > 1) { - pm_parser_err(parser, extra_codepoints_start, parser->current.end - 1, PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL); + pm_parser_err(parser, U32(extra_codepoints_start - parser->start), U32(parser->current.end - 1 - extra_codepoints_start), PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL); } if (parser->current.end == parser->end) { @@ -8939,7 +8939,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre // instead of us. pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); } else { - pm_parser_err(parser, unicode_codepoints_start, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_TERM); + pm_parser_err(parser, U32(unicode_codepoints_start - parser->start), U32(parser->current.end - unicode_codepoints_start), PM_ERR_ESCAPE_INVALID_UNICODE_TERM); } } @@ -9003,7 +9003,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; if (match(parser, 'u') || match(parser, 'U')) { - pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); return; } @@ -9039,7 +9039,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre if (peek(parser) != '-') { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_CONTROL); + pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_CONTROL); return; } @@ -9060,7 +9060,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; if (match(parser, 'u') || match(parser, 'U')) { - pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); return; } @@ -9079,7 +9079,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre default: { if (!char_is_ascii_printable(peeked)) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_CONTROL); + pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_CONTROL); return; } @@ -9097,7 +9097,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre if (peek(parser) != '-') { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_META); + pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); return; } @@ -9113,7 +9113,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; if (match(parser, 'u') || match(parser, 'U')) { - pm_parser_err(parser, parser->current.start, parser->current.end, PM_ERR_INVALID_ESCAPE_CHARACTER); + pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); return; } @@ -9132,7 +9132,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre default: if (!char_is_ascii_printable(peeked)) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_META); + pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); return; } @@ -9152,7 +9152,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre default: { if ((flags & (PM_ESCAPE_FLAG_CONTROL | PM_ESCAPE_FLAG_META)) && !char_is_ascii_printable(peeked)) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, parser->current.start, parser->current.end + width, PM_ERR_ESCAPE_INVALID_META); + pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); return; } if (parser->current.end < parser->end) { @@ -10355,7 +10355,7 @@ parser_lex(pm_parser_t *parser) { bool ident_error = false; if (quote != PM_HEREDOC_QUOTE_NONE && !match(parser, (uint8_t) quote)) { - pm_parser_err(parser, ident_start, ident_start + ident_length, PM_ERR_HEREDOC_IDENTIFIER); + pm_parser_err(parser, U32(ident_start - parser->start), U32(ident_length), PM_ERR_HEREDOC_IDENTIFIER); ident_error = true; } @@ -12366,7 +12366,7 @@ expect1(pm_parser_t *parser, pm_token_type_t type, pm_diagnostic_id_t diag_id) { if (accept1(parser, type)) return; const uint8_t *location = parser->previous.end; - pm_parser_err(parser, location, location, diag_id); + pm_parser_err(parser, U32(location - parser->start), 0, diag_id); parser->previous.start = location; parser->previous.type = 0; @@ -12381,7 +12381,7 @@ expect2(pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_di if (accept2(parser, type1, type2)) return; const uint8_t *location = parser->previous.end; - pm_parser_err(parser, location, location, diag_id); + pm_parser_err(parser, U32(location - parser->start), 0, diag_id); parser->previous.start = location; parser->previous.type = 0; @@ -13496,7 +13496,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for pm_node_t *expression = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT, (uint16_t) (depth + 1)); if (parsed_bare_hash) { - pm_parser_err(parser, operator.start, expression->location.end, PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT); + pm_parser_err(parser, U32(operator.start - parser->start), U32(expression->location.end - operator.start), PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT); } argument = UP(pm_splat_node_create(parser, &operator, expression)); @@ -13521,8 +13521,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for // ... operator. if (PM_NODE_TYPE_P(right, PM_RANGE_NODE)) { pm_range_node_t *range = (pm_range_node_t *) right; - const uint8_t *start = parser->start + range->operator_loc.start; - pm_parser_err(parser, start, start + range->operator_loc.length, PM_ERR_UNEXPECTED_RANGE_OPERATOR); + pm_parser_err(parser, range->operator_loc.start, range->operator_loc.length, PM_ERR_UNEXPECTED_RANGE_OPERATOR); } argument = UP(pm_range_node_create(parser, NULL, &operator, right)); @@ -13968,7 +13967,7 @@ parse_parameters( local.end -= 1; if (parser->encoding_changed ? parser->encoding->isupper_char(local.start, local.end - local.start) : pm_encoding_utf_8_isupper_char(local.start, local.end - local.start)) { - pm_parser_err(parser, local.start, local.end, PM_ERR_ARGUMENT_FORMAL_CONSTANT); + pm_parser_err(parser, U32(local.start - parser->start), PM_TOKEN_LENGTH(&local), PM_ERR_ARGUMENT_FORMAL_CONSTANT); } else if (local.end[-1] == '!' || local.end[-1] == '?') { PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, local, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE); } @@ -15968,7 +15967,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 if (!accept1(parser, PM_TOKEN_STRING_END)) { const uint8_t *location = parser->previous.end; if (location > parser->start && location[-1] == '\n') location--; - pm_parser_err(parser, location, location, PM_ERR_STRING_LITERAL_EOF); + pm_parser_err(parser, U32(location - parser->start), 0, PM_ERR_STRING_LITERAL_EOF); parser->previous.start = parser->previous.end; parser->previous.type = 0; @@ -16087,7 +16086,7 @@ parse_pattern_capture(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_c if (*location->start == '_') return; if (pm_constant_id_list_includes(captures, capture)) { - pm_parser_err(parser, location->start, location->end, PM_ERR_PATTERN_CAPTURE_DUPLICATE); + pm_parser_err(parser, U32(location->start - parser->start), PM_TOKEN_LENGTH(location), PM_ERR_PATTERN_CAPTURE_DUPLICATE); } else { pm_constant_id_list_append(captures, capture); } @@ -16327,7 +16326,7 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca if (pm_slice_is_valid_local(parser, value_loc->start, value_loc->end)) { depth = pm_parser_local_depth_constant_id(parser, constant_id); } else { - pm_parser_err(parser, key->base.location.start, key->base.location.end, PM_ERR_PATTERN_HASH_KEY_LOCALS); + pm_parser_err(parser, U32(key->base.location.start - parser->start), PM_NODE_LENGTH(key), PM_ERR_PATTERN_HASH_KEY_LOCALS); if ((value_loc->end > value_loc->start) && ((value_loc->end[-1] == '!') || (value_loc->end[-1] == '?'))) { PM_PARSER_ERR_LOCATION_FORMAT(parser, value_loc, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (value_loc->end - value_loc->start), (const char *) value_loc->start); @@ -16759,9 +16758,11 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm static bool parse_pattern_alternation_error_each(const pm_node_t *node, void *data) { switch (PM_NODE_TYPE(node)) { - case PM_LOCAL_VARIABLE_TARGET_NODE: - pm_parser_err((pm_parser_t *) data, node->location.start, node->location.end, PM_ERR_PATTERN_CAPTURE_IN_ALTERNATIVE); + case PM_LOCAL_VARIABLE_TARGET_NODE: { + pm_parser_t *parser = (pm_parser_t *) data; + pm_parser_err(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node), PM_ERR_PATTERN_CAPTURE_IN_ALTERNATIVE); return false; + } default: return true; } @@ -19058,7 +19059,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // syntax. if (!accepts_command_call && !match1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES)) { - pm_parser_err(parser, parser->previous.end, parser->previous.end + 1, PM_ERR_EXPECT_LPAREN_AFTER_NOT_LPAREN); + pm_parser_err(parser, U32(parser->previous.end - parser->start), 1, PM_ERR_EXPECT_LPAREN_AFTER_NOT_LPAREN); } else { accept1(parser, PM_TOKEN_NEWLINE); pm_parser_err_current(parser, PM_ERR_EXPECT_LPAREN_AFTER_NOT_OTHER); @@ -21999,7 +22000,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm parser->previous = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor }; parser->current = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor }; } else { - pm_parser_err(parser, parser->start, parser->start, PM_ERR_SCRIPT_NOT_FOUND); + pm_parser_err(parser, 0, 0, PM_ERR_SCRIPT_NOT_FOUND); pm_newline_list_clear(&parser->newline_list); } } From b3d12c3dfa27ede99f2bb4ea9143d8ce0136cd27 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 2 Dec 2025 22:20:30 -0500 Subject: [PATCH 18/22] Switch warning generation over to using slices --- src/prism.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/prism.c b/src/prism.c index aeb2b2c8c4..ff37e07e36 100644 --- a/src/prism.c +++ b/src/prism.c @@ -519,8 +519,8 @@ pm_parser_err_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_ * Append a warning to the list of warnings on the parser. */ static inline void -pm_parser_warn(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { - pm_diagnostic_list_append(&parser->warning_list, (uint32_t) (start - parser->start), (uint32_t) (end - start), diag_id); +pm_parser_warn(pm_parser_t *parser, uint32_t start, uint32_t length, pm_diagnostic_id_t diag_id) { + pm_diagnostic_list_append(&parser->warning_list, start, length, diag_id); } /** @@ -529,7 +529,7 @@ pm_parser_warn(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, pm */ static inline void pm_parser_warn_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_id_t diag_id) { - pm_parser_warn(parser, token->start, token->end, diag_id); + pm_parser_warn(parser, U32(token->start - parser->start), PM_TOKEN_LENGTH(token), diag_id); } /** @@ -538,7 +538,7 @@ pm_parser_warn_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic */ static inline void pm_parser_warn_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id_t diag_id) { - pm_parser_warn(parser, node->location.start, node->location.end, diag_id); + pm_parser_warn(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node), diag_id); } /** @@ -9772,7 +9772,7 @@ parser_lex(pm_parser_t *parser) { if (match_eol_offset(parser, 1)) { chomping = false; } else { - pm_parser_warn(parser, parser->current.end, parser->current.end + 1, PM_WARN_UNEXPECTED_CARRIAGE_RETURN); + pm_parser_warn(parser, U32(parser->current.end - parser->start), 1, PM_WARN_UNEXPECTED_CARRIAGE_RETURN); parser->current.end++; space_seen = true; } @@ -21719,7 +21719,7 @@ pm_strnstr(const char *big, const char *little, size_t big_length) { static void pm_parser_warn_shebang_carriage_return(pm_parser_t *parser, const uint8_t *start, size_t length) { if (length > 2 && start[length - 2] == '\r' && start[length - 1] == '\n') { - pm_parser_warn(parser, start, start + length, PM_WARN_SHEBANG_CARRIAGE_RETURN); + pm_parser_warn(parser, U32(start - parser->start), U32(length), PM_WARN_SHEBANG_CARRIAGE_RETURN); } } #endif From f662521773a306d1d451e1195ba6213f619d45de Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 2 Dec 2025 23:00:17 -0500 Subject: [PATCH 19/22] Mask setting start and end on a node location --- src/prism.c | 188 +++++++++++++++++++++++++++------------------------- 1 file changed, 96 insertions(+), 92 deletions(-) diff --git a/src/prism.c b/src/prism.c index ff37e07e36..7ef4df200c 100644 --- a/src/prism.c +++ b/src/prism.c @@ -35,6 +35,11 @@ pm_version(void) { #define PM_NODE_END(node_) (UP(node_)->location.end) #define PM_NODE_LENGTH(node_) U32(PM_NODE_END(node_) - PM_NODE_START(node_)) +#define PM_NODE_START_SET_NODE(left_, right_) (PM_NODE_START(left_) = PM_NODE_START(right_)) +#define PM_NODE_START_SET_TOKEN(node_, token_) (PM_NODE_START(node_) = PM_TOKEN_START(token_)) +#define PM_NODE_END_SET_NODE(left_, right_) (PM_NODE_END(left_) = PM_NODE_END(right_)) +#define PM_NODE_END_SET_TOKEN(node_, token_) (PM_NODE_END(node_) = PM_TOKEN_END(token_)) + #define PM_LOCATION_TOKEN_VALUE(token_) ((pm_location_t) { .start = PM_TOKEN_START(token_), .end = PM_TOKEN_END(token_) }) #define TOK2SLICE(parser_, token_) ((pm_slice_t) { .start = (uint32_t) ((token_)->start - (parser_)->start), .length = (uint32_t) ((token_)->end - (token_)->start) }) @@ -2074,10 +2079,10 @@ pm_arguments_node_size(pm_arguments_node_t *node) { static void pm_arguments_node_arguments_append(pm_arguments_node_t *node, pm_node_t *argument) { if (pm_arguments_node_size(node) == 0) { - node->base.location.start = argument->location.start; + PM_NODE_START_SET_NODE(node, argument); } - node->base.location.end = argument->location.end; + PM_NODE_END_SET_NODE(node, argument); pm_node_list_append(&node->arguments, argument); if (PM_NODE_TYPE_P(argument, PM_SPLAT_NODE)) { @@ -2121,11 +2126,11 @@ pm_array_node_create(pm_parser_t *parser, const pm_token_t *opening) { static inline void pm_array_node_elements_append(pm_array_node_t *node, pm_node_t *element) { if (!node->elements.size && !node->opening_loc.length) { - node->base.location.start = element->location.start; + PM_NODE_START_SET_NODE(node, element); } pm_node_list_append(&node->elements, element); - node->base.location.end = element->location.end; + PM_NODE_END_SET_NODE(node, element); // If the element is not a static literal, then the array is not a static // literal. Turn that flag off. @@ -2144,7 +2149,7 @@ pm_array_node_elements_append(pm_array_node_t *node, pm_node_t *element) { static void pm_array_node_close_set(const pm_parser_t *parser, pm_array_node_t *node, const pm_token_t *closing) { assert(closing->type == PM_TOKEN_BRACKET_RIGHT || closing->type == PM_TOKEN_STRING_END || closing->type == 0); - node->base.location.end = closing->end; + PM_NODE_END_SET_TOKEN(node, closing); node->closing_loc = TOK2SLICE(parser, closing); } @@ -2359,9 +2364,9 @@ static void pm_begin_node_rescue_clause_set(pm_begin_node_t *node, pm_rescue_node_t *rescue_clause) { // If the begin keyword doesn't exist, we set the start on the begin_node if (node->begin_keyword_loc.length == 0) { - node->base.location.start = rescue_clause->base.location.start; + PM_NODE_START_SET_NODE(node, rescue_clause); } - node->base.location.end = rescue_clause->base.location.end; + PM_NODE_END_SET_NODE(node, rescue_clause); node->rescue_clause = rescue_clause; } @@ -2370,7 +2375,7 @@ pm_begin_node_rescue_clause_set(pm_begin_node_t *node, pm_rescue_node_t *rescue_ */ static void pm_begin_node_else_clause_set(pm_begin_node_t *node, pm_else_node_t *else_clause) { - node->base.location.end = else_clause->base.location.end; + PM_NODE_END_SET_NODE(node, else_clause); node->else_clause = else_clause; } @@ -2379,7 +2384,7 @@ pm_begin_node_else_clause_set(pm_begin_node_t *node, pm_else_node_t *else_clause */ static void pm_begin_node_ensure_clause_set(pm_begin_node_t *node, pm_ensure_node_t *ensure_clause) { - node->base.location.end = ensure_clause->base.location.end; + PM_NODE_END_SET_NODE(node, ensure_clause); node->ensure_clause = ensure_clause; } @@ -2389,7 +2394,7 @@ pm_begin_node_ensure_clause_set(pm_begin_node_t *node, pm_ensure_node_t *ensure_ static void pm_begin_node_end_keyword_set(const pm_parser_t *parser, pm_begin_node_t *node, const pm_token_t *end_keyword) { assert(end_keyword->type == PM_TOKEN_KEYWORD_END || end_keyword->type == 0); - node->base.location.end = end_keyword->end; + PM_NODE_END_SET_TOKEN(node, end_keyword); node->end_keyword_loc = TOK2SLICE(parser, end_keyword); } @@ -2497,7 +2502,7 @@ pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *param static void pm_block_parameters_node_closing_set(const pm_parser_t *parser, pm_block_parameters_node_t *node, const pm_token_t *closing) { assert(closing->type == PM_TOKEN_PIPE || closing->type == PM_TOKEN_PARENTHESIS_RIGHT || closing->type == 0); - node->base.location.end = closing->end; + PM_NODE_END_SET_TOKEN(node, closing); node->closing_loc = TOK2SLICE(parser, closing); } @@ -2523,8 +2528,11 @@ static void pm_block_parameters_node_append_local(pm_block_parameters_node_t *node, const pm_block_local_variable_node_t *local) { pm_node_list_append(&node->locals, UP(local)); - if (node->base.location.start == NULL) node->base.location.start = local->base.location.start; - node->base.location.end = local->base.location.end; + if (PM_NODE_START(node) == NULL) { + PM_NODE_START_SET_NODE(node, local); + } + + PM_NODE_END_SET_NODE(node, local); } /** @@ -2606,7 +2614,7 @@ pm_call_node_aref_create(pm_parser_t *parser, pm_node_t *receiver, pm_arguments_ pm_call_node_t *node = pm_call_node_create(parser, flags); - node->base.location.start = receiver->location.start; + PM_NODE_START_SET_NODE(node, receiver); node->base.location.end = pm_arguments_end(parser, arguments); node->receiver = receiver; @@ -2657,7 +2665,7 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); - node->base.location.start = receiver->location.start; + PM_NODE_START_SET_NODE(node, receiver); const uint8_t *end = pm_arguments_end(parser, arguments); if (end == NULL) { end = message->end; @@ -2708,7 +2716,7 @@ static pm_call_node_t * pm_call_node_fcall_create(pm_parser_t *parser, pm_token_t *message, pm_arguments_t *arguments) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location.start = message->start; + PM_NODE_START_SET_TOKEN(node, message); node->base.location.end = pm_arguments_end(parser, arguments); node->message_loc = TOK2SLICE(parser, message); @@ -2745,13 +2753,13 @@ pm_call_node_not_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *me if (receiver != NULL) pm_conditional_predicate(parser, receiver, PM_CONDITIONAL_PREDICATE_TYPE_NOT); pm_call_node_t *node = pm_call_node_create(parser, receiver == NULL ? 0 : pm_call_node_ignore_visibility_flag(receiver)); - - node->base.location.start = message->start; + + PM_NODE_START_SET_TOKEN(node, message); if (arguments->closing_loc.length > 0) { node->base.location.end = parser->start + arguments->closing_loc.start + arguments->closing_loc.length; } else { assert(receiver != NULL); - node->base.location.end = receiver->location.end; + PM_NODE_END_SET_NODE(node, receiver); } node->receiver = receiver; @@ -2773,7 +2781,7 @@ pm_call_node_shorthand_create(pm_parser_t *parser, pm_node_t *receiver, pm_token pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); - node->base.location.start = receiver->location.start; + PM_NODE_START_SET_NODE(node, receiver); node->base.location.end = pm_arguments_end(parser, arguments); node->receiver = receiver; @@ -2800,8 +2808,8 @@ pm_call_node_unary_create(pm_parser_t *parser, pm_token_t *operator, pm_node_t * pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); - node->base.location.start = operator->start; - node->base.location.end = receiver->location.end; + PM_NODE_START_SET_TOKEN(node, operator); + PM_NODE_END_SET_NODE(node, receiver); node->receiver = receiver; node->message_loc = TOK2SLICE(parser, operator); @@ -3164,7 +3172,7 @@ pm_case_node_condition_append(pm_case_node_t *node, pm_node_t *condition) { assert(PM_NODE_TYPE_P(condition, PM_WHEN_NODE)); pm_node_list_append(&node->conditions, condition); - node->base.location.end = condition->location.end; + PM_NODE_END_SET_NODE(node, condition); } /** @@ -3173,7 +3181,7 @@ pm_case_node_condition_append(pm_case_node_t *node, pm_node_t *condition) { static void pm_case_node_else_clause_set(pm_case_node_t *node, pm_else_node_t *else_clause) { node->else_clause = else_clause; - node->base.location.end = else_clause->base.location.end; + PM_NODE_END_SET_NODE(node, else_clause); } /** @@ -3181,7 +3189,7 @@ pm_case_node_else_clause_set(pm_case_node_t *node, pm_else_node_t *else_clause) */ static void pm_case_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_node_t *node, const pm_token_t *end_keyword) { - node->base.location.end = end_keyword->end; + PM_NODE_END_SET_TOKEN(node, end_keyword); node->end_keyword_loc = TOK2SLICE(parser, end_keyword); } @@ -3212,7 +3220,7 @@ pm_case_match_node_condition_append(pm_case_match_node_t *node, pm_node_t *condi assert(PM_NODE_TYPE_P(condition, PM_IN_NODE)); pm_node_list_append(&node->conditions, condition); - node->base.location.end = condition->location.end; + PM_NODE_END_SET_NODE(node, condition); } /** @@ -3221,7 +3229,7 @@ pm_case_match_node_condition_append(pm_case_match_node_t *node, pm_node_t *condi static void pm_case_match_node_else_clause_set(pm_case_match_node_t *node, pm_else_node_t *else_clause) { node->else_clause = else_clause; - node->base.location.end = else_clause->base.location.end; + PM_NODE_END_SET_NODE(node, else_clause); } /** @@ -3229,7 +3237,7 @@ pm_case_match_node_else_clause_set(pm_case_match_node_t *node, pm_else_node_t *e */ static void pm_case_match_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_match_node_t *node, const pm_token_t *end_keyword) { - node->base.location.end = end_keyword->end; + PM_NODE_END_SET_TOKEN(node, end_keyword); node->end_keyword_loc = TOK2SLICE(parser, end_keyword); } @@ -4310,7 +4318,7 @@ pm_hash_node_elements_append(pm_hash_node_t *hash, pm_node_t *element) { static inline void pm_hash_node_closing_loc_set(const pm_parser_t *parser, pm_hash_node_t *hash, pm_token_t *token) { - hash->base.location.end = token->end; + PM_NODE_END_SET_TOKEN(hash, token); hash->closing_loc = TOK2SLICE(parser, token); } @@ -4410,13 +4418,13 @@ pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_to static inline void pm_if_node_end_keyword_loc_set(const pm_parser_t *parser, pm_if_node_t *node, const pm_token_t *keyword) { - node->base.location.end = keyword->end; + PM_NODE_END_SET_TOKEN(node, keyword); node->end_keyword_loc = TOK2SLICE(parser, keyword); } static inline void pm_else_node_end_keyword_loc_set(const pm_parser_t *parser, pm_else_node_t *node, const pm_token_t *keyword) { - node->base.location.end = keyword->end; + PM_NODE_END_SET_TOKEN(node, keyword); node->end_keyword_loc = TOK2SLICE(parser, keyword); } @@ -4731,11 +4739,11 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok static inline void pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expression_node_t *node, pm_node_t *part) { - if (node->base.location.start > part->location.start) { - node->base.location.start = part->location.start; + if (PM_NODE_START(node) > PM_NODE_START(part)) { + PM_NODE_START_SET_NODE(node, part); } if (node->base.location.end < part->location.end) { - node->base.location.end = part->location.end; + PM_NODE_END_SET_NODE(node, part); } pm_interpolated_node_append(UP(node), &node->parts, part); @@ -4744,7 +4752,7 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio static inline void pm_interpolated_regular_expression_node_closing_set(pm_parser_t *parser, pm_interpolated_regular_expression_node_t *node, const pm_token_t *closing) { node->closing_loc = TOK2SLICE(parser, closing); - node->base.location.end = closing->end; + PM_NODE_END_SET_TOKEN(node, closing); pm_node_flag_set(UP(node), pm_regular_expression_flags_create(parser, closing)); } @@ -4780,7 +4788,7 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ node->base.flags = (pm_node_flags_t) ((FL(node) | PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE) & ~PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); if (node->parts.size == 0 && node->opening_loc.length == 0) { - node->base.location.start = part->location.start; + PM_NODE_START_SET_NODE(node, part); } node->base.location.end = MAX(node->base.location.end, part->location.end); @@ -4901,13 +4909,13 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin static void pm_interpolated_string_node_closing_set(const pm_parser_t *parser, pm_interpolated_string_node_t *node, const pm_token_t *closing) { node->closing_loc = TOK2SLICE(parser, closing); - node->base.location.end = closing->end; + PM_NODE_END_SET_TOKEN(node, closing); } static void pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_t *part) { if (node->parts.size == 0 && node->opening_loc.length == 0) { - node->base.location.start = part->location.start; + PM_NODE_START_SET_NODE(node, part); } pm_interpolated_node_append(UP(node), &node->parts, part); @@ -4917,7 +4925,7 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ static void pm_interpolated_symbol_node_closing_loc_set(const pm_parser_t *parser, pm_interpolated_symbol_node_t *node, const pm_token_t *closing) { node->closing_loc = TOK2SLICE(parser, closing); - node->base.location.end = closing->end; + PM_NODE_END_SET_TOKEN(node, closing); } /** @@ -4964,13 +4972,13 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi static inline void pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_node_t *part) { pm_interpolated_node_append(UP(node), &node->parts, part); - node->base.location.end = part->location.end; + PM_NODE_END_SET_NODE(node, part); } static inline void pm_interpolated_xstring_node_closing_set(const pm_parser_t *parser, pm_interpolated_x_string_node_t *node, const pm_token_t *closing) { node->closing_loc = TOK2SLICE(parser, closing); - node->base.location.end = closing->end; + PM_NODE_END_SET_TOKEN(node, closing); } /** @@ -5028,10 +5036,10 @@ pm_keyword_hash_node_elements_append(pm_keyword_hash_node_t *hash, pm_node_t *el } pm_node_list_append(&hash->elements, element); - if (hash->base.location.start == NULL) { - hash->base.location.start = element->location.start; + if (PM_NODE_START(hash) == NULL) { + PM_NODE_START_SET_NODE(hash, element); } - hash->base.location.end = element->location.end; + PM_NODE_END_SET_NODE(hash, element); } /** @@ -5405,11 +5413,11 @@ pm_multi_target_node_targets_append(pm_parser_t *parser, pm_multi_target_node_t } if (node->base.location.start == NULL || (node->base.location.start > target->location.start)) { - node->base.location.start = target->location.start; + PM_NODE_START_SET_NODE(node, target); } if (node->base.location.end == NULL || (node->base.location.end < target->location.end)) { - node->base.location.end = target->location.end; + PM_NODE_END_SET_NODE(node, target); } } @@ -5418,7 +5426,7 @@ pm_multi_target_node_targets_append(pm_parser_t *parser, pm_multi_target_node_t */ static void pm_multi_target_node_opening_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *lparen) { - node->base.location.start = lparen->start; + PM_NODE_START_SET_TOKEN(node, lparen); node->lparen_loc = TOK2SLICE(parser, lparen); } @@ -5427,7 +5435,7 @@ pm_multi_target_node_opening_set(const pm_parser_t *parser, pm_multi_target_node */ static void pm_multi_target_node_closing_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *rparen) { - node->base.location.end = rparen->end; + PM_NODE_END_SET_TOKEN(node, rparen); node->rparen_loc = TOK2SLICE(parser, rparen); } @@ -5654,16 +5662,12 @@ pm_parameters_node_create(pm_parser_t *parser) { */ static void pm_parameters_node_location_set(pm_parameters_node_t *params, pm_node_t *param) { - if (params->base.location.start == NULL) { - params->base.location.start = param->location.start; - } else { - params->base.location.start = params->base.location.start < param->location.start ? params->base.location.start : param->location.start; + if ((params->base.location.start == NULL) || PM_NODE_START(params) > PM_NODE_START(param)) { + PM_NODE_START_SET_NODE(params, param); } - if (params->base.location.end == NULL) { - params->base.location.end = param->location.end; - } else { - params->base.location.end = params->base.location.end > param->location.end ? params->base.location.end : param->location.end; + if ((params->base.location.end == NULL) || (params->base.location.end < param->location.end)) { + PM_NODE_END_SET_NODE(params, param); } } @@ -5978,7 +5982,7 @@ pm_rescue_node_operator_set(const pm_parser_t *parser, pm_rescue_node_t *node, c static void pm_rescue_node_reference_set(pm_rescue_node_t *node, pm_node_t *reference) { node->reference = reference; - node->base.location.end = reference->location.end; + PM_NODE_END_SET_NODE(node, reference); } /** @@ -5988,7 +5992,7 @@ static void pm_rescue_node_statements_set(pm_rescue_node_t *node, pm_statements_node_t *statements) { node->statements = statements; if (pm_statements_node_body_length(statements) > 0) { - node->base.location.end = statements->base.location.end; + PM_NODE_END_SET_NODE(node, statements); } } @@ -5998,7 +6002,7 @@ pm_rescue_node_statements_set(pm_rescue_node_t *node, pm_statements_node_t *stat static void pm_rescue_node_subsequent_set(pm_rescue_node_t *node, pm_rescue_node_t *subsequent) { node->subsequent = subsequent; - node->base.location.end = subsequent->base.location.end; + PM_NODE_END_SET_NODE(node, subsequent); } /** @@ -6007,7 +6011,7 @@ pm_rescue_node_subsequent_set(pm_rescue_node_t *node, pm_rescue_node_t *subseque static void pm_rescue_node_exceptions_append(pm_rescue_node_t *node, pm_node_t *exception) { pm_node_list_append(&node->exceptions, exception); - node->base.location.end = exception->location.end; + PM_NODE_END_SET_NODE(node, exception); } /** @@ -6230,12 +6234,12 @@ pm_statements_node_location_set(pm_statements_node_t *node, const uint8_t *start */ static inline void pm_statements_node_body_update(pm_statements_node_t *node, pm_node_t *statement) { - if (pm_statements_node_body_length(node) == 0 || statement->location.start < node->base.location.start) { - node->base.location.start = statement->location.start; + if (pm_statements_node_body_length(node) == 0 || PM_NODE_START(statement) < PM_NODE_START(node)) { + PM_NODE_START_SET_NODE(node, statement); } - if (statement->location.end > node->base.location.end) { - node->base.location.end = statement->location.end; + if (PM_NODE_END(statement) > PM_NODE_END(node)) { + PM_NODE_END_SET_NODE(node, statement); } } @@ -6780,7 +6784,7 @@ pm_undef_node_create(pm_parser_t *parser, const pm_token_t *token) { */ static void pm_undef_node_append(pm_undef_node_t *node, pm_node_t *name) { - node->base.location.end = name->location.end; + PM_NODE_END_SET_NODE(node, name); pm_node_list_append(&node->names, name); } @@ -6834,7 +6838,7 @@ pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const static inline void pm_unless_node_end_keyword_loc_set(const pm_parser_t *parser, pm_unless_node_t *node, const pm_token_t *end_keyword) { node->end_keyword_loc = TOK2SLICE(parser, end_keyword); - node->base.location.end = end_keyword->end; + PM_NODE_END_SET_TOKEN(node, end_keyword); } /** @@ -6924,7 +6928,7 @@ pm_when_node_create(pm_parser_t *parser, const pm_token_t *keyword) { */ static void pm_when_node_conditions_append(pm_when_node_t *node, pm_node_t *condition) { - node->base.location.end = condition->location.end; + PM_NODE_END_SET_NODE(node, condition); pm_node_list_append(&node->conditions, condition); } @@ -6933,7 +6937,7 @@ pm_when_node_conditions_append(pm_when_node_t *node, pm_node_t *condition) { */ static inline void pm_when_node_then_keyword_loc_set(const pm_parser_t *parser, pm_when_node_t *node, const pm_token_t *then_keyword) { - node->base.location.end = then_keyword->end; + PM_NODE_END_SET_TOKEN(node, then_keyword); node->then_keyword_loc = TOK2SLICE(parser, then_keyword); } @@ -6943,7 +6947,7 @@ pm_when_node_then_keyword_loc_set(const pm_parser_t *parser, pm_when_node_t *nod static void pm_when_node_statements_set(pm_when_node_t *node, pm_statements_node_t *statements) { if (statements->base.location.end > node->base.location.end) { - node->base.location.end = statements->base.location.end; + PM_NODE_END_SET_NODE(node, statements); } node->statements = statements; @@ -12969,7 +12973,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod call->arguments = arguments; pm_arguments_node_arguments_append(arguments, value); - call->base.location.end = arguments->base.location.end; + PM_NODE_END_SET_NODE(call, arguments); call->equal_loc = TOK2SLICE(parser, operator); parse_write_name(parser, &call->name); @@ -12988,7 +12992,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod } pm_arguments_node_arguments_append(call->arguments, value); - target->location.end = value->location.end; + PM_NODE_END_SET_NODE(target, value); // Replace the name with "[]=". call->name = pm_parser_constant_id_constant(parser, "[]=", 3); @@ -14167,7 +14171,7 @@ parse_parameters( pm_do_loop_stack_pop(parser); // If we don't have any parameters, return `NULL` instead of an empty `ParametersNode`. - if (params->base.location.start == params->base.location.end) { + if (PM_NODE_START(params) == PM_NODE_END(params)) { pm_node_destroy(parser, UP(params)); return NULL; } @@ -16156,8 +16160,8 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures pm_array_pattern_node_t *pattern_node = (pm_array_pattern_node_t *) inner; if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { - pattern_node->base.location.start = node->location.start; - pattern_node->base.location.end = closing.end; + PM_NODE_START_SET_NODE(pattern_node, node); + PM_NODE_END_SET_TOKEN(pattern_node, &closing); pattern_node->constant = node; pattern_node->opening_loc = TOK2SLICE(parser, &opening); @@ -16172,8 +16176,8 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures pm_find_pattern_node_t *pattern_node = (pm_find_pattern_node_t *) inner; if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { - pattern_node->base.location.start = node->location.start; - pattern_node->base.location.end = closing.end; + PM_NODE_START_SET_NODE(pattern_node, node); + PM_NODE_END_SET_TOKEN(pattern_node, &closing); pattern_node->constant = node; pattern_node->opening_loc = TOK2SLICE(parser, &opening); @@ -16188,8 +16192,8 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures pm_hash_pattern_node_t *pattern_node = (pm_hash_pattern_node_t *) inner; if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { - pattern_node->base.location.start = node->location.start; - pattern_node->base.location.end = closing.end; + PM_NODE_START_SET_NODE(pattern_node, node); + PM_NODE_END_SET_TOKEN(pattern_node, &closing); pattern_node->constant = node; pattern_node->opening_loc = TOK2SLICE(parser, &opening); @@ -16526,8 +16530,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm case PM_ARRAY_PATTERN_NODE: { pm_array_pattern_node_t *pattern_node = (pm_array_pattern_node_t *) inner; if (pattern_node->opening_loc.length == 0) { - pattern_node->base.location.start = opening.start; - pattern_node->base.location.end = closing.end; + PM_NODE_START_SET_TOKEN(pattern_node, &opening); + PM_NODE_END_SET_TOKEN(pattern_node, &closing); pattern_node->opening_loc = TOK2SLICE(parser, &opening); pattern_node->closing_loc = TOK2SLICE(parser, &closing); @@ -16540,8 +16544,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm case PM_FIND_PATTERN_NODE: { pm_find_pattern_node_t *pattern_node = (pm_find_pattern_node_t *) inner; if (pattern_node->opening_loc.length == 0) { - pattern_node->base.location.start = opening.start; - pattern_node->base.location.end = closing.end; + PM_NODE_START_SET_TOKEN(pattern_node, &opening); + PM_NODE_END_SET_TOKEN(pattern_node, &closing); pattern_node->opening_loc = TOK2SLICE(parser, &opening); pattern_node->closing_loc = TOK2SLICE(parser, &closing); @@ -16600,8 +16604,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm expect1(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_PATTERN_TERM_BRACE); pm_token_t closing = parser->previous; - node->base.location.start = opening.start; - node->base.location.end = closing.end; + PM_NODE_START_SET_TOKEN(node, &opening); + PM_NODE_END_SET_TOKEN(node, &closing); node->opening_loc = TOK2SLICE(parser, &opening); node->closing_loc = TOK2SLICE(parser, &closing); @@ -17530,8 +17534,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b multi_target->lparen_loc = TOK2SLICE(parser, &opening); multi_target->rparen_loc = TOK2SLICE(parser, &parser->previous); - multi_target->base.location.start = opening.start; - multi_target->base.location.end = parser->previous.end; + PM_NODE_START_SET_TOKEN(multi_target, &opening); + PM_NODE_END_SET_TOKEN(multi_target, &parser->previous); pm_node_t *result; if (match1(parser, PM_TOKEN_COMMA) && (binding_power == PM_BINDING_POWER_STATEMENT)) { @@ -17928,7 +17932,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &PM_STRING_EMPTY)); } - node->location.end = opening.end; + PM_NODE_END_SET_TOKEN(node, &opening); } else if ((part = parse_string_part(parser, (uint16_t) (depth + 1))) == NULL) { // If we get here, then we tried to find something in the // heredoc but couldn't actually parse anything, so we'll just @@ -18340,7 +18344,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parse_rescues(parser, opening_newline_index, &begin_keyword, begin_node, PM_RESCUES_BEGIN, (uint16_t) (depth + 1)); expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_BEGIN_TERM); - begin_node->base.location.end = parser->previous.end; + PM_NODE_END_SET_TOKEN(begin_node, &parser->previous); pm_begin_node_end_keyword_set(parser, begin_node, &parser->previous); pop_block_exits(parser, previous_block_exits); @@ -19365,7 +19369,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b current = UP(pm_symbol_node_to_string_node(parser, (pm_symbol_node_t *) current)); pm_interpolated_symbol_node_append(interpolated, current); - interpolated->base.location.start = current->location.start; + PM_NODE_START_SET_NODE(interpolated, current); start_location_set = true; current = UP(interpolated); } else { @@ -19376,7 +19380,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_t *part = parse_string_part(parser, (uint16_t) (depth + 1)); pm_interpolated_symbol_node_append((pm_interpolated_symbol_node_t *) current, part); if (!start_location_set) { - current->location.start = part->location.start; + PM_NODE_START_SET_NODE(current, part); } break; } @@ -19396,7 +19400,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b current = UP(pm_symbol_node_to_string_node(parser, (pm_symbol_node_t *) current)); pm_interpolated_symbol_node_append(interpolated, current); - interpolated->base.location.start = current->location.start; + PM_NODE_START_SET_NODE(interpolated, current); start_location_set = true; current = UP(interpolated); } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_SYMBOL_NODE)) { @@ -19409,7 +19413,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_t *part = parse_string_part(parser, (uint16_t) (depth + 1)); pm_interpolated_symbol_node_append((pm_interpolated_symbol_node_t *) current, part); if (!start_location_set) { - current->location.start = part->location.start; + PM_NODE_START_SET_NODE(current, part); } break; } From c3c268bd5dc2d283cce5d9c96f2c7977971755da Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 3 Dec 2025 11:14:52 -0500 Subject: [PATCH 20/22] Remove locations entirely --- rust/ruby-prism/build.rs | 12 +- rust/ruby-prism/src/lib.rs | 136 +-- src/prism.c | 1068 ++++++++++----------- src/static_literals.c | 10 +- templates/ext/prism/api_node.c.erb | 20 +- templates/include/prism/ast.h.erb | 14 +- templates/src/node.c.erb | 9 +- templates/src/prettyprint.c.erb | 9 +- templates/src/serialize.c.erb | 12 +- test/prism/result/overlap_test.rb | 9 +- test/prism/result/source_location_test.rb | 8 +- 11 files changed, 587 insertions(+), 720 deletions(-) diff --git a/rust/ruby-prism/build.rs b/rust/ruby-prism/build.rs index 19a495f927..3f8eb211c7 100644 --- a/rust/ruby-prism/build.rs +++ b/rust/ruby-prism/build.rs @@ -253,9 +253,9 @@ fn write_node(file: &mut File, flags: &[Flags], node: &Node) -> Result<(), Box Location<'pr> {{")?; - writeln!(file, " let pointer: *mut pm_location_t = unsafe {{ &raw mut (*self.pointer).base.location }};")?; - writeln!(file, " Location::new(self.parser, unsafe {{ &(*pointer) }})")?; + writeln!(file, " pub fn location(&self) -> Slice<'pr> {{")?; + writeln!(file, " let pointer: *mut pm_slice_t = unsafe {{ &raw mut (*self.pointer).base.location }};")?; + writeln!(file, " Slice::new(self.parser, unsafe {{ &(*pointer) }})")?; writeln!(file, " }}")?; writeln!(file)?; writeln!(file, " /// Returns the flags of this node.")?; @@ -558,7 +558,7 @@ use std::ptr::NonNull; #[allow(clippy::wildcard_imports)] use ruby_prism_sys::*; -use crate::{{ConstantId, ConstantList, Integer, Location, Slice, NodeList}}; +use crate::{{ConstantId, ConstantList, Integer, Slice, NodeList}}; " )?; @@ -621,10 +621,10 @@ impl<'pr> Node<'pr> {{ writeln!(file, " /// Returns the location of this node.")?; writeln!(file, " #[must_use]")?; - writeln!(file, " pub fn location(&self) -> Location<'pr> {{")?; + writeln!(file, " pub fn location(&self) -> Slice<'pr> {{")?; writeln!(file, " match *self {{")?; for node in &config.nodes { - writeln!(file, " Self::{} {{ pointer, parser, .. }} => Location::new(parser, unsafe {{ &((*pointer.cast::()).location) }}),", node.name)?; + writeln!(file, " Self::{} {{ pointer, parser, .. }} => Slice::new(parser, unsafe {{ &((*pointer.cast::()).location) }}),", node.name)?; } writeln!(file, " }}")?; writeln!(file, " }}")?; diff --git a/rust/ruby-prism/src/lib.rs b/rust/ruby-prism/src/lib.rs index c427c84067..4600fd31ce 100644 --- a/rust/ruby-prism/src/lib.rs +++ b/rust/ruby-prism/src/lib.rs @@ -19,40 +19,44 @@ use std::mem::MaybeUninit; use std::ptr::NonNull; pub use self::bindings::*; -use ruby_prism_sys::{pm_comment_t, pm_comment_type_t, pm_constant_id_list_t, pm_constant_id_t, pm_diagnostic_t, pm_integer_t, pm_location_t, pm_magic_comment_t, pm_node_destroy, pm_node_list, pm_node_t, pm_parse, pm_parser_free, pm_parser_init, pm_parser_t, pm_slice_t}; +use ruby_prism_sys::{pm_comment_t, pm_comment_type_t, pm_constant_id_list_t, pm_constant_id_t, pm_diagnostic_t, pm_integer_t, pm_magic_comment_t, pm_node_destroy, pm_node_list, pm_node_t, pm_parse, pm_parser_free, pm_parser_init, pm_parser_t, pm_slice_t}; -/// A range in the source file. -pub struct Location<'pr> { +/// A range in the source file, represented as a start offset and length. +pub struct Slice<'pr> { parser: NonNull, - pub(crate) start: *const u8, - pub(crate) end: *const u8, + pub(crate) start: u32, + pub(crate) length: u32, marker: PhantomData<&'pr [u8]>, } -impl<'pr> Location<'pr> { +impl<'pr> Slice<'pr> { /// Returns a byte slice for the range. - /// # Panics - /// Panics if the end offset is not greater than the start offset. #[must_use] pub fn as_slice(&self) -> &'pr [u8] { unsafe { - let len = usize::try_from(self.end.offset_from(self.start)).expect("end should point to memory after start"); - std::slice::from_raw_parts(self.start, len) + let parser_start = (*self.parser.as_ptr()).start; + std::slice::from_raw_parts(parser_start.add(self.start as usize), self.length as usize) } } - /// Return a Location from the given `pm_location_t`. + /// Return a Slice from the given `pm_slice_t`. #[must_use] - pub(crate) const fn new(parser: NonNull, loc: &'pr pm_location_t) -> Self { - Location { + pub(crate) const fn new(parser: NonNull, slice: &'pr pm_slice_t) -> Self { + Slice { parser, - start: loc.start, - end: loc.end, + start: slice.start, + length: slice.length, marker: PhantomData, } } - /// Return a Location starting at self and ending at the end of other. + /// Returns the end offset from the beginning of the parsed source. + #[must_use] + pub fn end(&self) -> u32 { + self.start + self.length + } + + /// Return a Slice starting at self and ending at the end of other. /// Returns None if both locations did not originate from the same parser, /// or if self starts after other. #[must_use] @@ -60,83 +64,14 @@ impl<'pr> Location<'pr> { if self.parser != other.parser || self.start > other.start { None } else { - Some(Location { + Some(Slice { parser: self.parser, start: self.start, - end: other.end, + length: other.end() - self.start, marker: PhantomData, }) } } - - /// Return the start offset from the beginning of the parsed source. - /// # Panics - /// Panics if the start offset is not greater than the parser's start. - #[must_use] - pub fn start_offset(&self) -> usize { - unsafe { - let parser_start = (*self.parser.as_ptr()).start; - usize::try_from(self.start.offset_from(parser_start)).expect("start should point to memory after the parser's start") - } - } - - /// Return the end offset from the beginning of the parsed source. - /// # Panics - /// Panics if the end offset is not greater than the parser's start. - #[must_use] - pub fn end_offset(&self) -> usize { - unsafe { - let parser_start = (*self.parser.as_ptr()).start; - usize::try_from(self.end.offset_from(parser_start)).expect("end should point to memory after the parser's start") - } - } -} - -impl std::fmt::Debug for Location<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let slice: &[u8] = self.as_slice(); - - let mut visible = String::new(); - visible.push('"'); - - for &byte in slice { - let part: Vec = std::ascii::escape_default(byte).collect(); - visible.push_str(std::str::from_utf8(&part).unwrap()); - } - - visible.push('"'); - write!(f, "{visible}") - } -} - -/// A range in the source file, represented as a start offset and length. -pub struct Slice<'pr> { - parser: NonNull, - pub(crate) start: u32, - pub(crate) length: u32, - marker: PhantomData<&'pr [u8]>, -} - -impl<'pr> Slice<'pr> { - /// Returns a byte slice for the range. - #[must_use] - pub fn as_slice(&self) -> &'pr [u8] { - unsafe { - let parser_start = (*self.parser.as_ptr()).start; - std::slice::from_raw_parts(parser_start.add(self.start as usize), self.length as usize) - } - } - - /// Return a Slice from the given `pm_slice_t`. - #[must_use] - pub(crate) const fn new(parser: NonNull, slice: &'pr pm_slice_t) -> Self { - Slice { - parser, - start: slice.start, - length: slice.length, - marker: PhantomData, - } - } } impl std::fmt::Debug for Slice<'_> { @@ -572,21 +507,6 @@ impl<'pr> ParseResult<'pr> { unsafe { (*self.parser.as_ptr()).frozen_string_literal == 1 } } - /// Returns a slice of the source string that was parsed using the given - /// location range. - /// - /// # Panics - /// Panics if start offset or end offset are not valid offsets from the root. - #[must_use] - pub fn as_location_slice(&self, location: &Location<'pr>) -> &'pr [u8] { - let root = self.source.as_ptr(); - - let start = usize::try_from(unsafe { location.start.offset_from(root) }).expect("start should point to memory after root"); - let end = usize::try_from(unsafe { location.end.offset_from(root) }).expect("end should point to memory after root"); - - &self.source[start..end] - } - /// Returns a slice of the source string that was parsed using the given /// slice range. #[must_use] @@ -772,16 +692,16 @@ mod tests { let node = plus.arguments().unwrap().arguments().iter().next().unwrap(); let location = node.as_integer_node().unwrap().location(); - let slice = std::str::from_utf8(result.as_location_slice(&location)).unwrap(); + let slice = std::str::from_utf8(result.as_slice(&location)).unwrap(); assert_eq!(slice, "222"); - assert_eq!(6, location.start_offset()); - assert_eq!(9, location.end_offset()); + assert_eq!(6, location.start); + assert_eq!(9, location.end()); let recv_loc = plus.receiver().unwrap().location(); assert_eq!(recv_loc.as_slice(), b"111"); - assert_eq!(0, recv_loc.start_offset()); - assert_eq!(3, recv_loc.end_offset()); + assert_eq!(0, recv_loc.start); + assert_eq!(3, recv_loc.end()); let joined = recv_loc.join(&location).unwrap(); assert_eq!(joined.as_slice(), b"111 + 222"); @@ -806,7 +726,7 @@ mod tests { } let location = node.location(); - let slice = std::str::from_utf8(result.as_location_slice(&location)).unwrap(); + let slice = std::str::from_utf8(result.as_slice(&location)).unwrap(); assert_eq!(slice, "222"); diff --git a/src/prism.c b/src/prism.c index 7ef4df200c..622a952a15 100644 --- a/src/prism.c +++ b/src/prism.c @@ -27,25 +27,31 @@ pm_version(void) { #define FL PM_NODE_FLAGS #define UP PM_NODE_UPCAST -#define PM_TOKEN_START(token_) ((token_)->start) -#define PM_TOKEN_END(token_) ((token_)->end) -#define PM_TOKEN_LENGTH(token_) U32(PM_TOKEN_END(token_) - PM_TOKEN_START(token_)) +#define PM_SLICE_START(slice_) ((slice_)->start) +#define PM_SLICE_END(slice_) ((slice_)->start + (slice_)->length) + +#define PM_TOKEN_START(parser_, token_) U32((token_)->start - (parser_)->start) +#define PM_TOKEN_END(parser_, token_) U32((token_)->end - (parser_)->start) +#define PM_TOKEN_LENGTH(token_) U32((token_)->end - (token_)->start) +#define PM_TOKENS_LENGTH(left_, right_) U32((right_)->end - (left_)->start) #define PM_NODE_START(node_) (UP(node_)->location.start) -#define PM_NODE_END(node_) (UP(node_)->location.end) -#define PM_NODE_LENGTH(node_) U32(PM_NODE_END(node_) - PM_NODE_START(node_)) +#define PM_NODE_LENGTH(node_) (UP(node_)->location.length) +#define PM_NODE_END(node_) (UP(node_)->location.start + UP(node_)->location.length) +#define PM_NODES_LENGTH(left_, right_) (PM_NODE_END(right_) - PM_NODE_START(left_)) -#define PM_NODE_START_SET_NODE(left_, right_) (PM_NODE_START(left_) = PM_NODE_START(right_)) -#define PM_NODE_START_SET_TOKEN(node_, token_) (PM_NODE_START(node_) = PM_TOKEN_START(token_)) -#define PM_NODE_END_SET_NODE(left_, right_) (PM_NODE_END(left_) = PM_NODE_END(right_)) -#define PM_NODE_END_SET_TOKEN(node_, token_) (PM_NODE_END(node_) = PM_TOKEN_END(token_)) +#define PM_TOKEN_NODE_LENGTH(parser_, token_, node_) (PM_NODE_END(node_) - PM_TOKEN_START(parser_, token_)) +#define PM_NODE_TOKEN_LENGTH(parser_, node_, token_) (PM_TOKEN_END(parser_, token_) - PM_NODE_START(node_)) -#define PM_LOCATION_TOKEN_VALUE(token_) ((pm_location_t) { .start = PM_TOKEN_START(token_), .end = PM_TOKEN_END(token_) }) +#define PM_NODE_START_SET_NODE(left_, right_) (PM_NODE_START(left_) = PM_NODE_START(right_)) +#define PM_NODE_START_SET_TOKEN(parser_, node_, token_) (PM_NODE_START(node_) = PM_TOKEN_START(parser_, token_)) +#define PM_NODE_LENGTH_SET_NODE(left_, right_) (PM_NODE_LENGTH(left_) = PM_NODE_END(right_) - PM_NODE_START(left_)) +#define PM_NODE_LENGTH_SET_TOKEN(parser_, node_, token_) (PM_NODE_LENGTH(node_) = PM_TOKEN_END(parser_, token_) - PM_NODE_START(node_)) +#define PM_NODE_LENGTH_SET_SLICE(node_, slice_) (PM_NODE_LENGTH(node_) = PM_SLICE_END(slice_) - PM_NODE_START(node_)) -#define TOK2SLICE(parser_, token_) ((pm_slice_t) { .start = (uint32_t) ((token_)->start - (parser_)->start), .length = (uint32_t) ((token_)->end - (token_)->start) }) +#define TOK2SLICE(parser_, token_) ((pm_slice_t) { .start = PM_TOKEN_START(parser_, token_), .length = PM_TOKEN_LENGTH(token_) }) #define NTOK2SLICE(parser_, token_) ((token_) == NULL ? ((pm_slice_t) { 0 }) : TOK2SLICE(parser_, token_)) #define NTOK2PTR(token_) ((token_).start == NULL ? NULL : &(token_)) -#define LOC2SLICE TOK2SLICE /******************************************************************************/ /* Lex mode manipulations */ @@ -438,16 +444,13 @@ pm_parser_err(pm_parser_t *parser, uint32_t start, uint32_t length, pm_diagnosti } /** - * Append an error to the list of errors on the parser using a format string. - */ -#define PM_PARSER_ERR_FORMAT(parser_, start_, end_, diag_id_, ...) \ - pm_diagnostic_list_append_format(&(parser_)->error_list, (uint32_t) ((start_) - (parser_)->start), (uint32_t) ((end_) - (start_)), diag_id_, __VA_ARGS__) - -/** - * Append an error to the list of errors on the parser using the given slice. + * Append an error to the list of errors on the parser using the location of the + * given token. */ -#define PM_PARSER_ERR_FORMAT_SLICE(parser_, start_, length_, diag_id_, ...) \ - pm_diagnostic_list_append_format(&(parser_)->error_list, (start_), (length_), diag_id_, __VA_ARGS__) +static inline void +pm_parser_err_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_id_t diag_id) { + pm_parser_err(parser, PM_TOKEN_START(parser, token), PM_TOKEN_LENGTH(token), diag_id); +} /** * Append an error to the list of errors on the parser using the location of the @@ -455,15 +458,17 @@ pm_parser_err(pm_parser_t *parser, uint32_t start, uint32_t length, pm_diagnosti */ static inline void pm_parser_err_current(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current), diag_id); + pm_parser_err_token(parser, &parser->current, diag_id); } /** - * Append an error to the list of errors on the parser using the given location - * using a format string. + * Append an error to the list of errors on the parser using the location of the + * previous token. */ -#define PM_PARSER_ERR_LOCATION_FORMAT(parser, location, diag_id, ...) \ - PM_PARSER_ERR_FORMAT(parser, (location)->start, (location)->end, diag_id, __VA_ARGS__) +static inline void +pm_parser_err_previous(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { + pm_parser_err_token(parser, &parser->previous, diag_id); +} /** * Append an error to the list of errors on the parser using the location of the @@ -471,54 +476,42 @@ pm_parser_err_current(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { */ static inline void pm_parser_err_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node), diag_id); + pm_parser_err(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), diag_id); } /** - * Append an error to the list of errors on the parser using the location of the - * given node and a format string. - */ -#define PM_PARSER_ERR_NODE_FORMAT(parser, node, diag_id, ...) \ - PM_PARSER_ERR_FORMAT(parser, (node)->location.start, (node)->location.end, diag_id, __VA_ARGS__) - -/** - * Append an error to the list of errors on the parser using the location of the - * given node and a format string, and add on the content of the node. + * Append an error to the list of errors on the parser using a format string. */ -#define PM_PARSER_ERR_NODE_FORMAT_CONTENT(parser, node, diag_id) \ - PM_PARSER_ERR_NODE_FORMAT(parser, node, diag_id, (int) ((node)->location.end - (node)->location.start), (const char *) (node)->location.start) +#define PM_PARSER_ERR_FORMAT(parser_, start_, length_, diag_id_, ...) \ + pm_diagnostic_list_append_format(&(parser_)->error_list, start_, length_, diag_id_, __VA_ARGS__) /** * Append an error to the list of errors on the parser using the location of the - * previous token. + * given node and a format string. */ -static inline void -pm_parser_err_previous(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, U32(parser->previous.start - parser->start), PM_TOKEN_LENGTH(&parser->previous), diag_id); -} +#define PM_PARSER_ERR_NODE_FORMAT(parser_, node_, diag_id_, ...) \ + PM_PARSER_ERR_FORMAT(parser_, PM_NODE_START(node_), PM_NODE_LENGTH(node_), diag_id_, __VA_ARGS__) /** * Append an error to the list of errors on the parser using the location of the - * given token. + * given node and a format string, and add on the content of the node. */ -static inline void -pm_parser_err_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_id_t diag_id) { - pm_parser_err(parser, U32(token->start - parser->start), PM_TOKEN_LENGTH(token), diag_id); -} +#define PM_PARSER_ERR_NODE_FORMAT_CONTENT(parser_, node_, diag_id_) \ + PM_PARSER_ERR_NODE_FORMAT(parser_, node_, diag_id_, (int) PM_NODE_LENGTH(node_), (const char *) (parser_->start + PM_NODE_START(node_))) /** * Append an error to the list of errors on the parser using the location of the * given token and a format string. */ -#define PM_PARSER_ERR_TOKEN_FORMAT(parser, token, diag_id, ...) \ - PM_PARSER_ERR_FORMAT(parser, (token).start, (token).end, diag_id, __VA_ARGS__) +#define PM_PARSER_ERR_TOKEN_FORMAT(parser_, token_, diag_id, ...) \ + PM_PARSER_ERR_FORMAT(parser_, PM_TOKEN_START(parser_, token_), PM_TOKEN_LENGTH(token_), diag_id, __VA_ARGS__) /** * Append an error to the list of errors on the parser using the location of the * given token and a format string, and add on the content of the token. */ -#define PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, token, diag_id) \ - PM_PARSER_ERR_TOKEN_FORMAT(parser, token, diag_id, (int) ((token).end - (token).start), (const char *) (token).start) +#define PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser_, token_, diag_id_) \ + PM_PARSER_ERR_TOKEN_FORMAT(parser_, token_, diag_id_, (int) PM_TOKEN_LENGTH(token_), (const char *) (token_)->start) /** * Append a warning to the list of warnings on the parser. @@ -534,7 +527,7 @@ pm_parser_warn(pm_parser_t *parser, uint32_t start, uint32_t length, pm_diagnost */ static inline void pm_parser_warn_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic_id_t diag_id) { - pm_parser_warn(parser, U32(token->start - parser->start), PM_TOKEN_LENGTH(token), diag_id); + pm_parser_warn(parser, PM_TOKEN_START(parser, token), PM_TOKEN_LENGTH(token), diag_id); } /** @@ -543,43 +536,36 @@ pm_parser_warn_token(pm_parser_t *parser, const pm_token_t *token, pm_diagnostic */ static inline void pm_parser_warn_node(pm_parser_t *parser, const pm_node_t *node, pm_diagnostic_id_t diag_id) { - pm_parser_warn(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node), diag_id); + pm_parser_warn(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), diag_id); } /** * Append a warning to the list of warnings on the parser using a format string * and the given location. */ -#define PM_PARSER_WARN_FORMAT_LOCATION(parser_, start_, end_, diag_id_, ...) \ - pm_diagnostic_list_append_format(&(parser_)->warning_list, (uint32_t) ((start_) - (parser_)->start), (uint32_t) ((end_) - (start_)), diag_id_, __VA_ARGS__) - -/** - * Append a warning to the list of warnings on the parser using a format string - * and the given slice. - */ -#define PM_PARSER_WARN_FORMAT_SLICE(parser_, slice_, diag_id_, ...) \ - pm_diagnostic_list_append_format(&(parser_)->warning_list, (slice_)->start, (slice_)->length, diag_id_, __VA_ARGS__) +#define PM_PARSER_WARN_FORMAT(parser_, start_, length_, diag_id_, ...) \ + pm_diagnostic_list_append_format(&(parser_)->warning_list, start_, length_, diag_id_, __VA_ARGS__) /** * Append a warning to the list of warnings on the parser using the location of * the given token and a format string. */ -#define PM_PARSER_WARN_TOKEN_FORMAT(parser, token, diag_id, ...) \ - PM_PARSER_WARN_FORMAT_LOCATION(parser, (token).start, (token).end, diag_id, __VA_ARGS__) +#define PM_PARSER_WARN_TOKEN_FORMAT(parser_, token_, diag_id_, ...) \ + PM_PARSER_WARN_FORMAT(parser_, PM_TOKEN_START(parser_, token_), PM_TOKEN_LENGTH(token_), diag_id_, __VA_ARGS__) /** * Append a warning to the list of warnings on the parser using the location of * the given token and a format string, and add on the content of the token. */ -#define PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, token, diag_id) \ - PM_PARSER_WARN_TOKEN_FORMAT(parser, token, diag_id, (int) ((token).end - (token).start), (const char *) (token).start) +#define PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser_, token_, diag_id_) \ + PM_PARSER_WARN_TOKEN_FORMAT(parser_, token_, diag_id_, (int) PM_TOKEN_LENGTH(token_), (const char *) (token_)->start) /** * Append a warning to the list of warnings on the parser using the location of * the given node and a format string. */ -#define PM_PARSER_WARN_NODE_FORMAT(parser, node, diag_id, ...) \ - PM_PARSER_WARN_FORMAT_LOCATION(parser, (node)->location.start, (node)->location.end, diag_id, __VA_ARGS__) +#define PM_PARSER_WARN_NODE_FORMAT(parser_, node_, diag_id_, ...) \ + PM_PARSER_WARN_FORMAT(parser_, PM_NODE_START(node_), PM_NODE_LENGTH(node_), diag_id_, __VA_ARGS__) /** * Add an error for an expected heredoc terminator. This is a special function @@ -590,8 +576,8 @@ static void pm_parser_err_heredoc_term(pm_parser_t *parser, const uint8_t *ident_start, size_t ident_length) { PM_PARSER_ERR_FORMAT( parser, - ident_start, - ident_start + ident_length, + U32(ident_start - parser->start), + U32(ident_length), PM_ERR_HEREDOC_TERM, (int) ident_length, (const char *) ident_start @@ -1008,9 +994,10 @@ pm_locals_order(PRISM_ATTRIBUTE_UNUSED pm_parser_t *parser, pm_locals_t *locals, pm_constant_t *constant = pm_constant_pool_id_to_constant(&parser->constant_pool, local->name); if (constant->length >= 1 && *constant->start != '_') { - PM_PARSER_WARN_FORMAT_SLICE( + PM_PARSER_WARN_FORMAT( parser, - &local->location, + local->location.start, + local->location.length, PM_WARN_UNUSED_LOCAL_VARIABLE, (int) constant->length, (const char *) constant->start @@ -1605,25 +1592,24 @@ typedef struct { /** * Retrieve the end location of a `pm_arguments_t` object. */ -static inline const uint8_t * -pm_arguments_end(const pm_parser_t *parser, pm_arguments_t *arguments) { +static inline const pm_slice_t * +pm_arguments_end(pm_arguments_t *arguments) { if (arguments->block != NULL) { - const uint8_t *end = arguments->block->location.end; + uint32_t end = PM_NODE_END(arguments->block); if (arguments->closing_loc.length > 0) { - const uint8_t *arguments_end = parser->start + arguments->closing_loc.start + arguments->closing_loc.length; - + uint32_t arguments_end = PM_SLICE_END(&arguments->closing_loc); if (arguments_end > end) { - end = arguments_end; + return &arguments->closing_loc; } } - return end; + return &arguments->block->location; } if (arguments->closing_loc.length > 0) { - return parser->start + arguments->closing_loc.start + arguments->closing_loc.length; + return &arguments->closing_loc; } if (arguments->arguments != NULL) { - return arguments->arguments->base.location.end; + return &arguments->arguments->base.location; } return NULL; } @@ -1913,7 +1899,7 @@ pm_regular_expression_flags_create(pm_parser_t *parser, const pm_token_t *closin size_t unknown_flags_length = pm_buffer_length(&unknown_flags); if (unknown_flags_length != 0) { const char *word = unknown_flags_length >= 2 ? "options" : "option"; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_REGEXP_UNKNOWN_OPTIONS, word, unknown_flags_length, pm_buffer_value(&unknown_flags)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_REGEXP_UNKNOWN_OPTIONS, word, unknown_flags_length, pm_buffer_value(&unknown_flags)); } pm_buffer_free(&unknown_flags); } @@ -1947,32 +1933,32 @@ pm_node_alloc(PRISM_ATTRIBUTE_UNUSED pm_parser_t *parser, size_t size) { } #define PM_NODE_ALLOC(parser_, type_) (type_ *) pm_node_alloc(parser_, sizeof(type_)) -#define PM_NODE_INIT(parser_, type_, flags_, start_, end_) (pm_node_t) { \ +#define PM_NODE_INIT(parser_, type_, flags_, start_, length_) (pm_node_t) { \ .type = (type_), \ .flags = (flags_), \ .node_id = ++(parser_)->node_id, \ - .location = { .start = (start_), .end = (end_) } \ + .location = { .start = (start_), .length = (length_) } \ } -#define PM_NODE_INIT_UNSET(parser_, type_, flags_) PM_NODE_INIT(parser_, type_, flags_, NULL, NULL) -#define PM_NODE_INIT_BASE(parser_, type_, flags_) PM_NODE_INIT(parser_, type_, flags_, (parser_)->start, (parser_)->start) -#define PM_NODE_INIT_TOKEN(parser_, type_, flags_, token_) PM_NODE_INIT(parser_, type_, flags_, PM_TOKEN_START(token_), PM_TOKEN_END(token_)) -#define PM_NODE_INIT_NODE(parser_, type_, flags_, node_) PM_NODE_INIT(parser_, type_, flags_, PM_NODE_START(node_), PM_NODE_END(node_)) +#define PM_NODE_INIT_UNSET(parser_, type_, flags_) PM_NODE_INIT(parser_, type_, flags_, 0, 0) +#define PM_NODE_INIT_BASE(parser_, type_, flags_) PM_NODE_INIT(parser_, type_, flags_, 0, 0) +#define PM_NODE_INIT_TOKEN(parser_, type_, flags_, token_) PM_NODE_INIT(parser_, type_, flags_, PM_TOKEN_START(parser_, token_), PM_TOKEN_LENGTH(token_)) +#define PM_NODE_INIT_NODE(parser_, type_, flags_, node_) PM_NODE_INIT(parser_, type_, flags_, PM_NODE_START(node_), PM_NODE_LENGTH(node_)) -#define PM_NODE_INIT_TOKENS(parser_, type_, flags_, left_, right_) PM_NODE_INIT(parser_, type_, flags_, PM_TOKEN_START(left_), PM_TOKEN_END(right_)) -#define PM_NODE_INIT_NODES(parser_, type_, flags_, left_, right_) PM_NODE_INIT(parser_, type_, flags_, PM_NODE_START(left_), PM_NODE_END(right_)) -#define PM_NODE_INIT_TOKEN_NODE(parser_, type_, flags_, token_, node_) PM_NODE_INIT(parser_, type_, flags_, PM_TOKEN_START(token_), PM_NODE_END(node_)) -#define PM_NODE_INIT_NODE_TOKEN(parser_, type_, flags_, node_, token_) PM_NODE_INIT(parser_, type_, flags_, PM_NODE_START(node_), PM_TOKEN_END(token_)) +#define PM_NODE_INIT_TOKENS(parser_, type_, flags_, left_, right_) PM_NODE_INIT(parser_, type_, flags_, PM_TOKEN_START(parser_, left_), PM_TOKENS_LENGTH(left_, right_)) +#define PM_NODE_INIT_NODES(parser_, type_, flags_, left_, right_) PM_NODE_INIT(parser_, type_, flags_, PM_NODE_START(left_), PM_NODES_LENGTH(left_, right_)) +#define PM_NODE_INIT_TOKEN_NODE(parser_, type_, flags_, token_, node_) PM_NODE_INIT(parser_, type_, flags_, PM_TOKEN_START(parser_, token_), PM_TOKEN_NODE_LENGTH(parser_, token_, node_)) +#define PM_NODE_INIT_NODE_TOKEN(parser_, type_, flags_, node_, token_) PM_NODE_INIT(parser_, type_, flags_, PM_NODE_START(node_), PM_NODE_TOKEN_LENGTH(parser_, node_, token_)) /** * Allocate a new MissingNode node. */ static pm_missing_node_t * -pm_missing_node_create(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { +pm_missing_node_create(pm_parser_t *parser, uint32_t start, uint32_t length) { pm_missing_node_t *node = PM_NODE_ALLOC(parser, pm_missing_node_t); *node = (pm_missing_node_t) { - .base = PM_NODE_INIT(parser, PM_MISSING_NODE, 0, start, end) + .base = PM_NODE_INIT(parser, PM_MISSING_NODE, 0, start, length) }; return node; @@ -2082,7 +2068,7 @@ pm_arguments_node_arguments_append(pm_arguments_node_t *node, pm_node_t *argumen PM_NODE_START_SET_NODE(node, argument); } - PM_NODE_END_SET_NODE(node, argument); + PM_NODE_LENGTH_SET_NODE(node, argument); pm_node_list_append(&node->arguments, argument); if (PM_NODE_TYPE_P(argument, PM_SPLAT_NODE)) { @@ -2130,7 +2116,7 @@ pm_array_node_elements_append(pm_array_node_t *node, pm_node_t *element) { } pm_node_list_append(&node->elements, element); - PM_NODE_END_SET_NODE(node, element); + PM_NODE_LENGTH_SET_NODE(node, element); // If the element is not a static literal, then the array is not a static // literal. Turn that flag off. @@ -2149,7 +2135,7 @@ pm_array_node_elements_append(pm_array_node_t *node, pm_node_t *element) { static void pm_array_node_close_set(const pm_parser_t *parser, pm_array_node_t *node, const pm_token_t *closing) { assert(closing->type == PM_TOKEN_BRACKET_RIGHT || closing->type == PM_TOKEN_STRING_END || closing->type == 0); - PM_NODE_END_SET_TOKEN(node, closing); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); node->closing_loc = TOK2SLICE(parser, closing); } @@ -2263,14 +2249,14 @@ pm_array_pattern_node_requireds_append(pm_array_pattern_node_t *node, pm_node_t static pm_assoc_node_t * pm_assoc_node_create(pm_parser_t *parser, pm_node_t *key, const pm_token_t *operator, pm_node_t *value) { pm_assoc_node_t *node = PM_NODE_ALLOC(parser, pm_assoc_node_t); - const uint8_t *end; + uint32_t end; - if (value != NULL && value->location.end > key->location.end) { - end = value->location.end; + if (value != NULL && PM_NODE_END(value) > PM_NODE_END(key)) { + end = PM_NODE_END(value); } else if (operator != NULL) { - end = operator->end; + end = PM_TOKEN_END(parser, operator); } else { - end = key->location.end; + end = PM_NODE_END(key); } // Hash string keys will be frozen, so we can mark them as frozen here so @@ -2291,7 +2277,7 @@ pm_assoc_node_create(pm_parser_t *parser, pm_node_t *key, const pm_token_t *oper } *node = (pm_assoc_node_t) { - .base = PM_NODE_INIT(parser, PM_ASSOC_NODE, flags, key->location.start, end), + .base = PM_NODE_INIT(parser, PM_ASSOC_NODE, flags, PM_NODE_START(key), U32(end - PM_NODE_START(key))), .key = key, .operator_loc = NTOK2SLICE(parser, operator), .value = value @@ -2344,11 +2330,11 @@ static pm_begin_node_t * pm_begin_node_create(pm_parser_t *parser, const pm_token_t *begin_keyword, pm_statements_node_t *statements) { pm_begin_node_t *node = PM_NODE_ALLOC(parser, pm_begin_node_t); - const uint8_t *start = begin_keyword == NULL ? parser->start : begin_keyword->start; - const uint8_t *end = statements == NULL ? (begin_keyword == NULL ? parser->start : begin_keyword->end) : statements->base.location.end; + uint32_t start = begin_keyword == NULL ? 0 : PM_TOKEN_START(parser, begin_keyword); + uint32_t end = statements == NULL ? (begin_keyword == NULL ? 0 : PM_TOKEN_END(parser, begin_keyword)) : PM_NODE_END(statements); *node = (pm_begin_node_t) { - .base = PM_NODE_INIT(parser, PM_BEGIN_NODE, 0, start, end), + .base = PM_NODE_INIT(parser, PM_BEGIN_NODE, 0, start, U32(end - start)), .begin_keyword_loc = NTOK2SLICE(parser, begin_keyword), .statements = statements, .end_keyword_loc = { 0 } @@ -2362,11 +2348,10 @@ pm_begin_node_create(pm_parser_t *parser, const pm_token_t *begin_keyword, pm_st */ static void pm_begin_node_rescue_clause_set(pm_begin_node_t *node, pm_rescue_node_t *rescue_clause) { - // If the begin keyword doesn't exist, we set the start on the begin_node if (node->begin_keyword_loc.length == 0) { PM_NODE_START_SET_NODE(node, rescue_clause); } - PM_NODE_END_SET_NODE(node, rescue_clause); + PM_NODE_LENGTH_SET_NODE(node, rescue_clause); node->rescue_clause = rescue_clause; } @@ -2375,7 +2360,10 @@ pm_begin_node_rescue_clause_set(pm_begin_node_t *node, pm_rescue_node_t *rescue_ */ static void pm_begin_node_else_clause_set(pm_begin_node_t *node, pm_else_node_t *else_clause) { - PM_NODE_END_SET_NODE(node, else_clause); + if ((node->begin_keyword_loc.length == 0) && PM_NODE_START(node) == 0) { + PM_NODE_START_SET_NODE(node, else_clause); + } + PM_NODE_LENGTH_SET_NODE(node, else_clause); node->else_clause = else_clause; } @@ -2384,7 +2372,10 @@ pm_begin_node_else_clause_set(pm_begin_node_t *node, pm_else_node_t *else_clause */ static void pm_begin_node_ensure_clause_set(pm_begin_node_t *node, pm_ensure_node_t *ensure_clause) { - PM_NODE_END_SET_NODE(node, ensure_clause); + if ((node->begin_keyword_loc.length == 0) && PM_NODE_START(node) == 0) { + PM_NODE_START_SET_NODE(node, ensure_clause); + } + PM_NODE_LENGTH_SET_NODE(node, ensure_clause); node->ensure_clause = ensure_clause; } @@ -2394,7 +2385,7 @@ pm_begin_node_ensure_clause_set(pm_begin_node_t *node, pm_ensure_node_t *ensure_ static void pm_begin_node_end_keyword_set(const pm_parser_t *parser, pm_begin_node_t *node, const pm_token_t *end_keyword) { assert(end_keyword->type == PM_TOKEN_KEYWORD_END || end_keyword->type == 0); - PM_NODE_END_SET_TOKEN(node, end_keyword); + PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); node->end_keyword_loc = TOK2SLICE(parser, end_keyword); } @@ -2467,26 +2458,26 @@ static pm_block_parameters_node_t * pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *parameters, const pm_token_t *opening) { pm_block_parameters_node_t *node = PM_NODE_ALLOC(parser, pm_block_parameters_node_t); - const uint8_t *start; + uint32_t start; if (opening != NULL) { - start = opening->start; + start = PM_TOKEN_START(parser, opening); } else if (parameters != NULL) { - start = parameters->base.location.start; + start = PM_NODE_START(parameters); } else { - start = NULL; + start = 0; } - const uint8_t *end; + uint32_t end; if (parameters != NULL) { - end = parameters->base.location.end; + end = PM_NODE_END(parameters); } else if (opening != NULL) { - end = opening->end; + end = PM_TOKEN_END(parser, opening); } else { - end = NULL; + end = 0; } *node = (pm_block_parameters_node_t) { - .base = PM_NODE_INIT(parser, PM_BLOCK_PARAMETERS_NODE, 0, start, end), + .base = PM_NODE_INIT(parser, PM_BLOCK_PARAMETERS_NODE, 0, start, U32(end - start)), .parameters = parameters, .opening_loc = NTOK2SLICE(parser, opening), .closing_loc = { 0 }, @@ -2502,7 +2493,7 @@ pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *param static void pm_block_parameters_node_closing_set(const pm_parser_t *parser, pm_block_parameters_node_t *node, const pm_token_t *closing) { assert(closing->type == PM_TOKEN_PIPE || closing->type == PM_TOKEN_PARENTHESIS_RIGHT || closing->type == 0); - PM_NODE_END_SET_TOKEN(node, closing); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); node->closing_loc = TOK2SLICE(parser, closing); } @@ -2528,11 +2519,11 @@ static void pm_block_parameters_node_append_local(pm_block_parameters_node_t *node, const pm_block_local_variable_node_t *local) { pm_node_list_append(&node->locals, UP(local)); - if (PM_NODE_START(node) == NULL) { + if (PM_NODE_LENGTH(node) == 0) { PM_NODE_START_SET_NODE(node, local); } - PM_NODE_END_SET_NODE(node, local); + PM_NODE_LENGTH_SET_NODE(node, local); } /** @@ -2615,7 +2606,10 @@ pm_call_node_aref_create(pm_parser_t *parser, pm_node_t *receiver, pm_arguments_ pm_call_node_t *node = pm_call_node_create(parser, flags); PM_NODE_START_SET_NODE(node, receiver); - node->base.location.end = pm_arguments_end(parser, arguments); + + const pm_slice_t *end = pm_arguments_end(arguments); + assert(end != NULL && "unreachable"); + PM_NODE_LENGTH_SET_SLICE(node, end); node->receiver = receiver; node->message_loc.start = arguments->opening_loc.start; @@ -2640,8 +2634,8 @@ pm_call_node_binary_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver) | flags); - node->base.location.start = MIN(receiver->location.start, argument->location.start); - node->base.location.end = MAX(receiver->location.end, argument->location.end); + PM_NODE_START_SET_NODE(node, PM_NODE_START(receiver) < PM_NODE_START(argument) ? receiver : argument); + PM_NODE_LENGTH_SET_NODE(node, PM_NODE_END(receiver) > PM_NODE_END(argument) ? receiver : argument); node->receiver = receiver; node->message_loc = TOK2SLICE(parser, operator); @@ -2666,11 +2660,12 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); PM_NODE_START_SET_NODE(node, receiver); - const uint8_t *end = pm_arguments_end(parser, arguments); + const pm_slice_t *end = pm_arguments_end(arguments); if (end == NULL) { - end = message->end; + PM_NODE_LENGTH_SET_TOKEN(parser, node, message); + } else { + PM_NODE_LENGTH_SET_SLICE(node, end); } - node->base.location.end = end; node->receiver = receiver; node->call_operator_loc = TOK2SLICE(parser, operator); @@ -2698,8 +2693,7 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o static pm_call_node_t * pm_call_node_call_synthesized_create(pm_parser_t *parser, pm_node_t *receiver, const char *message, pm_arguments_node_t *arguments) { pm_call_node_t *node = pm_call_node_create(parser, 0); - node->base.location.start = parser->start; - node->base.location.end = parser->end; + node->base.location = (pm_slice_t) { .start = 0, .length = U32(parser->end - parser->start) }; node->receiver = receiver; node->arguments = arguments; @@ -2716,8 +2710,10 @@ static pm_call_node_t * pm_call_node_fcall_create(pm_parser_t *parser, pm_token_t *message, pm_arguments_t *arguments) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - PM_NODE_START_SET_TOKEN(node, message); - node->base.location.end = pm_arguments_end(parser, arguments); + PM_NODE_START_SET_TOKEN(parser, node, message); + const pm_slice_t *end = pm_arguments_end(arguments); + assert(end != NULL && "unreachable"); + PM_NODE_LENGTH_SET_SLICE(node, end); node->message_loc = TOK2SLICE(parser, message); node->opening_loc = arguments->opening_loc; @@ -2737,7 +2733,7 @@ static pm_call_node_t * pm_call_node_fcall_synthesized_create(pm_parser_t *parser, pm_arguments_node_t *arguments, pm_constant_id_t name) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location = ((pm_location_t) { .start = parser->start, .end = parser->start }); + node->base.location = (pm_slice_t) { 0 }; node->arguments = arguments; node->name = name; @@ -2753,13 +2749,13 @@ pm_call_node_not_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *me if (receiver != NULL) pm_conditional_predicate(parser, receiver, PM_CONDITIONAL_PREDICATE_TYPE_NOT); pm_call_node_t *node = pm_call_node_create(parser, receiver == NULL ? 0 : pm_call_node_ignore_visibility_flag(receiver)); - - PM_NODE_START_SET_TOKEN(node, message); + + PM_NODE_START_SET_TOKEN(parser, node, message); if (arguments->closing_loc.length > 0) { - node->base.location.end = parser->start + arguments->closing_loc.start + arguments->closing_loc.length; + PM_NODE_LENGTH_SET_SLICE(node, &arguments->closing_loc); } else { assert(receiver != NULL); - PM_NODE_END_SET_NODE(node, receiver); + PM_NODE_LENGTH_SET_NODE(node, receiver); } node->receiver = receiver; @@ -2782,7 +2778,9 @@ pm_call_node_shorthand_create(pm_parser_t *parser, pm_node_t *receiver, pm_token pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); PM_NODE_START_SET_NODE(node, receiver); - node->base.location.end = pm_arguments_end(parser, arguments); + const pm_slice_t *end = pm_arguments_end(arguments); + assert(end != NULL && "unreachable"); + PM_NODE_LENGTH_SET_SLICE(node, end); node->receiver = receiver; node->call_operator_loc = TOK2SLICE(parser, operator); @@ -2808,8 +2806,8 @@ pm_call_node_unary_create(pm_parser_t *parser, pm_token_t *operator, pm_node_t * pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); - PM_NODE_START_SET_TOKEN(node, operator); - PM_NODE_END_SET_NODE(node, receiver); + PM_NODE_START_SET_TOKEN(parser, node, operator); + PM_NODE_LENGTH_SET_NODE(node, receiver); node->receiver = receiver; node->message_loc = TOK2SLICE(parser, operator); @@ -2826,7 +2824,7 @@ static pm_call_node_t * pm_call_node_variable_call_create(pm_parser_t *parser, pm_token_t *message) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location = PM_LOCATION_TOKEN_VALUE(message); + node->base.location = TOK2SLICE(parser, message); node->message_loc = TOK2SLICE(parser, message); node->name = pm_parser_constant_id_token(parser, message); @@ -3153,7 +3151,7 @@ pm_case_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node pm_case_node_t *node = PM_NODE_ALLOC(parser, pm_case_node_t); *node = (pm_case_node_t) { - .base = PM_NODE_INIT(parser, PM_CASE_NODE, 0, case_keyword->start, end_keyword == NULL ? case_keyword->end : end_keyword->end), + .base = PM_NODE_INIT_TOKENS(parser, PM_CASE_NODE, 0, case_keyword, end_keyword == NULL ? case_keyword : end_keyword), .predicate = predicate, .else_clause = NULL, .case_keyword_loc = TOK2SLICE(parser, case_keyword), @@ -3172,7 +3170,7 @@ pm_case_node_condition_append(pm_case_node_t *node, pm_node_t *condition) { assert(PM_NODE_TYPE_P(condition, PM_WHEN_NODE)); pm_node_list_append(&node->conditions, condition); - PM_NODE_END_SET_NODE(node, condition); + PM_NODE_LENGTH_SET_NODE(node, condition); } /** @@ -3181,7 +3179,7 @@ pm_case_node_condition_append(pm_case_node_t *node, pm_node_t *condition) { static void pm_case_node_else_clause_set(pm_case_node_t *node, pm_else_node_t *else_clause) { node->else_clause = else_clause; - PM_NODE_END_SET_NODE(node, else_clause); + PM_NODE_LENGTH_SET_NODE(node, else_clause); } /** @@ -3189,7 +3187,7 @@ pm_case_node_else_clause_set(pm_case_node_t *node, pm_else_node_t *else_clause) */ static void pm_case_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_node_t *node, const pm_token_t *end_keyword) { - PM_NODE_END_SET_TOKEN(node, end_keyword); + PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); node->end_keyword_loc = TOK2SLICE(parser, end_keyword); } @@ -3220,7 +3218,7 @@ pm_case_match_node_condition_append(pm_case_match_node_t *node, pm_node_t *condi assert(PM_NODE_TYPE_P(condition, PM_IN_NODE)); pm_node_list_append(&node->conditions, condition); - PM_NODE_END_SET_NODE(node, condition); + PM_NODE_LENGTH_SET_NODE(node, condition); } /** @@ -3229,7 +3227,7 @@ pm_case_match_node_condition_append(pm_case_match_node_t *node, pm_node_t *condi static void pm_case_match_node_else_clause_set(pm_case_match_node_t *node, pm_else_node_t *else_clause) { node->else_clause = else_clause; - PM_NODE_END_SET_NODE(node, else_clause); + PM_NODE_LENGTH_SET_NODE(node, else_clause); } /** @@ -3237,7 +3235,7 @@ pm_case_match_node_else_clause_set(pm_case_match_node_t *node, pm_else_node_t *e */ static void pm_case_match_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_match_node_t *node, const pm_token_t *end_keyword) { - PM_NODE_END_SET_TOKEN(node, end_keyword); + PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); node->end_keyword_loc = TOK2SLICE(parser, end_keyword); } @@ -3274,7 +3272,7 @@ pm_class_variable_and_write_node_create(pm_parser_t *parser, pm_class_variable_r *node = (pm_class_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOC2SLICE(parser, &target->base.location), + .name_loc = target->base.location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3292,7 +3290,7 @@ pm_class_variable_operator_write_node_create(pm_parser_t *parser, pm_class_varia *node = (pm_class_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOC2SLICE(parser, &target->base.location), + .name_loc = target->base.location, .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) @@ -3312,7 +3310,7 @@ pm_class_variable_or_write_node_create(pm_parser_t *parser, pm_class_variable_re *node = (pm_class_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOC2SLICE(parser, &target->base.location), + .name_loc = target->base.location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3361,7 +3359,7 @@ pm_class_variable_write_node_create(pm_parser_t *parser, pm_class_variable_read_ *node = (pm_class_variable_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_WRITE_NODE, flags, read_node, value), .name = read_node->name, - .name_loc = LOC2SLICE(parser, &read_node->base.location), + .name_loc = read_node->base.location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3480,7 +3478,7 @@ pm_constant_and_write_node_create(pm_parser_t *parser, pm_constant_read_node_t * *node = (pm_constant_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_AND_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOC2SLICE(parser, &target->base.location), + .name_loc = target->base.location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3498,7 +3496,7 @@ pm_constant_operator_write_node_create(pm_parser_t *parser, pm_constant_read_nod *node = (pm_constant_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOC2SLICE(parser, &target->base.location), + .name_loc = target->base.location, .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) @@ -3518,7 +3516,7 @@ pm_constant_or_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *t *node = (pm_constant_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_OR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOC2SLICE(parser, &target->base.location), + .name_loc = target->base.location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3553,7 +3551,7 @@ pm_constant_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *targ *node = (pm_constant_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_WRITE_NODE, flags, target, value), .name = target->name, - .name_loc = LOC2SLICE(parser, &target->base.location), + .name_loc = target->base.location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -3776,7 +3774,7 @@ pm_find_pattern_node_create(pm_parser_t *parser, pm_node_list_t *nodes) { pm_node_t *right; if (nodes->size == 1) { - right = UP(pm_missing_node_create(parser, left->location.end, left->location.end)); + right = UP(pm_missing_node_create(parser, PM_NODE_END(left), 0)); } else { right = nodes->nodes[nodes->size - 1]; assert(PM_NODE_TYPE_P(right, PM_SPLAT_NODE)); @@ -3854,7 +3852,7 @@ pm_double_parse(pm_parser_t *parser, const pm_token_t *token) { // This should never happen, because we've already checked that the token // is in a valid format. However it's good to be safe. if ((eptr != buffer + length) || (errno != 0 && errno != ERANGE)) { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, (*token), PM_ERR_FLOAT_PARSE); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, token, PM_ERR_FLOAT_PARSE); xfree((void *) buffer); return 0.0; } @@ -3873,7 +3871,7 @@ pm_double_parse(pm_parser_t *parser, const pm_token_t *token) { ellipsis = ""; } - pm_diagnostic_list_append_format(&parser->warning_list, (uint32_t) (token->start - parser->start), (uint32_t) (token->end - token->start), PM_WARN_FLOAT_OUT_OF_RANGE, warn_width, (const char *) token->start, ellipsis); + pm_diagnostic_list_append_format(&parser->warning_list, PM_TOKEN_START(parser, token), PM_TOKEN_LENGTH(token), PM_WARN_FLOAT_OUT_OF_RANGE, warn_width, (const char *) token->start, ellipsis); value = (value < 0.0) ? -HUGE_VAL : HUGE_VAL; } @@ -4100,25 +4098,25 @@ static pm_hash_pattern_node_t * pm_hash_pattern_node_node_list_create(pm_parser_t *parser, pm_node_list_t *elements, pm_node_t *rest) { pm_hash_pattern_node_t *node = PM_NODE_ALLOC(parser, pm_hash_pattern_node_t); - const uint8_t *start; - const uint8_t *end; + uint32_t start; + uint32_t end; if (elements->size > 0) { if (rest) { - start = elements->nodes[0]->location.start; - end = rest->location.end; + start = PM_NODE_START(elements->nodes[0]); + end = PM_NODE_END(rest); } else { - start = elements->nodes[0]->location.start; - end = elements->nodes[elements->size - 1]->location.end; + start = PM_NODE_START(elements->nodes[0]); + end = PM_NODE_END(elements->nodes[elements->size - 1]); } } else { assert(rest != NULL); - start = rest->location.start; - end = rest->location.end; + start = PM_NODE_START(rest); + end = PM_NODE_END(rest); } *node = (pm_hash_pattern_node_t) { - .base = PM_NODE_INIT(parser, PM_HASH_PATTERN_NODE, 0, start, end), + .base = PM_NODE_INIT(parser, PM_HASH_PATTERN_NODE, 0, start, U32(end - start)), .constant = NULL, .elements = { 0 }, .rest = rest, @@ -4147,7 +4145,7 @@ pm_global_variable_write_name(pm_parser_t *parser, const pm_node_t *target) { case PM_NUMBERED_REFERENCE_READ_NODE: // This will only ever happen in the event of a syntax error, but we // still need to provide something for the node. - return pm_parser_constant_id_location(parser, target->location.start, target->location.end); + return pm_parser_constant_id_location(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target)); default: assert(false && "unreachable"); return (pm_constant_id_t) -1; @@ -4165,7 +4163,7 @@ pm_global_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, *node = (pm_global_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = LOC2SLICE(parser, &target->location), + .name_loc = target->location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4183,7 +4181,7 @@ pm_global_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *ta *node = (pm_global_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = LOC2SLICE(parser, &target->location), + .name_loc = target->location, .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) @@ -4203,7 +4201,7 @@ pm_global_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, *node = (pm_global_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = LOC2SLICE(parser, &target->location), + .name_loc = target->location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4252,7 +4250,7 @@ pm_global_variable_write_node_create(pm_parser_t *parser, pm_node_t *target, con *node = (pm_global_variable_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_WRITE_NODE, flags, target, value), .name = pm_global_variable_write_name(parser, target), - .name_loc = LOC2SLICE(parser, &target->location), + .name_loc = target->location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4318,7 +4316,7 @@ pm_hash_node_elements_append(pm_hash_node_t *hash, pm_node_t *element) { static inline void pm_hash_node_closing_loc_set(const pm_parser_t *parser, pm_hash_node_t *hash, pm_token_t *token) { - PM_NODE_END_SET_TOKEN(hash, token); + PM_NODE_LENGTH_SET_TOKEN(parser, hash, token); hash->closing_loc = TOK2SLICE(parser, token); } @@ -4337,19 +4335,21 @@ pm_if_node_create(pm_parser_t *parser, pm_conditional_predicate(parser, predicate, PM_CONDITIONAL_PREDICATE_TYPE_CONDITIONAL); pm_if_node_t *node = PM_NODE_ALLOC(parser, pm_if_node_t); - const uint8_t *end; + uint32_t start = PM_TOKEN_START(parser, if_keyword); + uint32_t end; + if (end_keyword != NULL) { - end = end_keyword->end; + end = PM_TOKEN_END(parser, end_keyword); } else if (subsequent != NULL) { - end = subsequent->location.end; + end = PM_NODE_END(subsequent); } else if (pm_statements_node_body_length(statements) != 0) { - end = statements->base.location.end; + end = PM_NODE_END(statements); } else { - end = predicate->location.end; + end = PM_NODE_END(predicate); } *node = (pm_if_node_t) { - .base = PM_NODE_INIT(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, if_keyword->start, end), + .base = PM_NODE_INIT(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, start, U32(end - start)), .if_keyword_loc = TOK2SLICE(parser, if_keyword), .predicate = predicate, .then_keyword_loc = NTOK2SLICE(parser, then_keyword), @@ -4418,13 +4418,13 @@ pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_to static inline void pm_if_node_end_keyword_loc_set(const pm_parser_t *parser, pm_if_node_t *node, const pm_token_t *keyword) { - PM_NODE_END_SET_TOKEN(node, keyword); + PM_NODE_LENGTH_SET_TOKEN(parser, node, keyword); node->end_keyword_loc = TOK2SLICE(parser, keyword); } static inline void pm_else_node_end_keyword_loc_set(const pm_parser_t *parser, pm_else_node_t *node, const pm_token_t *keyword) { - PM_NODE_END_SET_TOKEN(node, keyword); + PM_NODE_LENGTH_SET_TOKEN(parser, node, keyword); node->end_keyword_loc = TOK2SLICE(parser, keyword); } @@ -4563,17 +4563,19 @@ static pm_in_node_t * pm_in_node_create(pm_parser_t *parser, pm_node_t *pattern, pm_statements_node_t *statements, const pm_token_t *in_keyword, const pm_token_t *then_keyword) { pm_in_node_t *node = PM_NODE_ALLOC(parser, pm_in_node_t); - const uint8_t *end; + uint32_t start = PM_TOKEN_START(parser, in_keyword); + uint32_t end; + if (statements != NULL) { - end = statements->base.location.end; + end = PM_NODE_END(statements); } else if (then_keyword != NULL) { - end = then_keyword->end; + end = PM_TOKEN_END(parser, then_keyword); } else { - end = pattern->location.end; + end = PM_NODE_END(pattern); } *node = (pm_in_node_t) { - .base = PM_NODE_INIT(parser, PM_IN_NODE, 0, in_keyword->start, end), + .base = PM_NODE_INIT(parser, PM_IN_NODE, 0, start, U32(end - start)), .pattern = pattern, .statements = statements, .in_loc = TOK2SLICE(parser, in_keyword), @@ -4594,7 +4596,7 @@ pm_instance_variable_and_write_node_create(pm_parser_t *parser, pm_instance_vari *node = (pm_instance_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOC2SLICE(parser, &target->base.location), + .name_loc = target->base.location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4612,7 +4614,7 @@ pm_instance_variable_operator_write_node_create(pm_parser_t *parser, pm_instance *node = (pm_instance_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOC2SLICE(parser, &target->base.location), + .name_loc = target->base.location, .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) @@ -4632,7 +4634,7 @@ pm_instance_variable_or_write_node_create(pm_parser_t *parser, pm_instance_varia *node = (pm_instance_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = target->name, - .name_loc = LOC2SLICE(parser, &target->base.location), + .name_loc = target->base.location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4668,7 +4670,7 @@ pm_instance_variable_write_node_create(pm_parser_t *parser, pm_instance_variable *node = (pm_instance_variable_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_WRITE_NODE, flags, read_node, value), .name = read_node->name, - .name_loc = LOC2SLICE(parser, &read_node->base.location), + .name_loc = read_node->base.location, .operator_loc = TOK2SLICE(parser, operator), .value = value }; @@ -4742,8 +4744,8 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio if (PM_NODE_START(node) > PM_NODE_START(part)) { PM_NODE_START_SET_NODE(node, part); } - if (node->base.location.end < part->location.end) { - PM_NODE_END_SET_NODE(node, part); + if (PM_NODE_END(node) < PM_NODE_END(part)) { + PM_NODE_LENGTH_SET_NODE(node, part); } pm_interpolated_node_append(UP(node), &node->parts, part); @@ -4752,7 +4754,7 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio static inline void pm_interpolated_regular_expression_node_closing_set(pm_parser_t *parser, pm_interpolated_regular_expression_node_t *node, const pm_token_t *closing) { node->closing_loc = TOK2SLICE(parser, closing); - PM_NODE_END_SET_TOKEN(node, closing); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); pm_node_flag_set(UP(node), pm_regular_expression_flags_create(parser, closing)); } @@ -4791,7 +4793,9 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ PM_NODE_START_SET_NODE(node, part); } - node->base.location.end = MAX(node->base.location.end, part->location.end); + if (PM_NODE_END(part) > PM_NODE_END(node)) { + PM_NODE_LENGTH_SET_NODE(node, part); + } switch (PM_NODE_TYPE(part)) { case PM_STRING_NODE: @@ -4886,8 +4890,11 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin break; } + uint32_t start = opening == NULL ? 0 : PM_TOKEN_START(parser, opening); + uint32_t end = closing == NULL ? 0 : PM_TOKEN_END(parser, closing); + *node = (pm_interpolated_string_node_t) { - .base = PM_NODE_INIT(parser, PM_INTERPOLATED_STRING_NODE, flags, opening == NULL ? NULL : opening->start, closing == NULL ? NULL : closing->end), + .base = PM_NODE_INIT(parser, PM_INTERPOLATED_STRING_NODE, flags, start, U32(end - start)), .opening_loc = NTOK2SLICE(parser, opening), .closing_loc = NTOK2SLICE(parser, closing), .parts = { 0 } @@ -4909,7 +4916,7 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin static void pm_interpolated_string_node_closing_set(const pm_parser_t *parser, pm_interpolated_string_node_t *node, const pm_token_t *closing) { node->closing_loc = TOK2SLICE(parser, closing); - PM_NODE_END_SET_TOKEN(node, closing); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); } static void @@ -4919,13 +4926,16 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ } pm_interpolated_node_append(UP(node), &node->parts, part); - node->base.location.end = MAX(node->base.location.end, part->location.end); + + if (PM_NODE_END(part) > PM_NODE_END(node)) { + PM_NODE_LENGTH_SET_NODE(node, part); + } } static void pm_interpolated_symbol_node_closing_loc_set(const pm_parser_t *parser, pm_interpolated_symbol_node_t *node, const pm_token_t *closing) { node->closing_loc = TOK2SLICE(parser, closing); - PM_NODE_END_SET_TOKEN(node, closing); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); } /** @@ -4935,8 +4945,11 @@ static pm_interpolated_symbol_node_t * pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_node_list_t *parts, const pm_token_t *closing) { pm_interpolated_symbol_node_t *node = PM_NODE_ALLOC(parser, pm_interpolated_symbol_node_t); + uint32_t start = opening == NULL ? 0 : PM_TOKEN_START(parser, opening); + uint32_t end = closing == NULL ? 0 : PM_TOKEN_END(parser, closing); + *node = (pm_interpolated_symbol_node_t) { - .base = PM_NODE_INIT(parser, PM_INTERPOLATED_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening == NULL ? NULL : opening->start, closing == NULL ? NULL : closing->end), + .base = PM_NODE_INIT(parser, PM_INTERPOLATED_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, start, U32(end - start)), .opening_loc = NTOK2SLICE(parser, opening), .closing_loc = NTOK2SLICE(parser, closing), .parts = { 0 } @@ -4972,13 +4985,13 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi static inline void pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_node_t *part) { pm_interpolated_node_append(UP(node), &node->parts, part); - PM_NODE_END_SET_NODE(node, part); + PM_NODE_LENGTH_SET_NODE(node, part); } static inline void pm_interpolated_xstring_node_closing_set(const pm_parser_t *parser, pm_interpolated_x_string_node_t *node, const pm_token_t *closing) { node->closing_loc = TOK2SLICE(parser, closing); - PM_NODE_END_SET_TOKEN(node, closing); + PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); } /** @@ -5036,10 +5049,10 @@ pm_keyword_hash_node_elements_append(pm_keyword_hash_node_t *hash, pm_node_t *el } pm_node_list_append(&hash->elements, element); - if (PM_NODE_START(hash) == NULL) { + if (PM_NODE_LENGTH(hash) == 0) { PM_NODE_START_SET_NODE(hash, element); } - PM_NODE_END_SET_NODE(hash, element); + PM_NODE_LENGTH_SET_NODE(hash, element); } /** @@ -5135,7 +5148,7 @@ pm_local_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, *node = (pm_local_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_AND_WRITE_NODE, 0, target, value), - .name_loc = LOC2SLICE(parser, &target->location), + .name_loc = target->location, .operator_loc = TOK2SLICE(parser, operator), .value = value, .name = name, @@ -5154,7 +5167,7 @@ pm_local_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *tar *node = (pm_local_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), - .name_loc = LOC2SLICE(parser, &target->location), + .name_loc = target->location, .binary_operator_loc = TOK2SLICE(parser, operator), .value = value, .name = name, @@ -5176,7 +5189,7 @@ pm_local_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, c *node = (pm_local_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_OR_WRITE_NODE, 0, target, value), - .name_loc = LOC2SLICE(parser, &target->location), + .name_loc = target->location, .operator_loc = TOK2SLICE(parser, operator), .value = value, .name = name, @@ -5227,16 +5240,16 @@ pm_local_variable_read_node_missing_create(pm_parser_t *parser, const pm_token_t * Allocate and initialize a new LocalVariableWriteNode node. */ static pm_local_variable_write_node_t * -pm_local_variable_write_node_create(pm_parser_t *parser, pm_constant_id_t name, uint32_t depth, pm_node_t *value, const pm_location_t *name_loc, const pm_token_t *operator) { +pm_local_variable_write_node_create(pm_parser_t *parser, pm_constant_id_t name, uint32_t depth, pm_node_t *value, const pm_slice_t *name_loc, const pm_token_t *operator) { pm_local_variable_write_node_t *node = PM_NODE_ALLOC(parser, pm_local_variable_write_node_t); pm_node_flags_t flags = pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY); *node = (pm_local_variable_write_node_t) { - .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_LOCAL_VARIABLE_WRITE_NODE, flags, name_loc, value), + .base = PM_NODE_INIT(parser, PM_LOCAL_VARIABLE_WRITE_NODE, flags, name_loc->start, PM_NODE_END(value) - name_loc->start), .name = name, .depth = depth, .value = value, - .name_loc = LOC2SLICE(parser, name_loc), + .name_loc = *name_loc, .operator_loc = TOK2SLICE(parser, operator) }; @@ -5272,7 +5285,7 @@ pm_token_is_numbered_parameter(const pm_parser_t *parser, uint32_t start, uint32 static inline void pm_refute_numbered_parameter(pm_parser_t *parser, uint32_t start, uint32_t length) { if (pm_token_is_numbered_parameter(parser, start, length)) { - PM_PARSER_ERR_FORMAT_SLICE(parser, start, length, PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + start); + PM_PARSER_ERR_FORMAT(parser, start, length, PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + start); } } @@ -5281,12 +5294,12 @@ pm_refute_numbered_parameter(pm_parser_t *parser, uint32_t start, uint32_t lengt * name and depth. */ static pm_local_variable_target_node_t * -pm_local_variable_target_node_create(pm_parser_t *parser, const pm_location_t *location, pm_constant_id_t name, uint32_t depth) { - pm_refute_numbered_parameter(parser, U32(location->start - parser->start), PM_TOKEN_LENGTH(location)); +pm_local_variable_target_node_create(pm_parser_t *parser, const pm_slice_t *location, pm_constant_id_t name, uint32_t depth) { + pm_refute_numbered_parameter(parser, location->start, location->length); pm_local_variable_target_node_t *node = PM_NODE_ALLOC(parser, pm_local_variable_target_node_t); *node = (pm_local_variable_target_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_LOCAL_VARIABLE_TARGET_NODE, 0, location), + .base = PM_NODE_INIT(parser, PM_LOCAL_VARIABLE_TARGET_NODE, 0, location->start, location->length), .name = name, .depth = depth }; @@ -5403,7 +5416,7 @@ pm_multi_target_node_targets_append(pm_parser_t *parser, pm_multi_target_node_t if (node->rest == NULL) { node->rest = target; } else { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->current, PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST); pm_node_list_append(&node->rights, target); } } else if (node->rest == NULL) { @@ -5412,12 +5425,12 @@ pm_multi_target_node_targets_append(pm_parser_t *parser, pm_multi_target_node_t pm_node_list_append(&node->rights, target); } - if (node->base.location.start == NULL || (node->base.location.start > target->location.start)) { + if (PM_NODE_LENGTH(node) == 0 || (PM_NODE_START(node) > PM_NODE_START(target))) { PM_NODE_START_SET_NODE(node, target); } - if (node->base.location.end == NULL || (node->base.location.end < target->location.end)) { - PM_NODE_END_SET_NODE(node, target); + if (PM_NODE_LENGTH(node) == 0 || (PM_NODE_END(node) < PM_NODE_END(target))) { + PM_NODE_LENGTH_SET_NODE(node, target); } } @@ -5426,7 +5439,8 @@ pm_multi_target_node_targets_append(pm_parser_t *parser, pm_multi_target_node_t */ static void pm_multi_target_node_opening_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *lparen) { - PM_NODE_START_SET_TOKEN(node, lparen); + PM_NODE_START_SET_TOKEN(parser, node, lparen); + PM_NODE_LENGTH_SET_TOKEN(parser, node, lparen); node->lparen_loc = TOK2SLICE(parser, lparen); } @@ -5435,7 +5449,7 @@ pm_multi_target_node_opening_set(const pm_parser_t *parser, pm_multi_target_node */ static void pm_multi_target_node_closing_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *rparen) { - PM_NODE_END_SET_TOKEN(node, rparen); + PM_NODE_LENGTH_SET_TOKEN(parser, node, rparen); node->rparen_loc = TOK2SLICE(parser, rparen); } @@ -5523,11 +5537,11 @@ pm_no_keywords_parameter_node_create(pm_parser_t *parser, const pm_token_t *oper * Allocate and initialize a new NumberedParametersNode node. */ static pm_numbered_parameters_node_t * -pm_numbered_parameters_node_create(pm_parser_t *parser, const pm_location_t *location, uint8_t maximum) { +pm_numbered_parameters_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *closing, uint8_t maximum) { pm_numbered_parameters_node_t *node = PM_NODE_ALLOC(parser, pm_numbered_parameters_node_t); *node = (pm_numbered_parameters_node_t) { - .base = PM_NODE_INIT_TOKEN(parser, PM_NUMBERED_PARAMETERS_NODE, 0, location), + .base = PM_NODE_INIT_TOKENS(parser, PM_NUMBERED_PARAMETERS_NODE, 0, opening, closing), .maximum = maximum }; @@ -5574,7 +5588,7 @@ pm_numbered_reference_read_node_number(pm_parser_t *parser, const pm_token_t *to xfree(digits); if ((errno == ERANGE) || (value > NTH_REF_MAX)) { - PM_PARSER_WARN_FORMAT_LOCATION(parser, start, end, PM_WARN_INVALID_NUMBERED_REFERENCE, (int) (length + 1), (const char *) token->start); + PM_PARSER_WARN_FORMAT(parser, U32(start - parser->start), U32(length), PM_WARN_INVALID_NUMBERED_REFERENCE, (int) (length + 1), (const char *) token->start); value = 0; } @@ -5662,12 +5676,12 @@ pm_parameters_node_create(pm_parser_t *parser) { */ static void pm_parameters_node_location_set(pm_parameters_node_t *params, pm_node_t *param) { - if ((params->base.location.start == NULL) || PM_NODE_START(params) > PM_NODE_START(param)) { + if ((params->base.location.length == 0) || PM_NODE_START(params) > PM_NODE_START(param)) { PM_NODE_START_SET_NODE(params, param); } - if ((params->base.location.end == NULL) || (params->base.location.end < param->location.end)) { - PM_NODE_END_SET_NODE(params, param); + if ((params->base.location.length == 0) || (PM_NODE_END(params) < PM_NODE_END(param))) { + PM_NODE_LENGTH_SET_NODE(params, param); } } @@ -5865,8 +5879,11 @@ pm_range_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *ope flags |= PM_NODE_FLAG_STATIC_LITERAL; } + uint32_t start = left == NULL ? PM_TOKEN_START(parser, operator) : PM_NODE_START(left); + uint32_t end = right == NULL ? PM_TOKEN_END(parser, operator) : PM_NODE_END(right); + *node = (pm_range_node_t) { - .base = PM_NODE_INIT(parser, PM_RANGE_NODE, flags, (left == NULL ? operator->start : left->location.start), (right == NULL ? operator->end : right->location.end)), + .base = PM_NODE_INIT(parser, PM_RANGE_NODE, flags, start, U32(end - start)), .left = left, .right = right, .operator_loc = TOK2SLICE(parser, operator) @@ -5982,7 +5999,7 @@ pm_rescue_node_operator_set(const pm_parser_t *parser, pm_rescue_node_t *node, c static void pm_rescue_node_reference_set(pm_rescue_node_t *node, pm_node_t *reference) { node->reference = reference; - PM_NODE_END_SET_NODE(node, reference); + PM_NODE_LENGTH_SET_NODE(node, reference); } /** @@ -5992,7 +6009,7 @@ static void pm_rescue_node_statements_set(pm_rescue_node_t *node, pm_statements_node_t *statements) { node->statements = statements; if (pm_statements_node_body_length(statements) > 0) { - PM_NODE_END_SET_NODE(node, statements); + PM_NODE_LENGTH_SET_NODE(node, statements); } } @@ -6002,7 +6019,7 @@ pm_rescue_node_statements_set(pm_rescue_node_t *node, pm_statements_node_t *stat static void pm_rescue_node_subsequent_set(pm_rescue_node_t *node, pm_rescue_node_t *subsequent) { node->subsequent = subsequent; - PM_NODE_END_SET_NODE(node, subsequent); + PM_NODE_LENGTH_SET_NODE(node, subsequent); } /** @@ -6011,7 +6028,7 @@ pm_rescue_node_subsequent_set(pm_rescue_node_t *node, pm_rescue_node_t *subseque static void pm_rescue_node_exceptions_append(pm_rescue_node_t *node, pm_node_t *exception) { pm_node_list_append(&node->exceptions, exception); - PM_NODE_END_SET_NODE(node, exception); + PM_NODE_LENGTH_SET_NODE(node, exception); } /** @@ -6220,14 +6237,6 @@ pm_statements_node_body_length(pm_statements_node_t *node) { return node && node->body.size; } -/** - * Set the location of the given StatementsNode. - */ -static void -pm_statements_node_location_set(pm_statements_node_t *node, const uint8_t *start, const uint8_t *end) { - node->base.location = (pm_location_t) { .start = start, .end = end }; -} - /** * Update the location of the statements node based on the statement that is * being added to the list. @@ -6239,7 +6248,7 @@ pm_statements_node_body_update(pm_statements_node_t *node, pm_node_t *statement) } if (PM_NODE_END(statement) > PM_NODE_END(node)) { - PM_NODE_END_SET_NODE(node, statement); + PM_NODE_LENGTH_SET_NODE(node, statement); } } @@ -6297,11 +6306,11 @@ pm_string_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, break; } - const uint8_t *start = opening == NULL ? content->start : opening->start; - const uint8_t *end = closing == NULL ? content->end : closing->end; + uint32_t start = PM_TOKEN_START(parser, opening == NULL ? content : opening); + uint32_t end = PM_TOKEN_END(parser, closing == NULL ? content : closing); *node = (pm_string_node_t) { - .base = PM_NODE_INIT(parser, PM_STRING_NODE, flags, start, end), + .base = PM_NODE_INIT(parser, PM_STRING_NODE, flags, start, U32(end - start)), .opening_loc = NTOK2SLICE(parser, opening), .content_loc = TOK2SLICE(parser, content), .closing_loc = NTOK2SLICE(parser, closing), @@ -6338,13 +6347,11 @@ pm_super_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument assert(keyword->type == PM_TOKEN_KEYWORD_SUPER); pm_super_node_t *node = PM_NODE_ALLOC(parser, pm_super_node_t); - const uint8_t *end = pm_arguments_end(parser, arguments); - if (end == NULL) { - assert(false && "unreachable"); - } + const pm_slice_t *end = pm_arguments_end(arguments); + assert(end != NULL && "unreachable"); *node = (pm_super_node_t) { - .base = PM_NODE_INIT(parser, PM_SUPER_NODE, 0, keyword->start, end), + .base = PM_NODE_INIT(parser, PM_SUPER_NODE, 0, PM_TOKEN_START(parser, keyword), PM_SLICE_END(end) - PM_TOKEN_START(parser, keyword)), .keyword_loc = TOK2SLICE(parser, keyword), .lparen_loc = arguments->opening_loc, .arguments = arguments->arguments, @@ -6380,7 +6387,7 @@ parse_symbol_encoding_validate_utf8(pm_parser_t *parser, const pm_token_t *locat size_t width = pm_encoding_utf_8_char_width(cursor, end - cursor); if (width == 0) { - pm_parser_err(parser, U32(location->start - parser->start), PM_TOKEN_LENGTH(location), PM_ERR_INVALID_SYMBOL); + pm_parser_err(parser, PM_TOKEN_START(parser, location), PM_TOKEN_LENGTH(location), PM_ERR_INVALID_SYMBOL); break; } @@ -6400,7 +6407,7 @@ parse_symbol_encoding_validate_other(pm_parser_t *parser, const pm_token_t *loca size_t width = encoding->char_width(cursor, end - cursor); if (width == 0) { - pm_parser_err(parser, U32(location->start - parser->start), PM_TOKEN_LENGTH(location), PM_ERR_INVALID_SYMBOL); + pm_parser_err(parser, PM_TOKEN_START(parser, location), PM_TOKEN_LENGTH(location), PM_ERR_INVALID_SYMBOL); break; } @@ -6460,13 +6467,13 @@ parse_and_validate_regular_expression_encoding_modifier(pm_parser_t *parser, con if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) { if (!ascii_only) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name); } } else if (parser->encoding != modifier_encoding) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH, modifier, parser->encoding->name); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH, modifier, parser->encoding->name); if (modifier == 'n' && !ascii_only) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_REGEXP_NON_ESCAPED_MBC, (int) pm_string_length(source), (const char *) pm_string_source(source)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_REGEXP_NON_ESCAPED_MBC, (int) pm_string_length(source), (const char *) pm_string_source(source)); } } @@ -6477,18 +6484,18 @@ parse_and_validate_regular_expression_encoding_modifier(pm_parser_t *parser, con bool mixed_encoding = false; if (mixed_encoding) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_MULTIBYTE_ESCAPE, (int) pm_string_length(source), (const char *) pm_string_source(source)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_ESCAPE, (int) pm_string_length(source), (const char *) pm_string_source(source)); } else if (modifier != 'n' && parser->explicit_encoding == PM_ENCODING_ASCII_8BIT_ENTRY) { // TODO (nirvdrum 21-Feb-2024): Validate the content is valid in the modifier encoding. Do this on-demand so we don't pay the cost of computation unnecessarily. bool valid_string_in_modifier_encoding = true; if (!valid_string_in_modifier_encoding) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_MULTIBYTE_ESCAPE, (int) pm_string_length(source), (const char *) pm_string_source(source)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_ESCAPE, (int) pm_string_length(source), (const char *) pm_string_source(source)); } } else if (modifier != 'u' && parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) { // TODO (nirvdrum 21-Feb-2024): There's currently no way to tell if the source used hex or Unicode character escapes from `explicit_encoding` alone. If the source encoding was already UTF-8, both character escape types would set `explicit_encoding` to UTF-8, but need to be processed differently. Skip for now. if (parser->encoding != PM_ENCODING_UTF_8_ENTRY) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, (int) pm_string_length(source), (const char *) pm_string_source(source)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, (int) pm_string_length(source), (const char *) pm_string_source(source)); } } @@ -6507,7 +6514,7 @@ parse_and_validate_regular_expression_encoding(pm_parser_t *parser, const pm_str // TODO (nirvdrum 22-Feb-2024): CRuby reports a special Regexp-specific error for invalid Unicode ranges. We either need to scan again or modify the "invalid Unicode escape sequence" message we already report. bool valid_unicode_range = true; if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY && !valid_unicode_range) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_REGEXP_INVALID_UNICODE_RANGE, (int) pm_string_length(source), (const char *) pm_string_source(source)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_REGEXP_INVALID_UNICODE_RANGE, (int) pm_string_length(source), (const char *) pm_string_source(source)); return flags; } @@ -6516,7 +6523,7 @@ parse_and_validate_regular_expression_encoding(pm_parser_t *parser, const pm_str if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY && parser->explicit_encoding == NULL && !ascii_only) { // CRuby will continue processing even though a SyntaxError has already been detected. It may result in the // following error message appearing twice. We do the same for compatibility. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name); } /** @@ -6573,11 +6580,11 @@ static pm_symbol_node_t * pm_symbol_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, const pm_token_t *value, const pm_token_t *closing, const pm_string_t *unescaped, pm_node_flags_t flags) { pm_symbol_node_t *node = PM_NODE_ALLOC(parser, pm_symbol_node_t); - const uint8_t *start = opening == NULL ? value->start : opening->start; - const uint8_t *end = closing == NULL ? value->end : closing->end; + uint32_t start = opening == NULL ? PM_TOKEN_START(parser, value) : PM_TOKEN_START(parser, opening); + uint32_t end = closing == NULL ? PM_TOKEN_END(parser, value) : PM_TOKEN_END(parser, closing); *node = (pm_symbol_node_t) { - .base = PM_NODE_INIT(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL | flags, start, end), + .base = PM_NODE_INIT(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL | flags, start, U32(end - start)), .opening_loc = NTOK2SLICE(parser, opening), .value_loc = NTOK2SLICE(parser, value), .closing_loc = NTOK2SLICE(parser, closing), @@ -6784,7 +6791,7 @@ pm_undef_node_create(pm_parser_t *parser, const pm_token_t *token) { */ static void pm_undef_node_append(pm_undef_node_t *node, pm_node_t *name) { - PM_NODE_END_SET_NODE(node, name); + PM_NODE_LENGTH_SET_NODE(node, name); pm_node_list_append(&node->names, name); } @@ -6838,7 +6845,7 @@ pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const static inline void pm_unless_node_end_keyword_loc_set(const pm_parser_t *parser, pm_unless_node_t *node, const pm_token_t *end_keyword) { node->end_keyword_loc = TOK2SLICE(parser, end_keyword); - PM_NODE_END_SET_TOKEN(node, end_keyword); + PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); } /** @@ -6853,7 +6860,7 @@ pm_loop_modifier_block_exits(pm_parser_t *parser, pm_statements_node_t *statemen // All of the block exits that we want to remove should be within the // statements, and since we are modifying the statements, we shouldn't have // to check the end location. - const uint8_t *start = statements->base.location.start; + uint32_t start = statements->base.location.start; for (size_t index = parser->current_block_exits->size; index > 0; index--) { pm_node_t *block_exit = parser->current_block_exits->nodes[index - 1]; @@ -6928,7 +6935,7 @@ pm_when_node_create(pm_parser_t *parser, const pm_token_t *keyword) { */ static void pm_when_node_conditions_append(pm_when_node_t *node, pm_node_t *condition) { - PM_NODE_END_SET_NODE(node, condition); + PM_NODE_LENGTH_SET_NODE(node, condition); pm_node_list_append(&node->conditions, condition); } @@ -6937,7 +6944,7 @@ pm_when_node_conditions_append(pm_when_node_t *node, pm_node_t *condition) { */ static inline void pm_when_node_then_keyword_loc_set(const pm_parser_t *parser, pm_when_node_t *node, const pm_token_t *then_keyword) { - PM_NODE_END_SET_TOKEN(node, then_keyword); + PM_NODE_LENGTH_SET_TOKEN(parser, node, then_keyword); node->then_keyword_loc = TOK2SLICE(parser, then_keyword); } @@ -6946,8 +6953,8 @@ pm_when_node_then_keyword_loc_set(const pm_parser_t *parser, pm_when_node_t *nod */ static void pm_when_node_statements_set(pm_when_node_t *node, pm_statements_node_t *statements) { - if (statements->base.location.end > node->base.location.end) { - PM_NODE_END_SET_NODE(node, statements); + if (PM_NODE_END(statements) > PM_NODE_END(node)) { + PM_NODE_LENGTH_SET_NODE(node, statements); } node->statements = statements; @@ -7047,19 +7054,21 @@ static pm_yield_node_t * pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_slice_t *lparen_loc, pm_arguments_node_t *arguments, const pm_slice_t *rparen_loc) { pm_yield_node_t *node = PM_NODE_ALLOC(parser, pm_yield_node_t); - const uint8_t *end; + uint32_t start = PM_TOKEN_START(parser, keyword); + uint32_t end; + if (rparen_loc->length > 0) { - end = parser->start + rparen_loc->start + rparen_loc->length; + end = PM_SLICE_END(rparen_loc); } else if (arguments != NULL) { - end = arguments->base.location.end; + end = PM_NODE_END(arguments); } else if (lparen_loc->length > 0) { - end = parser->start + lparen_loc->start + lparen_loc->length; + end = PM_SLICE_END(lparen_loc); } else { - end = keyword->end; + end = PM_TOKEN_END(parser, keyword); } *node = (pm_yield_node_t) { - .base = PM_NODE_INIT(parser, PM_YIELD_NODE, 0, keyword->start, end), + .base = PM_NODE_INIT(parser, PM_YIELD_NODE, 0, start, U32(end - start)), .keyword_loc = TOK2SLICE(parser, keyword), .lparen_loc = *lparen_loc, .arguments = arguments, @@ -7104,7 +7113,7 @@ pm_parser_local_depth(pm_parser_t *parser, pm_token_t *token) { */ static inline void pm_parser_local_add(pm_parser_t *parser, pm_constant_id_t constant_id, const uint8_t *start, const uint8_t *end, uint32_t reads) { - pm_locals_write(&parser->current_scope->locals, constant_id, (uint32_t) (start - parser->start), (uint32_t) (end - start), reads); + pm_locals_write(&parser->current_scope->locals, constant_id, U32(start - parser->start), U32(end - start), reads); } /** @@ -7164,7 +7173,7 @@ static bool pm_parser_parameter_name_check(pm_parser_t *parser, const pm_token_t *name) { // We want to check whether the parameter name is a numbered parameter or // not. - pm_refute_numbered_parameter(parser, (uint32_t) (name->start - parser->start), PM_TOKEN_LENGTH(name)); + pm_refute_numbered_parameter(parser, PM_TOKEN_START(parser, name), PM_TOKEN_LENGTH(name)); // Otherwise we'll fetch the constant id for the parameter name and check // whether it's already in the current scope. @@ -7597,7 +7606,7 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { case PM_MAGIC_COMMENT_BOOLEAN_VALUE_INVALID: PM_PARSER_WARN_TOKEN_FORMAT( parser, - parser->current, + &parser->current, PM_WARN_INVALID_MAGIC_COMMENT_VALUE, (int) key_length, (const char *) key_source, @@ -7624,7 +7633,7 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { case PM_MAGIC_COMMENT_BOOLEAN_VALUE_INVALID: PM_PARSER_WARN_TOKEN_FORMAT( parser, - parser->current, + &parser->current, PM_WARN_INVALID_MAGIC_COMMENT_VALUE, (int) key_length, (const char *) key_source, @@ -7659,7 +7668,7 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { } else { PM_PARSER_WARN_TOKEN_FORMAT( parser, - parser->current, + &parser->current, PM_WARN_INVALID_MAGIC_COMMENT_VALUE, (int) key_length, (const char *) key_source, @@ -7677,8 +7686,8 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { // Allocate a new magic comment node to append to the parser's list. pm_magic_comment_t *magic_comment; if ((magic_comment = (pm_magic_comment_t *) xcalloc(1, sizeof(pm_magic_comment_t))) != NULL) { - magic_comment->key = (pm_slice_t) { .start = (uint32_t) (key_start - parser->start), .length = (uint32_t) key_length }; - magic_comment->value = (pm_slice_t) { .start = (uint32_t) (value_start - parser->start), .length = value_length }; + magic_comment->key = (pm_slice_t) { .start = U32(key_start - parser->start), .length = U32(key_length) }; + magic_comment->value = (pm_slice_t) { .start = U32(value_start - parser->start), .length = value_length }; pm_list_append(&parser->magic_comment_list, (pm_list_node_t *) magic_comment); } } @@ -8201,7 +8210,7 @@ lex_global_variable(pm_parser_t *parser) { // $0 isn't allowed to be followed by anything. pm_diagnostic_id_t diag_id = parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3 : PM_ERR_INVALID_VARIABLE_GLOBAL; - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->current, diag_id); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &parser->current, diag_id); } return PM_TOKEN_GLOBAL_VARIABLE; @@ -8238,8 +8247,8 @@ lex_global_variable(pm_parser_t *parser) { // If we get here, then we have a $ followed by something that // isn't recognized as a global variable. pm_diagnostic_id_t diag_id = parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3 : PM_ERR_INVALID_VARIABLE_GLOBAL; - const uint8_t *end = parser->current.end + parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - PM_PARSER_ERR_FORMAT(parser, parser->current.start, end, diag_id, (int) (end - parser->current.start), (const char *) parser->current.start); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + PM_PARSER_ERR_FORMAT(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), diag_id, (int) (PM_TOKEN_LENGTH(&parser->current) + U32(width)), (const char *) parser->current.start); } return PM_TOKEN_GLOBAL_VARIABLE; @@ -8647,7 +8656,7 @@ escape_write_unicode(pm_parser_t *parser, pm_buffer_t *buffer, const uint8_t fla // literal. if (value >= 0x80 || flags & PM_ESCAPE_FLAG_SINGLE) { if (parser->explicit_encoding != NULL && parser->explicit_encoding != PM_ENCODING_UTF_8_ENTRY) { - PM_PARSER_ERR_FORMAT(parser, start, end, PM_ERR_MIXED_ENCODING, parser->explicit_encoding->name); + PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(end - start), PM_ERR_MIXED_ENCODING, parser->explicit_encoding->name); } parser->explicit_encoding = PM_ENCODING_UTF_8_ENTRY; @@ -8669,7 +8678,7 @@ static inline void escape_write_byte_encoded(pm_parser_t *parser, pm_buffer_t *buffer, uint8_t byte) { if (byte >= 0x80) { if (parser->explicit_encoding != NULL && parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY && parser->encoding != PM_ENCODING_UTF_8_ENTRY) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_MIXED_ENCODING, parser->encoding->name); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_MIXED_ENCODING, parser->encoding->name); } parser->explicit_encoding = parser->encoding; @@ -8740,7 +8749,7 @@ escape_read_warn(pm_parser_t *parser, uint8_t flags, uint8_t flag, const char *t PM_PARSER_WARN_TOKEN_FORMAT( parser, - parser->current, + &parser->current, PM_WARN_INVALID_CHARACTER, FLAG(flags), FLAG(flag), @@ -8868,7 +8877,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre if (parser->current.end == parser->end) { const uint8_t *start = parser->current.end - 2; - PM_PARSER_ERR_FORMAT(parser, start, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_SHORT, 2, start); + PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(parser->current.end - start), PM_ERR_ESCAPE_INVALID_UNICODE_SHORT, 2, start); } else if (peek(parser) == '{') { const uint8_t *unicode_codepoints_start = parser->current.end - 2; parser->current.end++; @@ -8907,8 +8916,8 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre // error instead of us. pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); } else { - pm_parser_err(parser, U32(parser->current.end - parser->start), 0, PM_ERR_ESCAPE_INVALID_UNICODE); - pm_parser_err(parser, U32(parser->current.end - parser->start), 0, PM_ERR_ESCAPE_INVALID_UNICODE_TERM); + pm_parser_err(parser, PM_TOKEN_END(parser, &parser->current), 0, PM_ERR_ESCAPE_INVALID_UNICODE); + pm_parser_err(parser, PM_TOKEN_END(parser, &parser->current), 0, PM_ERR_ESCAPE_INVALID_UNICODE_TERM); } return; @@ -8933,7 +8942,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre } if (parser->current.end == parser->end) { - PM_PARSER_ERR_FORMAT(parser, start, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (parser->current.end - start), start); + PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(parser->current.end - start), PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (parser->current.end - start), start); } else if (peek(parser) == '}') { parser->current.end++; } else { @@ -8958,7 +8967,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre pm_buffer_append_bytes(regular_expression_buffer, start, (size_t) (parser->current.end - start)); } else { const uint8_t *start = parser->current.end - 2; - PM_PARSER_ERR_FORMAT(parser, start, parser->current.end, PM_ERR_ESCAPE_INVALID_UNICODE_SHORT, 2, start); + PM_PARSER_ERR_FORMAT(parser, U32(start - parser->start), U32(parser->current.end - start), PM_ERR_ESCAPE_INVALID_UNICODE_SHORT, 2, start); } } else if (length == 4) { uint32_t value = escape_unicode(parser, parser->current.end, 4); @@ -9007,7 +9016,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; if (match(parser, 'u') || match(parser, 'U')) { - pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); return; } @@ -9043,7 +9052,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre if (peek(parser) != '-') { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_CONTROL); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_CONTROL); return; } @@ -9064,7 +9073,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; if (match(parser, 'u') || match(parser, 'U')) { - pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); return; } @@ -9083,7 +9092,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre default: { if (!char_is_ascii_printable(peeked)) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_CONTROL); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_CONTROL); return; } @@ -9101,7 +9110,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre if (peek(parser) != '-') { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); return; } @@ -9117,7 +9126,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre parser->current.end++; if (match(parser, 'u') || match(parser, 'U')) { - pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current), PM_ERR_INVALID_ESCAPE_CHARACTER); return; } @@ -9136,7 +9145,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre default: if (!char_is_ascii_printable(peeked)) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); return; } @@ -9156,7 +9165,7 @@ escape_read(pm_parser_t *parser, pm_buffer_t *buffer, pm_buffer_t *regular_expre default: { if ((flags & (PM_ESCAPE_FLAG_CONTROL | PM_ESCAPE_FLAG_META)) && !char_is_ascii_printable(peeked)) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); - pm_parser_err(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); + pm_parser_err(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current) + U32(width), PM_ERR_ESCAPE_INVALID_META); return; } if (parser->current.end < parser->end) { @@ -9269,7 +9278,7 @@ lex_at_variable(pm_parser_t *parser) { } size_t width = parser->encoding->char_width(parser->current.end, end - parser->current.end); - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start); } else { pm_diagnostic_id_t diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_CLASS_VARIABLE_BARE : PM_ERR_INSTANCE_VARIABLE_BARE; pm_parser_err_token(parser, &parser->current, diag_id); @@ -9323,7 +9332,7 @@ lex_embdoc(pm_parser_t *parser) { if (newline == NULL) { parser->current.end = parser->end; } else { - pm_newline_list_append(&parser->newline_list, (uint32_t) (newline - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, U32(newline - parser->start + 1)); parser->current.end = newline + 1; } @@ -9357,7 +9366,7 @@ lex_embdoc(pm_parser_t *parser) { if (newline == NULL) { parser->current.end = parser->end; } else { - pm_newline_list_append(&parser->newline_list, (uint32_t) (newline - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, U32(newline - parser->start + 1)); parser->current.end = newline + 1; } @@ -9377,7 +9386,7 @@ lex_embdoc(pm_parser_t *parser) { if (newline == NULL) { parser->current.end = parser->end; } else { - pm_newline_list_append(&parser->newline_list, (uint32_t) (newline - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, U32(newline - parser->start + 1)); parser->current.end = newline + 1; } @@ -9691,7 +9700,7 @@ pm_lex_percent_delimiter(pm_parser_t *parser) { parser_flush_heredoc_end(parser); } else { // Otherwise, we'll add the newline to the list of newlines. - pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end + eol_length - parser->start)); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current) + U32(eol_length)); } uint8_t delimiter = *parser->current.end; @@ -9776,7 +9785,7 @@ parser_lex(pm_parser_t *parser) { if (match_eol_offset(parser, 1)) { chomping = false; } else { - pm_parser_warn(parser, U32(parser->current.end - parser->start), 1, PM_WARN_UNEXPECTED_CARRIAGE_RETURN); + pm_parser_warn(parser, PM_TOKEN_END(parser, &parser->current), 1, PM_WARN_UNEXPECTED_CARRIAGE_RETURN); parser->current.end++; space_seen = true; } @@ -9789,7 +9798,7 @@ parser_lex(pm_parser_t *parser) { parser->heredoc_end = NULL; } else { parser->current.end += eol_length + 1; - pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start)); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current)); space_seen = true; } } else if (pm_char_is_inline_whitespace(*parser->current.end)) { @@ -9883,7 +9892,7 @@ parser_lex(pm_parser_t *parser) { } if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start)); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current)); } } @@ -10082,7 +10091,7 @@ parser_lex(pm_parser_t *parser) { // , case ',': if ((parser->previous.type == PM_TOKEN_COMMA) && (parser->enclosure_nesting > 0)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_ARRAY_TERM, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_ARRAY_TERM, pm_token_type_human(parser->current.type)); } lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL); @@ -10208,7 +10217,7 @@ parser_lex(pm_parser_t *parser) { } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_USTAR_STAR; } else if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "**", "argument prefix"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "**", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10233,7 +10242,7 @@ parser_lex(pm_parser_t *parser) { } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_USTAR; } else if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "*", "argument prefix"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "*", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10392,7 +10401,7 @@ parser_lex(pm_parser_t *parser) { } else { // Otherwise, we want to indicate that the body of the // heredoc starts on the character after the next newline. - pm_newline_list_append(&parser->newline_list, (uint32_t) (body_start - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, U32(body_start - parser->start + 1)); body_start++; } @@ -10411,7 +10420,7 @@ parser_lex(pm_parser_t *parser) { } if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "<<", "here document"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "<<", "here document"); } if (lex_state_operator_p(parser)) { @@ -10537,7 +10546,7 @@ parser_lex(pm_parser_t *parser) { } else if (lex_state_beg_p(parser)) { type = PM_TOKEN_UAMPERSAND; } else if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "&", "argument prefix"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "&", "argument prefix"); } if (lex_state_operator_p(parser)) { @@ -10613,7 +10622,7 @@ parser_lex(pm_parser_t *parser) { } if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "+", "unary operator"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "+", "unary operator"); } lex_state_set(parser, PM_LEX_STATE_BEG); @@ -10654,7 +10663,7 @@ parser_lex(pm_parser_t *parser) { } if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "-", "unary operator"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "-", "unary operator"); } lex_state_set(parser, PM_LEX_STATE_BEG); @@ -10753,7 +10762,7 @@ parser_lex(pm_parser_t *parser) { } if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "/", "regexp literal"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "/", "regexp literal"); } if (lex_state_operator_p(parser)) { @@ -10938,7 +10947,7 @@ parser_lex(pm_parser_t *parser) { } if (ambiguous_operator_p(parser, space_seen)) { - PM_PARSER_WARN_TOKEN_FORMAT(parser, parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "%", "string literal"); + PM_PARSER_WARN_TOKEN_FORMAT(parser, &parser->current, PM_WARN_AMBIGUOUS_BINARY_OPERATOR, "%", "string literal"); } lex_state_set(parser, lex_state_operator_p(parser) ? PM_LEX_STATE_ARG : PM_LEX_STATE_BEG); @@ -10974,40 +10983,40 @@ parser_lex(pm_parser_t *parser) { // token after adding an appropriate error message. if (!width) { if (*parser->current.start >= 0x80) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *parser->current.start); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *parser->current.start); } else if (*parser->current.start == '\\') { switch (peek_at(parser, parser->current.start + 1)) { case ' ': parser->current.end++; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped space"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped space"); break; case '\f': parser->current.end++; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped form feed"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped form feed"); break; case '\t': parser->current.end++; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped horizontal tab"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped horizontal tab"); break; case '\v': parser->current.end++; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped vertical tab"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped vertical tab"); break; case '\r': if (peek_at(parser, parser->current.start + 2) != '\n') { parser->current.end++; - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped carriage return"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "escaped carriage return"); break; } PRISM_FALLTHROUGH default: - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "backslash"); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, "backslash"); break; } } else if (char_is_ascii_printable(*parser->current.start)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_PRINTABLE_CHARACTER, *parser->current.start); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_PRINTABLE_CHARACTER, *parser->current.start); } else { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_INVALID_CHARACTER, *parser->current.start); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_INVALID_CHARACTER, *parser->current.start); } goto lex_next_token; @@ -11033,15 +11042,15 @@ parser_lex(pm_parser_t *parser) { // correct column information for it. const uint8_t *cursor = parser->current.end; while ((cursor = next_newline(cursor, parser->end - cursor)) != NULL) { - pm_newline_list_append(&parser->newline_list, (uint32_t) (++cursor - parser->start)); + pm_newline_list_append(&parser->newline_list, U32(++cursor - parser->start)); } parser->current.end = parser->end; parser->current.type = PM_TOKEN___END__; parser_lex_callback(parser); - parser->data_loc.start = (uint32_t) (parser->current.start - parser->start); - parser->data_loc.length = (uint32_t) (parser->current.end - parser->current.start); + parser->data_loc.start = PM_TOKEN_START(parser, &parser->current); + parser->data_loc.length = PM_TOKEN_LENGTH(&parser->current); LEX(PM_TOKEN_EOF); } @@ -11066,7 +11075,7 @@ parser_lex(pm_parser_t *parser) { !(last_state & (PM_LEX_STATE_DOT | PM_LEX_STATE_FNAME)) && (type == PM_TOKEN_IDENTIFIER) && ((pm_parser_local_depth(parser, &parser->current) != -1) || - pm_token_is_numbered_parameter(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current))) + pm_token_is_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))) ) { lex_state_set(parser, PM_LEX_STATE_END | PM_LEX_STATE_LABEL); } @@ -11094,7 +11103,7 @@ parser_lex(pm_parser_t *parser) { whitespace += 1; } } else { - whitespace = pm_strspn_whitespace_newlines(parser->current.end, parser->end - parser->current.end, &parser->newline_list, (uint32_t) (parser->current.end - parser->start)); + whitespace = pm_strspn_whitespace_newlines(parser->current.end, parser->end - parser->current.end, &parser->newline_list, PM_TOKEN_END(parser, &parser->current)); } if (whitespace > 0) { @@ -11209,7 +11218,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } else { // ... else track the newline. - pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current) + 1); } parser->current.end++; @@ -11347,7 +11356,7 @@ parser_lex(pm_parser_t *parser) { // would have already have added the newline to the // list. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start)); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current)); } } else { parser->current.end = breakpoint + 1; @@ -11394,7 +11403,7 @@ parser_lex(pm_parser_t *parser) { // If we've hit a newline, then we need to track that in // the list of newlines. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, (uint32_t) (breakpoint - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, U32(breakpoint - parser->start + 1)); parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); break; @@ -11442,7 +11451,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } else { // ... else track the newline. - pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current) + 1); } parser->current.end++; @@ -11607,7 +11616,7 @@ parser_lex(pm_parser_t *parser) { // would have already have added the newline to the // list. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start)); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current)); } } else { parser->current.end = breakpoint + 1; @@ -11652,7 +11661,7 @@ parser_lex(pm_parser_t *parser) { // for the terminator in case the terminator is a // newline character. if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, (uint32_t) (breakpoint - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, U32(breakpoint - parser->start + 1)); parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); break; @@ -11706,7 +11715,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } else { // ... else track the newline. - pm_newline_list_append(&parser->newline_list, (uint32_t) (parser->current.end - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, PM_TOKEN_END(parser, &parser->current) + 1); } parser->current.end++; @@ -11835,7 +11844,7 @@ parser_lex(pm_parser_t *parser) { (memcmp(terminator_start, ident_start, ident_length) == 0) ) { if (newline != NULL) { - pm_newline_list_append(&parser->newline_list, (uint32_t) (newline - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, U32(newline - parser->start + 1)); } parser->current.end = terminator_end; @@ -11907,7 +11916,7 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } - pm_newline_list_append(&parser->newline_list, (uint32_t) (breakpoint - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, U32(breakpoint - parser->start + 1)); // If we have a - or ~ heredoc, then we can match after // some leading whitespace. @@ -12025,7 +12034,7 @@ parser_lex(pm_parser_t *parser) { // string content. if (heredoc_lex_mode->indent == PM_HEREDOC_INDENT_TILDE) { const uint8_t *end = parser->current.end; - pm_newline_list_append(&parser->newline_list, (uint32_t) (end - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, U32(end - parser->start + 1)); // Here we want the buffer to only // include up to the backslash. @@ -12624,7 +12633,7 @@ parse_unwriteable_target(pm_parser_t *parser, pm_node_t *target) { default: break; } - pm_constant_id_t name = pm_parser_constant_id_location(parser, target->location.start, target->location.end); + pm_constant_id_t name = pm_parser_constant_id_location(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target)); pm_local_variable_target_node_t *result = pm_local_variable_target_node_create(parser, &target->location, name, 0); pm_node_destroy(parser, target); @@ -12686,8 +12695,8 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p target->type = PM_GLOBAL_VARIABLE_TARGET_NODE; return target; case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(parser, U32(target->location.start - parser->start), PM_NODE_LENGTH(target))) { - PM_PARSER_ERR_FORMAT(parser, target->location.start, target->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, target->location.start); + if (pm_token_is_numbered_parameter(parser, PM_NODE_START(target), PM_NODE_LENGTH(target))) { + PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(target), PM_NODE_LENGTH(target), PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + PM_NODE_START(target)); pm_node_unreference(parser, target); } @@ -12755,12 +12764,8 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p // When it was parsed in the prefix position, foo was seen as a // method call with no receiver and no arguments. Now we have an // =, so we know it's a local variable write. - const pm_location_t message_loc = { - .start = parser->start + call->message_loc.start, - .end = parser->start + call->message_loc.start + call->message_loc.length - }; - - pm_constant_id_t name = pm_parser_local_add_slice(parser, &call->message_loc, 0); + pm_slice_t message_loc = call->message_loc; + pm_constant_id_t name = pm_parser_local_add_slice(parser, &message_loc, 0); pm_node_destroy(parser, target); return UP(pm_local_variable_target_node_create(parser, &message_loc, name, 0)); @@ -12874,22 +12879,21 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod case PM_LOCAL_VARIABLE_READ_NODE: { pm_local_variable_read_node_t *local_read = (pm_local_variable_read_node_t *) target; + pm_slice_t location = target->location; pm_constant_id_t name = local_read->name; - pm_location_t name_loc = target->location; - uint32_t depth = local_read->depth; pm_scope_t *scope = pm_parser_scope_find(parser, depth); - if (pm_token_is_numbered_parameter(parser, U32(target->location.start - parser->start), PM_NODE_LENGTH(target))) { + if (pm_token_is_numbered_parameter(parser, PM_NODE_START(target), PM_NODE_LENGTH(target))) { pm_diagnostic_id_t diag_id = (scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_FOUND) ? PM_ERR_EXPRESSION_NOT_WRITABLE_NUMBERED : PM_ERR_PARAMETER_NUMBERED_RESERVED; - PM_PARSER_ERR_FORMAT(parser, target->location.start, target->location.end, diag_id, target->location.start); + PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(target), PM_NODE_LENGTH(target), diag_id, parser->start + PM_NODE_START(target)); pm_node_unreference(parser, target); } pm_locals_unread(&scope->locals, name); pm_node_destroy(parser, target); - return UP(pm_local_variable_write_node_create(parser, name, depth, value, &name_loc, operator)); + return UP(pm_local_variable_write_node_create(parser, name, depth, value, &location, operator)); } case PM_IT_LOCAL_VARIABLE_READ_NODE: { pm_constant_id_t name = pm_parser_local_add_constant(parser, "it", 2); @@ -12943,17 +12947,14 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod // When it was parsed in the prefix position, foo was seen as a // method call with no receiver and no arguments. Now we have an // =, so we know it's a local variable write. - const pm_location_t message = { - .start = parser->start + call->message_loc.start, - .end = parser->start + call->message_loc.start + call->message_loc.length - }; + pm_slice_t message_loc = call->message_loc; - pm_refute_numbered_parameter(parser, call->message_loc.start, call->message_loc.length); - pm_parser_local_add_slice(parser, &call->message_loc, 0); + pm_refute_numbered_parameter(parser, message_loc.start, message_loc.length); + pm_parser_local_add_slice(parser, &message_loc, 0); pm_node_destroy(parser, target); - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, message.start, message.end); - target = UP(pm_local_variable_write_node_create(parser, constant_id, 0, value, &message, operator)); + pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, parser->start + PM_SLICE_START(&message_loc), parser->start + PM_SLICE_END(&message_loc)); + target = UP(pm_local_variable_write_node_create(parser, constant_id, 0, value, &message_loc, operator)); return target; } @@ -12973,7 +12974,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod call->arguments = arguments; pm_arguments_node_arguments_append(arguments, value); - PM_NODE_END_SET_NODE(call, arguments); + PM_NODE_LENGTH_SET_NODE(call, arguments); call->equal_loc = TOK2SLICE(parser, operator); parse_write_name(parser, &call->name); @@ -12992,7 +12993,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod } pm_arguments_node_arguments_append(call->arguments, value); - PM_NODE_END_SET_NODE(target, value); + PM_NODE_LENGTH_SET_NODE(target, value); // Replace the name with "[]=". call->name = pm_parser_constant_id_constant(parser, "[]=", 3); @@ -13047,7 +13048,7 @@ parse_unwriteable_write(pm_parser_t *parser, pm_node_t *target, const pm_token_t default: break; } - pm_constant_id_t name = pm_parser_local_add_location(parser, target->location.start, target->location.end, 1); + pm_constant_id_t name = pm_parser_local_add_location(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target), 1); pm_local_variable_write_node_t *result = pm_local_variable_write_node_create(parser, name, 0, value, &target->location, equals); pm_node_destroy(parser, target); @@ -13209,7 +13210,7 @@ parse_statements(pm_parser_t *parser, pm_context_t context, uint16_t depth) { // This is an inlined version of accept1 because the error that we // want to add has varargs. If this happens again, we should // probably extract a helper function. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; parser->previous.type = 0; } @@ -13244,12 +13245,12 @@ pm_hash_key_static_literals_add(pm_parser_t *parser, pm_static_literals_t *liter pm_diagnostic_list_append_format( &parser->warning_list, - (uint32_t) (duplicated->location.start - parser->start), - (uint32_t) (duplicated->location.end - duplicated->location.start), + duplicated->location.start, + duplicated->location.length, PM_WARN_DUPLICATED_HASH_KEY, (int) pm_buffer_length(&buffer), pm_buffer_value(&buffer), - pm_newline_list_line_column(&parser->newline_list, (uint32_t) (node->location.start - parser->start), parser->start_line).line + pm_newline_list_line_column(&parser->newline_list, PM_NODE_START(node), parser->start_line).line ); pm_buffer_free(&buffer); @@ -13267,11 +13268,11 @@ pm_when_clause_static_literals_add(pm_parser_t *parser, pm_static_literals_t *li if ((previous = pm_static_literals_add(&parser->newline_list, parser->start, parser->start_line, literals, node, false)) != NULL) { pm_diagnostic_list_append_format( &parser->warning_list, - (uint32_t) (node->location.start - parser->start), - (uint32_t) (node->location.end - node->location.start), + PM_NODE_START(node), + PM_NODE_LENGTH(node), PM_WARN_DUPLICATED_WHEN_CLAUSE, - pm_newline_list_line_column(&parser->newline_list, (uint32_t) (node->location.start - parser->start), parser->start_line).line, - pm_newline_list_line_column(&parser->newline_list, (uint32_t) (previous->location.start - parser->start), parser->start_line).line + pm_newline_list_line_column(&parser->newline_list, PM_NODE_START(node), parser->start_line).line, + pm_newline_list_line_column(&parser->newline_list, PM_NODE_START(previous), parser->start_line).line ); } } @@ -13330,7 +13331,7 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod pm_token_t identifier = { .type = PM_TOKEN_IDENTIFIER, .start = label.start, .end = label.end - 1 }; if (identifier.end[-1] == '!' || identifier.end[-1] == '?') { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, identifier, PM_ERR_INVALID_LOCAL_VARIABLE_READ); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &identifier, PM_ERR_INVALID_LOCAL_VARIABLE_READ); } else { depth = pm_parser_local_depth(parser, &identifier); } @@ -13342,7 +13343,7 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod } } - value->location.end++; + value->location.length++; value = UP(pm_implicit_node_create(parser, value)); } @@ -13500,7 +13501,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for pm_node_t *expression = parse_value_expression(parser, PM_BINDING_POWER_DEFINED, false, false, PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT, (uint16_t) (depth + 1)); if (parsed_bare_hash) { - pm_parser_err(parser, U32(operator.start - parser->start), U32(expression->location.end - operator.start), PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT); + pm_parser_err(parser, PM_TOKEN_START(parser, &operator), PM_NODE_END(expression) - PM_TOKEN_START(parser, &operator), PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT); } argument = UP(pm_splat_node_create(parser, &operator, expression)); @@ -13931,7 +13932,7 @@ parse_parameters( // reads of that parameter, then we need to warn that we // have a circular definition. if ((parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3) && (pm_locals_reads(&parser->current_scope->locals, name_id) != reads)) { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, name, PM_ERR_PARAMETER_CIRCULAR); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &name, PM_ERR_PARAMETER_CIRCULAR); } context_pop(parser); @@ -13971,9 +13972,9 @@ parse_parameters( local.end -= 1; if (parser->encoding_changed ? parser->encoding->isupper_char(local.start, local.end - local.start) : pm_encoding_utf_8_isupper_char(local.start, local.end - local.start)) { - pm_parser_err(parser, U32(local.start - parser->start), PM_TOKEN_LENGTH(&local), PM_ERR_ARGUMENT_FORMAL_CONSTANT); + pm_parser_err(parser, PM_TOKEN_START(parser, &local), PM_TOKEN_LENGTH(&local), PM_ERR_ARGUMENT_FORMAL_CONSTANT); } else if (local.end[-1] == '!' || local.end[-1] == '?') { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, local, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &local, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE); } bool repeated = pm_parser_parameter_name_check(parser, &local); @@ -14022,7 +14023,7 @@ parse_parameters( if (accepts_blocks_in_defaults) pm_accepts_block_stack_pop(parser); if (parser->version <= PM_OPTIONS_VERSION_CRUBY_3_3 && (pm_locals_reads(&parser->current_scope->locals, name_id) != reads)) { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, local, PM_ERR_PARAMETER_CIRCULAR); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &local, PM_ERR_PARAMETER_CIRCULAR); } param = UP(pm_optional_keyword_parameter_node_create(parser, &name, value)); @@ -14195,7 +14196,7 @@ token_newline_index(const pm_parser_t *parser) { // start of a heredoc, so we cannot rely on looking at the previous // offset of the newline list, and instead must go through the whole // process of a binary search for the line number. - return (size_t) pm_newline_list_line(&parser->newline_list, (uint32_t) (parser->current.start - parser->start), 0); + return (size_t) pm_newline_list_line(&parser->newline_list, PM_TOKEN_START(parser, &parser->current), 0); } } @@ -14267,10 +14268,10 @@ parser_warn_indentation_mismatch(pm_parser_t *parser, size_t opening_newline_ind if (allow_indent && (closing_column > opening_column)) return; // Otherwise, add a warning. - PM_PARSER_WARN_FORMAT_LOCATION( + PM_PARSER_WARN_FORMAT( parser, - closing_token->start, - closing_token->end, + PM_TOKEN_START(parser, closing_token), + PM_TOKEN_LENGTH(closing_token), PM_WARN_INDENTATION_MISMATCH, (int) (closing_token->end - closing_token->start), (const char *) closing_token->start, @@ -14397,11 +14398,10 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ // since we won't know the end until we've found all subsequent // clauses. This sets the end location on all rescues once we know it. if (current != NULL) { - const uint8_t *end_to_set = current->base.location.end; pm_rescue_node_t *clause = parent_node->rescue_clause; while (clause != NULL) { - clause->base.location.end = end_to_set; + PM_NODE_LENGTH_SET_NODE(clause, current); clause = clause->subsequent; } } @@ -14497,7 +14497,9 @@ static pm_begin_node_t * parse_rescues_implicit_begin(pm_parser_t *parser, size_t opening_newline_index, const pm_token_t *opening, const uint8_t *start, pm_statements_node_t *statements, pm_rescues_type_t type, uint16_t depth) { pm_begin_node_t *node = pm_begin_node_create(parser, NULL, statements); parse_rescues(parser, opening_newline_index, opening, node, type, (uint16_t) (depth + 1)); - node->base.location.start = start; + + node->base.location.start = U32(start - parser->start); + PM_NODE_LENGTH_SET_TOKEN(parser, node, &parser->current); return node; } @@ -14648,8 +14650,8 @@ parse_blocklike_parameters(pm_parser_t *parser, pm_node_t *parameters, const pm_ pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_OUTER_BLOCK); } else if (parser->current_scope->parameters & PM_SCOPE_PARAMETERS_NUMBERED_INNER) { pm_parser_err_node(parser, node, PM_ERR_NUMBERED_PARAMETER_INNER_BLOCK); - } else if (pm_token_is_numbered_parameter(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node))) { - numbered_parameter = MAX(numbered_parameter, (uint8_t) (node->location.start[1] - '0')); + } else if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) { + numbered_parameter = MAX(numbered_parameter, (uint8_t) (parser->start[node->location.start + 1] - '0')); } else { assert(false && "unreachable"); } @@ -14668,9 +14670,7 @@ parse_blocklike_parameters(pm_parser_t *parser, pm_node_t *parameters, const pm_ for (pm_scope_t *scope = parser->current_scope->previous; scope != NULL && !scope->closed; scope = scope->previous) { scope->parameters |= PM_SCOPE_PARAMETERS_NUMBERED_INNER; } - - const pm_location_t location = { .start = opening->start, .end = closing->end }; - return UP(pm_numbered_parameters_node_create(parser, &location, numbered_parameter)); + return UP(pm_numbered_parameters_node_create(parser, opening, closing, numbered_parameter)); } if (it_parameter) { @@ -14765,7 +14765,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept parse_arguments(parser, arguments, accepts_block, PM_TOKEN_PARENTHESIS_RIGHT, (uint16_t) (depth + 1)); if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_ARGUMENT_TERM_PAREN, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_ARGUMENT_TERM_PAREN, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; parser->previous.type = 0; } @@ -14786,7 +14786,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept // then we have a trailing comma where we need to check whether it is // allowed or not. if (parser->previous.type == PM_TOKEN_COMMA && !match1(parser, PM_TOKEN_SEMICOLON)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_EXPECT_ARGUMENT, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_EXPECT_ARGUMENT, pm_token_type_human(parser->current.type)); } pm_accepts_block_stack_pop(parser); @@ -15123,7 +15123,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl if (context == PM_CONTEXT_IF) { while (match1(parser, PM_TOKEN_KEYWORD_ELSIF)) { if (parser_end_of_line_p(parser)) { - PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, parser->current, PM_WARN_KEYWORD_EOL); + PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_WARN_KEYWORD_EOL); } parser_warn_indentation_mismatch(parser, opening_newline_index, &keyword, false, false); @@ -15417,7 +15417,7 @@ parse_string_part(pm_parser_t *parser, uint16_t depth) { // missing node. default: expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_EMBVAR_INVALID); - variable = UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + variable = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); break; } @@ -15627,7 +15627,7 @@ parse_undef_argument(pm_parser_t *parser, uint16_t depth) { } default: pm_parser_err_current(parser, PM_ERR_UNDEF_ARGUMENT); - return UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); } } @@ -15672,7 +15672,7 @@ parse_alias_argument(pm_parser_t *parser, bool first, uint16_t depth) { return UP(pm_global_variable_read_node_create(parser, &parser->previous)); default: pm_parser_err_current(parser, PM_ERR_ALIAS_ARGUMENT); - return UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); } } @@ -15684,7 +15684,7 @@ static pm_node_t * parse_variable(pm_parser_t *parser) { pm_constant_id_t name_id = pm_parser_constant_id_token(parser, &parser->previous); int depth; - bool is_numbered_param = pm_token_is_numbered_parameter(parser, U32(parser->previous.start - parser->start), PM_TOKEN_LENGTH(&parser->previous)); + bool is_numbered_param = pm_token_is_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous)); if (!is_numbered_param && ((depth = pm_parser_local_depth_constant_id(parser, name_id)) != -1)) { return UP(pm_local_variable_read_node_create_constant_id(parser, &parser->previous, name_id, (uint32_t) depth, false)); @@ -15754,7 +15754,7 @@ parse_method_definition_name(pm_parser_t *parser) { parser_lex(parser); return parser->previous; case PM_TOKEN_IDENTIFIER: - pm_refute_numbered_parameter(parser, U32(parser->current.start - parser->start), PM_TOKEN_LENGTH(&parser->current)); + pm_refute_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current)); parser_lex(parser); return parser->previous; case PM_CASE_OPERATOR: @@ -15762,7 +15762,7 @@ parse_method_definition_name(pm_parser_t *parser) { parser_lex(parser); return parser->previous; default: - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_DEF_NAME, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_DEF_NAME, pm_token_type_human(parser->current.type)); return (pm_token_t) { .type = 0, .start = parser->current.start, .end = parser->current.end }; } } @@ -15946,7 +15946,7 @@ parse_strings(pm_parser_t *parser, pm_node_t *current, bool accepts_label, uint1 } else if (accept1(parser, PM_TOKEN_STRING_END)) { node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped)); } else { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_STRING_LITERAL_TERM, pm_token_type_human(parser->previous.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_STRING_LITERAL_TERM, pm_token_type_human(parser->previous.type)); parser->previous.start = parser->previous.end; parser->previous.type = 0; node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped)); @@ -16085,12 +16085,12 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flag * an error to the parser. */ static void -parse_pattern_capture(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_constant_id_t capture, const pm_location_t *location) { +parse_pattern_capture(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_constant_id_t capture, const pm_slice_t *location) { // Skip this capture if it starts with an underscore. - if (*location->start == '_') return; + if (parser->start[location->start] == '_') return; if (pm_constant_id_list_includes(captures, capture)) { - pm_parser_err(parser, U32(location->start - parser->start), PM_TOKEN_LENGTH(location), PM_ERR_PATTERN_CAPTURE_DUPLICATE); + pm_parser_err(parser, location->start, location->length, PM_ERR_PATTERN_CAPTURE_DUPLICATE); } else { pm_constant_id_list_append(captures, capture); } @@ -16161,7 +16161,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { PM_NODE_START_SET_NODE(pattern_node, node); - PM_NODE_END_SET_TOKEN(pattern_node, &closing); + PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->constant = node; pattern_node->opening_loc = TOK2SLICE(parser, &opening); @@ -16177,7 +16177,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { PM_NODE_START_SET_NODE(pattern_node, node); - PM_NODE_END_SET_TOKEN(pattern_node, &closing); + PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->constant = node; pattern_node->opening_loc = TOK2SLICE(parser, &opening); @@ -16193,7 +16193,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures if (pattern_node->constant == NULL && pattern_node->opening_loc.length == 0) { PM_NODE_START_SET_NODE(pattern_node, node); - PM_NODE_END_SET_TOKEN(pattern_node, &closing); + PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->constant = node; pattern_node->opening_loc = TOK2SLICE(parser, &opening); @@ -16236,10 +16236,10 @@ parse_pattern_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) { pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2SLICE(parser, &parser->previous)); name = UP(pm_local_variable_target_node_create( parser, - &PM_LOCATION_TOKEN_VALUE(&parser->previous), + &TOK2SLICE(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) )); @@ -16272,10 +16272,10 @@ parse_pattern_keyword_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2SLICE(parser, &parser->previous)); value = UP(pm_local_variable_target_node_create( parser, - &PM_LOCATION_TOKEN_VALUE(&parser->previous), + &TOK2SLICE(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) )); @@ -16318,33 +16318,30 @@ pm_slice_is_valid_local(const pm_parser_t *parser, const uint8_t *start, const u static pm_node_t * parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_symbol_node_t *key) { const pm_slice_t *value_slice = &((pm_symbol_node_t *) key)->value_loc; - const pm_location_t value_location = { - .start = parser->start + value_slice->start, - .end = parser->start + value_slice->start + value_slice->length - }; - const pm_location_t *value_loc = &value_location; + const uint8_t *start = parser->start + PM_SLICE_START(value_slice); + const uint8_t *end = parser->start + PM_SLICE_END(value_slice); - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); + pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, start, end); int depth = -1; - if (pm_slice_is_valid_local(parser, value_loc->start, value_loc->end)) { + if (pm_slice_is_valid_local(parser, start, end)) { depth = pm_parser_local_depth_constant_id(parser, constant_id); } else { - pm_parser_err(parser, U32(key->base.location.start - parser->start), PM_NODE_LENGTH(key), PM_ERR_PATTERN_HASH_KEY_LOCALS); + pm_parser_err(parser, PM_NODE_START(key), PM_NODE_LENGTH(key), PM_ERR_PATTERN_HASH_KEY_LOCALS); - if ((value_loc->end > value_loc->start) && ((value_loc->end[-1] == '!') || (value_loc->end[-1] == '?'))) { - PM_PARSER_ERR_LOCATION_FORMAT(parser, value_loc, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (value_loc->end - value_loc->start), (const char *) value_loc->start); + if ((end > start) && ((end[-1] == '!') || (end[-1] == '?'))) { + PM_PARSER_ERR_FORMAT(parser, value_slice->start, value_slice->length, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (end - start), (const char *) start); } } if (depth == -1) { - pm_parser_local_add(parser, constant_id, value_loc->start, value_loc->end, 0); + pm_parser_local_add(parser, constant_id, start, end, 0); } - parse_pattern_capture(parser, captures, constant_id, value_loc); + parse_pattern_capture(parser, captures, constant_id, value_slice); pm_local_variable_target_node_t *target = pm_local_variable_target_node_create( parser, - value_loc, + value_slice, constant_id, (uint32_t) (depth == -1 ? 0 : depth) ); @@ -16405,7 +16402,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node pm_diagnostic_id_t diag_id = PM_NODE_TYPE_P(first_node, PM_INTERPOLATED_SYMBOL_NODE) ? PM_ERR_PATTERN_HASH_KEY_INTERPOLATED : PM_ERR_PATTERN_HASH_KEY_LABEL; pm_parser_err_node(parser, first_node, diag_id); - pm_node_t *value = UP(pm_missing_node_create(parser, first_node->location.start, first_node->location.end)); + pm_node_t *value = UP(pm_missing_node_create(parser, PM_NODE_START(first_node), PM_NODE_LENGTH(first_node))); pm_node_t *assoc = UP(pm_assoc_node_create(parser, first_node, NULL, value)); pm_node_list_append(&assocs, assoc); @@ -16461,7 +16458,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node if (PM_NODE_TYPE_P(key, PM_SYMBOL_NODE)) { value = parse_pattern_hash_implicit_value(parser, captures, (pm_symbol_node_t *) key); } else { - value = UP(pm_missing_node_create(parser, key->location.end, key->location.end)); + value = UP(pm_missing_node_create(parser, PM_NODE_END(key), 0)); } } else { value = parse_pattern(parser, captures, PM_PARSE_PATTERN_SINGLE, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY, (uint16_t) (depth + 1)); @@ -16500,10 +16497,10 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2SLICE(parser, &parser->previous)); return UP(pm_local_variable_target_node_create( parser, - &PM_LOCATION_TOKEN_VALUE(&parser->previous), + &TOK2SLICE(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) )); @@ -16530,8 +16527,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm case PM_ARRAY_PATTERN_NODE: { pm_array_pattern_node_t *pattern_node = (pm_array_pattern_node_t *) inner; if (pattern_node->opening_loc.length == 0) { - PM_NODE_START_SET_TOKEN(pattern_node, &opening); - PM_NODE_END_SET_TOKEN(pattern_node, &closing); + PM_NODE_START_SET_TOKEN(parser, pattern_node, &opening); + PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->opening_loc = TOK2SLICE(parser, &opening); pattern_node->closing_loc = TOK2SLICE(parser, &closing); @@ -16544,8 +16541,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm case PM_FIND_PATTERN_NODE: { pm_find_pattern_node_t *pattern_node = (pm_find_pattern_node_t *) inner; if (pattern_node->opening_loc.length == 0) { - PM_NODE_START_SET_TOKEN(pattern_node, &opening); - PM_NODE_END_SET_TOKEN(pattern_node, &closing); + PM_NODE_START_SET_TOKEN(parser, pattern_node, &opening); + PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->opening_loc = TOK2SLICE(parser, &opening); pattern_node->closing_loc = TOK2SLICE(parser, &closing); @@ -16590,10 +16587,10 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm first_node = parse_expression(parser, PM_BINDING_POWER_MAX, false, true, PM_ERR_PATTERN_HASH_KEY_LABEL, (uint16_t) (depth + 1)); break; default: { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_PATTERN_HASH_KEY, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_PATTERN_HASH_KEY, pm_token_type_human(parser->current.type)); parser_lex(parser); - first_node = UP(pm_missing_node_create(parser, parser->previous.start, parser->previous.end)); + first_node = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous))); break; } } @@ -16604,8 +16601,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm expect1(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_PATTERN_TERM_BRACE); pm_token_t closing = parser->previous; - PM_NODE_START_SET_TOKEN(node, &opening); - PM_NODE_END_SET_TOKEN(node, &closing); + PM_NODE_START_SET_TOKEN(parser, node, &opening); + PM_NODE_LENGTH_SET_TOKEN(parser, node, &closing); node->opening_loc = TOK2SLICE(parser, &opening); node->closing_loc = TOK2SLICE(parser, &closing); @@ -16628,7 +16625,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm } default: { pm_parser_err_token(parser, &operator, PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE); - pm_node_t *right = UP(pm_missing_node_create(parser, operator.start, operator.end)); + pm_node_t *right = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &operator), PM_TOKEN_LENGTH(&operator))); return UP(pm_range_node_create(parser, NULL, &operator, right)); } } @@ -16642,7 +16639,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm // Call nodes (arithmetic operations) are not allowed in patterns if (PM_NODE_TYPE(node) == PM_CALL_NODE) { pm_parser_err_node(parser, node, diag_id); - pm_missing_node_t *missing_node = pm_missing_node_create(parser, node->location.start, node->location.end); + pm_missing_node_t *missing_node = pm_missing_node_create(parser, PM_NODE_START(node), PM_NODE_LENGTH(node)); pm_node_destroy(parser, node); return UP(missing_node); } @@ -16678,7 +16675,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pm_node_t *variable = UP(parse_variable(parser)); if (variable == NULL) { - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->previous, PM_ERR_NO_LOCAL_VARIABLE); + PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, &parser->previous, PM_ERR_NO_LOCAL_VARIABLE); variable = UP(pm_local_variable_read_node_missing_create(parser, &parser->previous, 0)); } @@ -16732,7 +16729,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm // If we get here, then we have a pin operator followed by something // not understood. We'll create a missing node and return that. pm_parser_err_token(parser, &operator, PM_ERR_PATTERN_EXPRESSION_AFTER_PIN); - pm_node_t *variable = UP(pm_missing_node_create(parser, operator.start, operator.end)); + pm_node_t *variable = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &operator), PM_TOKEN_LENGTH(&operator))); return UP(pm_pinned_variable_node_create(parser, &operator, variable)); } } @@ -16755,7 +16752,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm } default: pm_parser_err_current(parser, diag_id); - return UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); } } @@ -16764,7 +16761,7 @@ parse_pattern_alternation_error_each(const pm_node_t *node, void *data) { switch (PM_NODE_TYPE(node)) { case PM_LOCAL_VARIABLE_TARGET_NODE: { pm_parser_t *parser = (pm_parser_t *) data; - pm_parser_err(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node), PM_ERR_PATTERN_CAPTURE_IN_ALTERNATIVE); + pm_parser_err(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), PM_ERR_PATTERN_CAPTURE_IN_ALTERNATIVE); return false; } default: @@ -16839,7 +16836,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p } default: { pm_parser_err_current(parser, diag_id); - pm_node_t *right = UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + pm_node_t *right = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); if (!alternation) { node = right; @@ -16866,10 +16863,10 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2SLICE(parser, &parser->previous)); pm_local_variable_target_node_t *target = pm_local_variable_target_node_create( parser, - &PM_LOCATION_TOKEN_VALUE(&parser->previous), + &TOK2SLICE(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) ); @@ -17024,23 +17021,27 @@ parse_negative_numeric(pm_node_t *node) { case PM_INTEGER_NODE: { pm_integer_node_t *cast = (pm_integer_node_t *) node; cast->base.location.start--; + cast->base.location.length++; cast->value.negative = true; break; } case PM_FLOAT_NODE: { pm_float_node_t *cast = (pm_float_node_t *) node; cast->base.location.start--; + cast->base.location.length++; cast->value = -cast->value; break; } case PM_RATIONAL_NODE: { pm_rational_node_t *cast = (pm_rational_node_t *) node; cast->base.location.start--; + cast->base.location.length++; cast->numerator.negative = true; break; } case PM_IMAGINARY_NODE: node->location.start--; + node->location.length++; parse_negative_numeric(((pm_imaginary_node_t *) node)->numeric); break; default: @@ -17058,22 +17059,22 @@ static void pm_parser_err_prefix(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { switch (diag_id) { case PM_ERR_HASH_KEY: { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, diag_id, pm_token_type_human(parser->previous.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, diag_id, pm_token_type_human(parser->previous.type)); break; } case PM_ERR_HASH_VALUE: case PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR: { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, diag_id, pm_token_type_human(parser->current.type)); break; } case PM_ERR_UNARY_RECEIVER: { const char *human = (parser->current.type == PM_TOKEN_EOF ? "end-of-input" : pm_token_type_human(parser->current.type)); - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, diag_id, human, parser->previous.start[0]); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, diag_id, human, parser->previous.start[0]); break; } case PM_ERR_UNARY_DISALLOWED: case PM_ERR_EXPECT_ARGUMENT: { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, diag_id, pm_token_type_human(parser->current.type)); break; } default: @@ -17300,15 +17301,15 @@ typedef struct { static void parse_regular_expression_error(const uint8_t *start, const uint8_t *end, const char *message, void *data) { parse_regular_expression_error_data_t *callback_data = (parse_regular_expression_error_data_t *) data; - pm_location_t location; + pm_token_t location; if (callback_data->shared) { - location = (pm_location_t) { .start = start, .end = end }; + location = (pm_token_t) { .type = 0, .start = start, .end = end }; } else { - location = (pm_location_t) { .start = callback_data->start, .end = callback_data->end }; + location = (pm_token_t) { .type = 0, .start = callback_data->start, .end = callback_data->end }; } - PM_PARSER_ERR_FORMAT(callback_data->parser, location.start, location.end, PM_ERR_REGEXP_PARSE_ERROR, message); + PM_PARSER_ERR_FORMAT(callback_data->parser, PM_TOKEN_START(callback_data->parser, &location), PM_TOKEN_LENGTH(&location), PM_ERR_REGEXP_PARSE_ERROR, message); } /** @@ -17319,8 +17320,8 @@ parse_regular_expression_errors(pm_parser_t *parser, pm_regular_expression_node_ const pm_string_t *unescaped = &node->unescaped; parse_regular_expression_error_data_t error_data = { .parser = parser, - .start = node->base.location.start, - .end = node->base.location.end, + .start = parser->start + PM_NODE_START(node), + .end = parser->start + PM_NODE_END(node), .shared = unescaped->type == PM_STRING_SHARED }; @@ -17360,10 +17361,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } else { // If there was no comma, then we need to add a syntax // error. - const uint8_t *location = parser->previous.end; - PM_PARSER_ERR_FORMAT(parser, location, location, PM_ERR_ARRAY_SEPARATOR, pm_token_type_human(parser->current.type)); - - parser->previous.start = location; + PM_PARSER_ERR_FORMAT(parser, PM_TOKEN_END(parser, &parser->previous), 0, PM_ERR_ARRAY_SEPARATOR, pm_token_type_human(parser->current.type)); + parser->previous.start = parser->previous.end; parser->previous.type = 0; } } @@ -17438,7 +17437,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b accept1(parser, PM_TOKEN_NEWLINE); if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_ARRAY_TERM, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_ARRAY_TERM, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; parser->previous.type = 0; } @@ -17534,8 +17533,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b multi_target->lparen_loc = TOK2SLICE(parser, &opening); multi_target->rparen_loc = TOK2SLICE(parser, &parser->previous); - PM_NODE_START_SET_TOKEN(multi_target, &opening); - PM_NODE_END_SET_TOKEN(multi_target, &parser->previous); + PM_NODE_START_SET_TOKEN(parser, multi_target, &opening); + PM_NODE_LENGTH_SET_TOKEN(parser, multi_target, &parser->previous); pm_node_t *result; if (match1(parser, PM_TOKEN_COMMA) && (binding_power == PM_BINDING_POWER_STATEMENT)) { @@ -17586,7 +17585,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we didn't find a terminator and we didn't find a right // parenthesis, then this is a syntax error. if (!terminator_found && !match1(parser, PM_TOKEN_EOF)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); } // Parse each statement within the parentheses. @@ -17617,7 +17616,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } else if (!match1(parser, PM_TOKEN_EOF)) { // If we're at the end of the file, then we're going to add // an error after this for the ) anyway. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(parser->current.type)); } } @@ -17641,9 +17640,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } if (PM_NODE_TYPE_P(statement, PM_MULTI_TARGET_NODE)) { - const uint8_t *offset = statement->location.end; + const uint8_t *offset = parser->start + PM_NODE_END(statement); pm_token_t operator = { .type = PM_TOKEN_EQUAL, .start = offset, .end = offset }; - pm_node_t *value = UP(pm_missing_node_create(parser, offset, offset)); + pm_node_t *value = UP(pm_missing_node_create(parser, PM_NODE_END(statement), 0)); statement = UP(pm_multi_write_node_create(parser, (pm_multi_target_node_t *) statement, &operator, value)); statements->body.nodes[statements->body.size - 1] = statement; @@ -17855,11 +17854,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b call->closing_loc = arguments.closing_loc; call->block = arguments.block; - const uint8_t *end = pm_arguments_end(parser, &arguments); - if (!end) { - end = parser->start + call->message_loc.start + call->message_loc.length; + const pm_slice_t *end = pm_arguments_end(&arguments); + if (end == NULL) { + PM_NODE_LENGTH_SET_SLICE(call, &call->message_loc); + } else { + PM_NODE_LENGTH_SET_SLICE(call, end); } - call->base.location.end = end; } } else { // Otherwise, we know the identifier is in the local table. This @@ -17886,7 +17886,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // purposes of warnings. assert(PM_NODE_TYPE_P(node, PM_LOCAL_VARIABLE_READ_NODE)); - if (pm_token_is_numbered_parameter(parser, U32(identifier.start - parser->start), PM_TOKEN_LENGTH(&identifier))) { + if (pm_token_is_numbered_parameter(parser, PM_TOKEN_START(parser, &identifier), PM_TOKEN_LENGTH(&identifier))) { pm_node_unreference(parser, node); } else { pm_local_variable_read_node_t *cast = (pm_local_variable_read_node_t *) node; @@ -17932,7 +17932,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b node = UP(pm_string_node_create_unescaped(parser, &opening, &content, &parser->previous, &PM_STRING_EMPTY)); } - PM_NODE_END_SET_TOKEN(node, &opening); + PM_NODE_LENGTH_SET_TOKEN(parser, node, &opening); } else if ((part = parse_string_part(parser, (uint16_t) (depth + 1))) == NULL) { // If we get here, then we tried to find something in the // heredoc but couldn't actually parse anything, so we'll just @@ -17940,7 +17940,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // // parse_string_part handles its own errors, so there is no need // for us to add one here. - node = UP(pm_missing_node_create(parser, parser->previous.start, parser->previous.end)); + node = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous))); } else if (PM_NODE_TYPE_P(part, PM_STRING_NODE) && match2(parser, PM_TOKEN_HEREDOC_END, PM_TOKEN_EOF)) { // If we get here, then the part that we parsed was plain string // content and we're at the end of the heredoc, so we can return @@ -17951,10 +17951,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b cast->opening_loc = TOK2SLICE(parser, &opening); cast->closing_loc = TOK2SLICE(parser, &parser->current); - cast->base.location = (pm_location_t) { - .start = parser->start + cast->opening_loc.start, - .end = parser->start + cast->opening_loc.start + cast->opening_loc.length - }; + cast->base.location = cast->opening_loc; if (lex_mode.quote == PM_HEREDOC_QUOTE_BACKTICK) { assert(sizeof(pm_string_node_t) == sizeof(pm_x_string_node_t)); @@ -17989,10 +17986,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b expect1_heredoc_term(parser, lex_mode.ident_start, lex_mode.ident_length); pm_interpolated_xstring_node_closing_set(parser, cast, &parser->previous); - cast->base.location = (pm_location_t) { - .start = parser->start + cast->opening_loc.start, - .end = parser->start + cast->opening_loc.start + cast->opening_loc.length - }; + cast->base.location = cast->opening_loc; node = UP(cast); } else { pm_interpolated_string_node_t *cast = pm_interpolated_string_node_create(parser, &opening, &parts, &opening); @@ -18001,10 +17995,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b expect1_heredoc_term(parser, lex_mode.ident_start, lex_mode.ident_length); pm_interpolated_string_node_closing_set(parser, cast, &parser->previous); - cast->base.location = (pm_location_t) { - .start = parser->start + cast->opening_loc.start, - .end = parser->start + cast->opening_loc.start + cast->opening_loc.length - }; + cast->base.location = cast->opening_loc; node = UP(cast); } @@ -18344,7 +18335,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parse_rescues(parser, opening_newline_index, &begin_keyword, begin_node, PM_RESCUES_BEGIN, (uint16_t) (depth + 1)); expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_BEGIN_TERM); - PM_NODE_END_SET_TOKEN(begin_node, &parser->previous); + PM_NODE_LENGTH_SET_TOKEN(parser, begin_node, &parser->previous); pm_begin_node_end_keyword_set(parser, begin_node, &parser->previous); pop_block_exits(parser, previous_block_exits); @@ -18398,7 +18389,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // Reject `foo && return bar`. if (!accepts_command_call && arguments.arguments != NULL) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, next, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(next.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &next, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(next.type)); } } } @@ -18421,7 +18412,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } default: assert(false && "unreachable"); - return UP(pm_missing_node_create(parser, parser->previous.start, parser->previous.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous))); } } case PM_TOKEN_KEYWORD_SUPER: { @@ -18479,7 +18470,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_parser_scope_push(parser, true); if (!match2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_SINGLETON_CLASS_DELIMITER, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_SINGLETON_CLASS_DELIMITER, pm_token_type_human(parser->current.type)); } pm_node_t *statements = NULL; @@ -18616,7 +18607,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b operator = parser->previous; name = parse_method_definition_name(parser); } else { - pm_refute_numbered_parameter(parser, U32(parser->previous.start - parser->start), PM_TOKEN_LENGTH(&parser->previous)); + pm_refute_numbered_parameter(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous)); pm_parser_scope_push(parser, true); name = parser->previous; @@ -18688,7 +18679,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b name = parse_method_definition_name(parser); } else { if (!valid_name) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, identifier, PM_ERR_DEF_NAME, pm_token_type_human(identifier.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &identifier, PM_ERR_DEF_NAME, pm_token_type_human(identifier.type)); } name = identifier; @@ -18750,7 +18741,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b context_pop(parser); if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_DEF_PARAMS_TERM_PAREN, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_DEF_PARAMS_TERM_PAREN, pm_token_type_human(parser->current.type)); parser->previous.start = parser->previous.end; parser->previous.type = 0; } @@ -18796,7 +18787,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser->current_context->context == PM_CONTEXT_DEFAULT_PARAMS && parser->current_context->prev->context == PM_CONTEXT_BLOCK_PARAMETERS ) { - PM_PARSER_ERR_FORMAT(parser, def_keyword.start, parser->previous.end, PM_ERR_UNEXPECTED_PARAMETER_DEFAULT_VALUE, "endless method definition"); + PM_PARSER_ERR_FORMAT(parser, PM_TOKEN_START(parser, &def_keyword), PM_TOKENS_LENGTH(&def_keyword, &parser->previous), PM_ERR_UNEXPECTED_PARAMETER_DEFAULT_VALUE, "endless method definition"); } equal = parser->previous; @@ -18973,7 +18964,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b index = parse_expression(parser, PM_BINDING_POWER_INDEX, false, false, PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA, (uint16_t) (depth + 1)); } else { pm_parser_err_token(parser, &for_keyword, PM_ERR_FOR_INDEX); - index = UP(pm_missing_node_create(parser, for_keyword.start, for_keyword.end)); + index = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &for_keyword), PM_TOKEN_LENGTH(&for_keyword))); } // Now, if there are multiple index expressions, parse them out. @@ -18997,7 +18988,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b do_keyword = parser->previous; } else { if (!match2(parser, PM_TOKEN_SEMICOLON, PM_TOKEN_NEWLINE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_FOR_DELIMITER, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_FOR_DELIMITER, pm_token_type_human(parser->current.type)); } } @@ -19013,7 +19004,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } case PM_TOKEN_KEYWORD_IF: if (parser_end_of_line_p(parser)) { - PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, parser->current, PM_WARN_KEYWORD_EOL); + PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_WARN_KEYWORD_EOL); } size_t opening_newline_index = token_newline_index(parser); @@ -19063,13 +19054,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // syntax. if (!accepts_command_call && !match1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES)) { - pm_parser_err(parser, U32(parser->previous.end - parser->start), 1, PM_ERR_EXPECT_LPAREN_AFTER_NOT_LPAREN); + pm_parser_err(parser, PM_TOKEN_END(parser, &parser->previous), 1, PM_ERR_EXPECT_LPAREN_AFTER_NOT_LPAREN); } else { accept1(parser, PM_TOKEN_NEWLINE); pm_parser_err_current(parser, PM_ERR_EXPECT_LPAREN_AFTER_NOT_OTHER); } - return UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); } accept1(parser, PM_TOKEN_NEWLINE); @@ -19781,7 +19772,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // still lex past it though and create a missing node place. if (binding_power != PM_BINDING_POWER_STATEMENT) { pm_parser_err_prefix(parser, diag_id); - return UP(pm_missing_node_create(parser, parser->previous.start, parser->previous.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous))); } pm_token_t operator = parser->previous; @@ -19984,17 +19975,17 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we get here, then we are assuming this token is closing a // parent context, so we'll indicate that to the user so that // they know how we behaved. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT, pm_token_type_human(parser->current.type), context_human(recoverable)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT, pm_token_type_human(parser->current.type), context_human(recoverable)); } else if (diag_id == PM_ERR_CANNOT_PARSE_EXPRESSION) { // We're going to make a special case here, because "cannot // parse expression" is pretty generic, and we know here that we // have an unexpected token. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_UNEXPECTED_TOKEN_IGNORE, pm_token_type_human(parser->current.type)); } else { pm_parser_err_prefix(parser, diag_id); } - return UP(pm_missing_node_create(parser, parser->previous.start, parser->previous.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->previous), PM_TOKEN_LENGTH(&parser->previous))); } } } @@ -20330,7 +20321,8 @@ parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { length = pm_buffer_length(&unescaped); } - pm_location_t location; + const uint8_t *start; + const uint8_t *end; pm_constant_id_t name; // If the name of the capture group isn't a valid identifier, we do @@ -20343,12 +20335,14 @@ parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { if (callback_data->shared) { // If the unescaped string is a slice of the source, then we can // copy the names directly. The pointers will line up. - location = (pm_location_t) { .start = source, .end = source + length }; - name = pm_parser_constant_id_location(parser, location.start, location.end); + start = source; + end = source + length; + name = pm_parser_constant_id_location(parser, start, end); } else { // Otherwise, the name is a slice of the malloc-ed owned string, // in which case we need to copy it out into a new string. - location = (pm_location_t) { .start = call->receiver->location.start, .end = call->receiver->location.end }; + start = parser->start + PM_NODE_START(call->receiver); + end = parser->start + PM_NODE_END(call->receiver); void *memory = xmalloc(length); if (memory == NULL) abort(); @@ -20373,7 +20367,7 @@ parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { // If the identifier is not already a local, then we will add it to // the local table. - pm_parser_local_add(parser, name, location.start, location.end, 0); + pm_parser_local_add(parser, name, start, end, 0); } // Here we lazily create the MatchWriteNode since we know we're @@ -20384,7 +20378,7 @@ parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { // Next, create the local variable target and add it to the list of // targets for the match. - pm_node_t *target = UP(pm_local_variable_target_node_create(parser, &location, name, depth == -1 ? 0 : (uint32_t) depth)); + pm_node_t *target = UP(pm_local_variable_target_node_create(parser, &TOK2SLICE(parser, &((pm_token_t) { .type = 0, .start = start, .end = end })), name, depth == -1 ? 0 : (uint32_t) depth)); pm_node_list_append(&callback_data->match->targets, target); } @@ -20406,8 +20400,8 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * parse_regular_expression_error_data_t error_data = { .parser = parser, - .start = call->receiver->location.start, - .end = call->receiver->location.end, + .start = parser->start + PM_NODE_START(call->receiver), + .end = parser->start + PM_NODE_END(call->receiver), .shared = content->type == PM_STRING_SHARED }; @@ -20444,7 +20438,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // variable before parsing the value, in case the value // references the variable. if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) { - pm_parser_local_add_location(parser, node->location.start, node->location.end, 0); + pm_parser_local_add_location(parser, parser->start + PM_NODE_START(node), parser->start + PM_NODE_END(node), 0); } parser_lex(parser); @@ -20548,8 +20542,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node))) { - PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, node->location.start); + if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) { + PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.length, PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + node->location.start); pm_node_unreference(parser, node); } @@ -20680,8 +20674,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node))) { - PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, node->location.start); + if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) { + PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + PM_NODE_START(node)); pm_node_unreference(parser, node); } @@ -20822,8 +20816,8 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t return result; } case PM_LOCAL_VARIABLE_READ_NODE: { - if (pm_token_is_numbered_parameter(parser, U32(node->location.start - parser->start), PM_NODE_LENGTH(node))) { - PM_PARSER_ERR_FORMAT(parser, node->location.start, node->location.end, PM_ERR_PARAMETER_NUMBERED_RESERVED, node->location.start); + if (pm_token_is_numbered_parameter(parser, PM_NODE_START(node), PM_NODE_LENGTH(node))) { + PM_PARSER_ERR_FORMAT(parser, PM_NODE_START(node), PM_NODE_LENGTH(node), PM_ERR_PARAMETER_NUMBERED_RESERVED, parser->start + PM_NODE_START(node)); pm_node_unreference(parser, node); } @@ -20883,7 +20877,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // In this case we have an operator but we don't know what it's for. // We need to treat it as an error. For now, we'll mark it as an error // and just skip right past it. - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->previous, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, pm_token_type_human(parser->current.type)); return node; } } @@ -20994,21 +20988,21 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t case PM_RESCUE_MODIFIER_NODE: { pm_rescue_modifier_node_t *cast = (pm_rescue_modifier_node_t *) node; if (PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } case PM_AND_NODE: { pm_and_node_t *cast = (pm_and_node_t *) node; if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } case PM_OR_NODE: { pm_or_node_t *cast = (pm_or_node_t *) node; if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } @@ -21024,7 +21018,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t case PM_TOKEN_LESS: case PM_TOKEN_LESS_EQUAL: { if (PM_NODE_TYPE_P(node, PM_CALL_NODE) && PM_NODE_FLAG_P(node, PM_CALL_NODE_FLAGS_COMPARISON)) { - PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, parser->current, PM_WARN_COMPARISON_AFTER_COMPARISON); + PM_PARSER_WARN_TOKEN_FORMAT_CONTENT(parser, &parser->current, PM_WARN_COMPARISON_AFTER_COMPARISON); } parser_lex(parser); @@ -21047,21 +21041,21 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t case PM_RESCUE_MODIFIER_NODE: { pm_rescue_modifier_node_t *cast = (pm_rescue_modifier_node_t *) node; if (PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->rescue_expression, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } case PM_AND_NODE: { pm_and_node_t *cast = (pm_and_node_t *) node; if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } case PM_OR_NODE: { pm_or_node_t *cast = (pm_or_node_t *) node; if (PM_NODE_TYPE_P(cast->right, PM_MATCH_PREDICATE_NODE) || PM_NODE_TYPE_P(cast->right, PM_MATCH_REQUIRED_NODE)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &operator, PM_ERR_EXPECT_EOL_AFTER_STATEMENT, pm_token_type_human(operator.type)); } break; } @@ -21082,7 +21076,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t break; } default: { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_EXPECT_MESSAGE, pm_token_type_human(parser->current.type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_EXPECT_MESSAGE, pm_token_type_human(parser->current.type)); message = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; } } @@ -21160,7 +21154,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // accidentally move past a ':' token that occurs after the syntax // error. pm_token_t colon = (pm_token_t) { .type = 0, .start = parser->previous.end, .end = parser->previous.end }; - pm_node_t *false_expression = UP(pm_missing_node_create(parser, colon.start, colon.end)); + pm_node_t *false_expression = UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &colon), PM_TOKEN_LENGTH(&colon))); context_pop(parser); pop_block_exits(parser, previous_block_exits); @@ -21377,7 +21371,7 @@ static pm_node_t * parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool accepts_command_call, bool accepts_label, pm_diagnostic_id_t diag_id, uint16_t depth) { if (PRISM_UNLIKELY(depth >= PRISM_DEPTH_MAXIMUM)) { pm_parser_err_current(parser, PM_ERR_NESTING_TOO_DEEP); - return UP(pm_missing_node_create(parser, parser->current.start, parser->current.end)); + return UP(pm_missing_node_create(parser, PM_TOKEN_START(parser, &parser->current), PM_TOKEN_LENGTH(&parser->current))); } pm_node_t *node = parse_expression_prefix(parser, binding_power, accepts_command_call, accepts_label, diag_id, depth); @@ -21478,7 +21472,7 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc // If this is a non-assoc operator and we are about to parse the // exact same operator, then we need to add an error. if (match1(parser, current_token_type)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_type_human(parser->current.type), pm_token_type_human(current_token_type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_type_human(parser->current.type), pm_token_type_human(current_token_type)); break; } @@ -21491,7 +21485,7 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc // if (PM_NODE_TYPE_P(node, PM_RANGE_NODE) && ((pm_range_node_t *) node)->right == NULL) { if (match4(parser, PM_TOKEN_UAMPERSAND, PM_TOKEN_USTAR, PM_TOKEN_DOT, PM_TOKEN_AMPERSAND_DOT)) { - PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_type_human(parser->current.type), pm_token_type_human(current_token_type)); + PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_NON_ASSOCIATIVE_OPERATOR, pm_token_type_human(parser->current.type), pm_token_type_human(current_token_type)); break; } @@ -21682,7 +21676,7 @@ parse_program(pm_parser_t *parser) { // correct the location information. if (statements == NULL) { statements = pm_statements_node_create(parser); - pm_statements_node_location_set(statements, parser->start, parser->start); + statements->base.location = (pm_slice_t) { 0 }; } return UP(pm_program_node_create(parser, &locals, statements)); @@ -21975,7 +21969,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm const uint8_t *newline = next_newline(cursor, parser->end - cursor); while (newline != NULL) { - pm_newline_list_append(&parser->newline_list, (uint32_t) (newline - parser->start + 1)); + pm_newline_list_append(&parser->newline_list, U32(newline - parser->start + 1)); cursor = newline + 1; newline = next_newline(cursor, parser->end - cursor); diff --git a/src/static_literals.c b/src/static_literals.c index 35110b9535..13a52378dd 100644 --- a/src/static_literals.c +++ b/src/static_literals.c @@ -95,7 +95,7 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node) } case PM_SOURCE_LINE_NODE: { // Source lines hash their line number. - const pm_line_column_t line_column = pm_newline_list_line_column(metadata->newline_list, (uint32_t) (node->location.start - metadata->start), metadata->start_line); + const pm_line_column_t line_column = pm_newline_list_line_column(metadata->newline_list, node->location.start, metadata->start_line); const int32_t *value = &line_column.line; return murmur_hash((const uint8_t *) value, sizeof(int32_t)); } @@ -243,7 +243,7 @@ pm_int64_value(const pm_static_literals_metadata_t *metadata, const pm_node_t *n return integer->negative ? -value : value; } case PM_SOURCE_LINE_NODE: - return (int64_t) pm_newline_list_line_column(metadata->newline_list, (uint32_t) (node->location.start - metadata->start), metadata->start_line).line; + return (int64_t) pm_newline_list_line_column(metadata->newline_list, node->location.start, metadata->start_line).line; default: assert(false && "unreachable"); return 0; @@ -511,12 +511,12 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met const double value = ((const pm_float_node_t *) node)->value; if (PRISM_ISINF(value)) { - if (*node->location.start == '-') { + if (metadata->start[node->location.start] == '-') { pm_buffer_append_byte(buffer, '-'); } pm_buffer_append_string(buffer, "Infinity", 8); } else if (value == 0.0) { - if (*node->location.start == '-') { + if (metadata->start[node->location.start] == '-') { pm_buffer_append_byte(buffer, '-'); } pm_buffer_append_string(buffer, "0.0", 3); @@ -585,7 +585,7 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met break; } case PM_SOURCE_LINE_NODE: - pm_buffer_append_format(buffer, "%d", pm_newline_list_line_column(metadata->newline_list, (uint32_t) (node->location.start - metadata->start), metadata->start_line).line); + pm_buffer_append_format(buffer, "%d", pm_newline_list_line_column(metadata->newline_list, node->location.start, metadata->start_line).line); break; case PM_STRING_NODE: { const pm_string_t *unescaped = &((const pm_string_node_t *) node)->unescaped; diff --git a/templates/ext/prism/api_node.c.erb b/templates/ext/prism/api_node.c.erb index 31c611c416..2960d3e410 100644 --- a/templates/ext/prism/api_node.c.erb +++ b/templates/ext/prism/api_node.c.erb @@ -11,22 +11,6 @@ extern VALUE rb_cPrismLocation; static VALUE rb_cPrism<%= node.name %>; <%- end -%> -static VALUE -pm_location_new(const pm_parser_t *parser, const uint8_t *start, const uint8_t *end, VALUE source, bool freeze) { - if (freeze) { - VALUE location_argv[] = { - source, - LONG2FIX(start - parser->start), - LONG2FIX(end - start) - }; - - return rb_obj_freeze(rb_class_new_instance(3, location_argv, rb_cPrismLocation)); - } else { - uint64_t value = ((((uint64_t) (start - parser->start)) << 32) | ((uint32_t) (end - start))); - return ULL2NUM(value); - } -} - static VALUE pm_slice_new(const uint32_t start, const uint32_t length, VALUE source, bool freeze) { if (freeze) { @@ -46,7 +30,7 @@ pm_slice_new(const uint32_t start, const uint32_t length, VALUE source, bool fre VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze) { ID type = rb_intern(pm_token_type_name(token->type)); - VALUE location = pm_location_new(parser, token->start, token->end, source, freeze); + VALUE location = pm_slice_new((uint32_t) (token->start - parser->start), (uint32_t) (token->end - token->start), source, freeze); VALUE slice = rb_enc_str_new((const char *) token->start, token->end - token->start, encoding); if (freeze) rb_obj_freeze(slice); @@ -216,7 +200,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi argv[1] = ULONG2NUM(node->node_id); // location - argv[2] = pm_location_new(parser, node->location.start, node->location.end, source, freeze); + argv[2] = pm_slice_new(node->location.start, node->location.length, source, freeze); // flags argv[3] = ULONG2NUM(node->flags); diff --git a/templates/include/prism/ast.h.erb b/templates/include/prism/ast.h.erb index 85c941011c..c96b664531 100644 --- a/templates/include/prism/ast.h.erb +++ b/templates/include/prism/ast.h.erb @@ -45,18 +45,6 @@ typedef struct { const uint8_t *end; } pm_token_t; -/** - * This represents a range of bytes in the source string to which a node or - * token corresponds. - */ -typedef struct { - /** A pointer to the start location of the range in the source. */ - const uint8_t *start; - - /** A pointer to the end location of the range in the source. */ - const uint8_t *end; -} pm_location_t; - /** * This struct represents a slice in the source code, defined by an offset and * a length. Note that we have confirmation that we can represent all slices @@ -148,7 +136,7 @@ typedef struct pm_node { * This is the location of the node in the source. It's a range of bytes * containing a start and an end. */ - pm_location_t location; + pm_slice_t location; } pm_node_t; /** diff --git a/templates/src/node.c.erb b/templates/src/node.c.erb index a9cffb99a6..e642de6958 100644 --- a/templates/src/node.c.erb +++ b/templates/src/node.c.erb @@ -225,13 +225,6 @@ pm_dump_json_constant(pm_buffer_t *buffer, const pm_parser_t *parser, pm_constan pm_buffer_append_byte(buffer, '"'); } -static void -pm_dump_json_location(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_location_t *location) { - uint32_t start = (uint32_t) (location->start - parser->start); - uint32_t end = (uint32_t) (location->end - parser->start); - pm_buffer_append_format(buffer, "{\"start\":%" PRIu32 ",\"end\":%" PRIu32 "}", start, end); -} - static void pm_dump_json_slice(pm_buffer_t *buffer, const pm_slice_t *slice) { pm_buffer_append_format(buffer, "{\"start\":%" PRIu32 ",\"length\":%" PRIu32 "}", slice->start, slice->length); @@ -248,7 +241,7 @@ pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *no pm_buffer_append_string(buffer, "{\"type\":\"<%= node.name %>\",\"location\":", <%= node.name.bytesize + 22 %>); const pm_<%= node.human %>_t *cast = (const pm_<%= node.human %>_t *) node; - pm_dump_json_location(buffer, parser, &cast->base.location); + pm_dump_json_slice(buffer, &cast->base.location); <%- [*node.flags, *node.fields].each_with_index do |field, index| -%> // Dump the <%= field.name %> field diff --git a/templates/src/prettyprint.c.erb b/templates/src/prettyprint.c.erb index aa904e80bc..67668ff063 100644 --- a/templates/src/prettyprint.c.erb +++ b/templates/src/prettyprint.c.erb @@ -10,13 +10,6 @@ void pm_prettyprint(void) {} #else -static inline void -prettyprint_location(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_location_t *location) { - pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, (uint32_t) (location->start - parser->start), parser->start_line); - pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, (uint32_t) (location->end - parser->start), parser->start_line); - pm_buffer_append_format(output_buffer, "(%" PRIi32 ",%" PRIu32 ")-(%" PRIi32 ",%" PRIu32 ")", start.line, start.column, end.line, end.column); -} - static inline void prettyprint_slice(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_slice_t *slice) { pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, slice->start, parser->start_line); @@ -42,7 +35,7 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node; <%- end -%> pm_buffer_append_string(output_buffer, "@ <%= node.name %> (location: ", <%= node.name.length + 14 %>); - prettyprint_location(output_buffer, parser, &node->location); + prettyprint_slice(output_buffer, parser, &node->location); pm_buffer_append_string(output_buffer, ")\n", 2); <%- (fields = [*node.flags, *node.fields]).each_with_index do |field, index| -%> <%- preadd = index == fields.length - 1 ? " " : "| " -%> diff --git a/templates/src/serialize.c.erb b/templates/src/serialize.c.erb index 4cae0c5c2d..3dd28fc83a 100644 --- a/templates/src/serialize.c.erb +++ b/templates/src/serialize.c.erb @@ -20,16 +20,6 @@ pm_sizet_to_u32(size_t value) { } static void -pm_serialize_location(const pm_parser_t *parser, const pm_location_t *location, pm_buffer_t *buffer) { - assert(location->start); - assert(location->end); - assert(location->start <= location->end); - - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(location->start - parser->start)); - pm_buffer_append_varuint(buffer, pm_ptrdifft_to_u32(location->end - location->start)); -} - -PRISM_ATTRIBUTE_UNUSED static void pm_serialize_slice(const pm_slice_t *slice, pm_buffer_t *buffer) { pm_buffer_append_varuint(buffer, slice->start); pm_buffer_append_varuint(buffer, slice->length); @@ -83,7 +73,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { <%- if Prism::Template::INCLUDE_NODE_ID -%> pm_buffer_append_varuint(buffer, node->node_id); <%- end -%> - pm_serialize_location(parser, &node->location, buffer); + pm_serialize_slice(&node->location, buffer); switch (PM_NODE_TYPE(node)) { // We do not need to serialize a ScopeNode ever as diff --git a/test/prism/result/overlap_test.rb b/test/prism/result/overlap_test.rb index 155bc870d3..d605eeca44 100644 --- a/test/prism/result/overlap_test.rb +++ b/test/prism/result/overlap_test.rb @@ -33,8 +33,13 @@ def assert_overlap(fixture) queue << child if compare - assert_operator current.location.start_offset, :<=, child.location.start_offset - assert_operator current.location.end_offset, :>=, child.location.end_offset + assert_operator current.location.start_offset, :<=, child.location.start_offset, -> { + "[#{fixture.full_path}] Parent node #{current.class} at #{current.location} does not start before child node #{child.class} at #{child.location}" + } + + assert_operator current.location.end_offset, :>=, child.location.end_offset, -> { + "[#{fixture.full_path}] Parent node #{current.class} at #{current.location} does not end after child node #{child.class} at #{child.location}" + } end end end diff --git a/test/prism/result/source_location_test.rb b/test/prism/result/source_location_test.rb index 38b971d02b..993150f581 100644 --- a/test/prism/result/source_location_test.rb +++ b/test/prism/result/source_location_test.rb @@ -935,16 +935,16 @@ def assert_location(kind, source, expected = 0...source.length, **options) node = yield node if block_given? if expected.begin == 0 - assert_equal 0, node.location.start_column + assert_equal 0, node.location.start_column, "#{kind} start_column" end if expected.end == source.length - assert_equal source.split("\n").last.length, node.location.end_column + assert_equal source.split("\n").last.length, node.location.end_column, "#{kind} end_column" end assert_kind_of kind, node - assert_equal expected.begin, node.location.start_offset - assert_equal expected.end, node.location.end_offset + assert_equal expected.begin, node.location.start_offset, "#{kind} start_offset" + assert_equal expected.end, node.location.end_offset, "#{kind} end_offset" end end end From b6de2f9ad220d3f44ebbcd28a02c3ee928b0c907 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 3 Dec 2025 11:29:20 -0500 Subject: [PATCH 21/22] Rename slice back to location --- docs/configuration.md | 4 +- ext/prism/extension.c | 12 +- include/prism/parser.h | 10 +- rust/ruby-prism/build.rs | 24 +- rust/ruby-prism/src/lib.rs | 40 +- src/prism.c | 574 +++++++++++------------ templates/ext/prism/api_node.c.erb | 19 +- templates/include/prism/ast.h.erb | 16 +- templates/include/prism/diagnostic.h.erb | 2 +- templates/src/node.c.erb | 10 +- templates/src/prettyprint.c.erb | 22 +- templates/src/serialize.c.erb | 18 +- 12 files changed, 373 insertions(+), 378 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 1e5297297f..04887d6d9c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -60,8 +60,8 @@ The available values for `type` are: * `string` - A field that is a string. For example, this is used as the name of the method in a call node, since it cannot directly reference the source string (as in `@-` or `foo=`). This is a `pm_string_t` in C. * `constant` - A field that is an integer that represents an index in the constant pool. This is a `pm_constant_id_t` in C. * `constant[]` - A field that is an array of constants. This is a `pm_constant_id_list_t` in C. -* `location` - A field that is a location. This is a `pm_slice_t` in C. -* `location?` - A field that is a location that is optionally present. This is a `pm_slice_t` in C, but if the value is not present then the `length` field will be `0`. +* `location` - A field that is a location. This is a `pm_location_t` in C. +* `location?` - A field that is a location that is optionally present. This is a `pm_location_t` in C, but if the value is not present then the `length` field will be `0`. * `uint8` - A field that is an 8-bit unsigned integer. This is a `uint8_t` in C. * `uint32` - A field that is a 32-bit unsigned integer. This is a `uint32_t` in C. diff --git a/ext/prism/extension.c b/ext/prism/extension.c index 8a1f165bdd..400546a4ce 100644 --- a/ext/prism/extension.c +++ b/ext/prism/extension.c @@ -461,17 +461,17 @@ parser_location(VALUE source, bool freeze, uint32_t start, uint32_t length) { } /** - * Create a new Location instance from the given parser and slice. + * Create a new Location instance from the given parser and location. */ -#define PARSER_LOCATION_SLICE(source, freeze, slice) \ - parser_location(source, freeze, slice.start, slice.length) +#define PARSER_LOCATION(source, freeze, location) \ + parser_location(source, freeze, location.start, location.length) /** * Build a new Comment instance from the given parser and comment. */ static inline VALUE parser_comment(VALUE source, bool freeze, const pm_comment_t *comment) { - VALUE argv[] = { PARSER_LOCATION_SLICE(source, freeze, comment->location) }; + VALUE argv[] = { PARSER_LOCATION(source, freeze, comment->location) }; VALUE type = (comment->type == PM_COMMENT_EMBDOC) ? rb_cPrismEmbDocComment : rb_cPrismInlineComment; return rb_class_new_instance_freeze(1, argv, type, freeze); } @@ -554,7 +554,7 @@ parser_errors(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bo ) { VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(error->diag_id))); VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(error->message, encoding)); - VALUE location = PARSER_LOCATION_SLICE(source, freeze, error->location); + VALUE location = PARSER_LOCATION(source, freeze, error->location); VALUE level = Qnil; switch (error->level) { @@ -594,7 +594,7 @@ parser_warnings(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, ) { VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(warning->diag_id))); VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(warning->message, encoding)); - VALUE location = PARSER_LOCATION_SLICE(source, freeze, warning->location); + VALUE location = PARSER_LOCATION(source, freeze, warning->location); VALUE level = Qnil; switch (warning->level) { diff --git a/include/prism/parser.h b/include/prism/parser.h index be78015bd6..a8d840d3bf 100644 --- a/include/prism/parser.h +++ b/include/prism/parser.h @@ -463,7 +463,7 @@ typedef struct pm_comment { pm_list_node_t node; /** The location of the comment in the source. */ - pm_slice_t location; + pm_location_t location; /** The type of comment that we've found. */ pm_comment_type_t type; @@ -480,10 +480,10 @@ typedef struct { pm_list_node_t node; /** The key of the magic comment. */ - pm_slice_t key; + pm_location_t key; /** The value of the magic comment. */ - pm_slice_t value; + pm_location_t value; } pm_magic_comment_t; /** @@ -531,7 +531,7 @@ typedef struct { pm_constant_id_t name; /** The location of the local variable in the source. */ - pm_slice_t location; + pm_location_t location; /** The index of the local variable in the local table. */ uint32_t index; @@ -722,7 +722,7 @@ struct pm_parser { * and the rest of the content of the file. This content is loaded into the * DATA constant when the file being parsed is the main file being executed. */ - pm_slice_t data_loc; + pm_location_t data_loc; /** The list of warnings that have been found while parsing. */ pm_list_t warning_list; diff --git a/rust/ruby-prism/build.rs b/rust/ruby-prism/build.rs index 3f8eb211c7..042a123dd4 100644 --- a/rust/ruby-prism/build.rs +++ b/rust/ruby-prism/build.rs @@ -253,9 +253,9 @@ fn write_node(file: &mut File, flags: &[Flags], node: &Node) -> Result<(), Box Slice<'pr> {{")?; - writeln!(file, " let pointer: *mut pm_slice_t = unsafe {{ &raw mut (*self.pointer).base.location }};")?; - writeln!(file, " Slice::new(self.parser, unsafe {{ &(*pointer) }})")?; + writeln!(file, " pub fn location(&self) -> Location<'pr> {{")?; + writeln!(file, " let pointer: *mut pm_location_t = unsafe {{ &raw mut (*self.pointer).base.location }};")?; + writeln!(file, " Location::new(self.parser, unsafe {{ &(*pointer) }})")?; writeln!(file, " }}")?; writeln!(file)?; writeln!(file, " /// Returns the flags of this node.")?; @@ -364,19 +364,19 @@ fn write_node(file: &mut File, flags: &[Flags], node: &Node) -> Result<(), Box { - writeln!(file, " pub fn {}(&self) -> Slice<'pr> {{", field.name)?; - writeln!(file, " let pointer: *mut pm_slice_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; - writeln!(file, " Slice::new(self.parser, unsafe {{ &(*pointer) }})")?; + writeln!(file, " pub fn {}(&self) -> Location<'pr> {{", field.name)?; + writeln!(file, " let pointer: *mut pm_location_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; + writeln!(file, " Location::new(self.parser, unsafe {{ &(*pointer) }})")?; writeln!(file, " }}")?; }, NodeFieldType::OptionalLocation => { - writeln!(file, " pub fn {}(&self) -> Option> {{", field.name)?; - writeln!(file, " let pointer: *mut pm_slice_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; + writeln!(file, " pub fn {}(&self) -> Option> {{", field.name)?; + writeln!(file, " let pointer: *mut pm_location_t = unsafe {{ &raw mut (*self.pointer).{} }};", field.name)?; writeln!(file, " let length = unsafe {{ (*pointer).length }};")?; writeln!(file, " if length == 0 {{")?; writeln!(file, " None")?; writeln!(file, " }} else {{")?; - writeln!(file, " Some(Slice::new(self.parser, unsafe {{ &(*pointer) }}))")?; + writeln!(file, " Some(Location::new(self.parser, unsafe {{ &(*pointer) }}))")?; writeln!(file, " }}")?; writeln!(file, " }}")?; }, @@ -558,7 +558,7 @@ use std::ptr::NonNull; #[allow(clippy::wildcard_imports)] use ruby_prism_sys::*; -use crate::{{ConstantId, ConstantList, Integer, Slice, NodeList}}; +use crate::{{ConstantId, ConstantList, Integer, Location, NodeList}}; " )?; @@ -621,10 +621,10 @@ impl<'pr> Node<'pr> {{ writeln!(file, " /// Returns the location of this node.")?; writeln!(file, " #[must_use]")?; - writeln!(file, " pub fn location(&self) -> Slice<'pr> {{")?; + writeln!(file, " pub fn location(&self) -> Location<'pr> {{")?; writeln!(file, " match *self {{")?; for node in &config.nodes { - writeln!(file, " Self::{} {{ pointer, parser, .. }} => Slice::new(parser, unsafe {{ &((*pointer.cast::()).location) }}),", node.name)?; + writeln!(file, " Self::{} {{ pointer, parser, .. }} => Location::new(parser, unsafe {{ &((*pointer.cast::()).location) }}),", node.name)?; } writeln!(file, " }}")?; writeln!(file, " }}")?; diff --git a/rust/ruby-prism/src/lib.rs b/rust/ruby-prism/src/lib.rs index 4600fd31ce..f54437abca 100644 --- a/rust/ruby-prism/src/lib.rs +++ b/rust/ruby-prism/src/lib.rs @@ -19,17 +19,17 @@ use std::mem::MaybeUninit; use std::ptr::NonNull; pub use self::bindings::*; -use ruby_prism_sys::{pm_comment_t, pm_comment_type_t, pm_constant_id_list_t, pm_constant_id_t, pm_diagnostic_t, pm_integer_t, pm_magic_comment_t, pm_node_destroy, pm_node_list, pm_node_t, pm_parse, pm_parser_free, pm_parser_init, pm_parser_t, pm_slice_t}; +use ruby_prism_sys::{pm_comment_t, pm_comment_type_t, pm_constant_id_list_t, pm_constant_id_t, pm_diagnostic_t, pm_integer_t, pm_magic_comment_t, pm_node_destroy, pm_node_list, pm_node_t, pm_parse, pm_parser_free, pm_parser_init, pm_parser_t, pm_location_t}; /// A range in the source file, represented as a start offset and length. -pub struct Slice<'pr> { +pub struct Location<'pr> { parser: NonNull, pub(crate) start: u32, pub(crate) length: u32, marker: PhantomData<&'pr [u8]>, } -impl<'pr> Slice<'pr> { +impl<'pr> Location<'pr> { /// Returns a byte slice for the range. #[must_use] pub fn as_slice(&self) -> &'pr [u8] { @@ -39,13 +39,13 @@ impl<'pr> Slice<'pr> { } } - /// Return a Slice from the given `pm_slice_t`. + /// Return a Location from the given `pm_location_t`. #[must_use] - pub(crate) const fn new(parser: NonNull, slice: &'pr pm_slice_t) -> Self { - Slice { + pub(crate) const fn new(parser: NonNull, location: &'pr pm_location_t) -> Self { + Location { parser, - start: slice.start, - length: slice.length, + start: location.start, + length: location.length, marker: PhantomData, } } @@ -56,7 +56,7 @@ impl<'pr> Slice<'pr> { self.start + self.length } - /// Return a Slice starting at self and ending at the end of other. + /// Return a Location starting at self and ending at the end of other. /// Returns None if both locations did not originate from the same parser, /// or if self starts after other. #[must_use] @@ -64,7 +64,7 @@ impl<'pr> Slice<'pr> { if self.parser != other.parser || self.start > other.start { None } else { - Some(Slice { + Some(Location { parser: self.parser, start: self.start, length: other.end() - self.start, @@ -74,7 +74,7 @@ impl<'pr> Slice<'pr> { } } -impl std::fmt::Debug for Slice<'_> { +impl std::fmt::Debug for Location<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let slice: &[u8] = self.as_slice(); @@ -332,8 +332,8 @@ impl<'pr> Diagnostic<'pr> { /// The location of the diagnostic in the source. #[must_use] - pub const fn location(&self) -> Slice<'pr> { - Slice::new(self.parser, unsafe { &self.diag.as_ref().location }) + pub const fn location(&self) -> Location<'pr> { + Location::new(self.parser, unsafe { &self.diag.as_ref().location }) } } @@ -377,8 +377,8 @@ impl<'pr> Comment<'pr> { /// The location of the comment in the source. #[must_use] - pub const fn location(&self) -> Slice<'pr> { - Slice::new(self.parser, unsafe { &self.content.as_ref().location }) + pub const fn location(&self) -> Location<'pr> { + Location::new(self.parser, unsafe { &self.content.as_ref().location }) } } @@ -510,9 +510,9 @@ impl<'pr> ParseResult<'pr> { /// Returns a slice of the source string that was parsed using the given /// slice range. #[must_use] - pub fn as_slice(&self, slice: &Slice<'pr>) -> &'pr [u8] { - let start = slice.start as usize; - let end = start + slice.length as usize; + pub fn as_slice(&self, location: &Location<'pr>) -> &'pr [u8] { + let start = location.start as usize; + let end = start + location.length as usize; &self.source[start..end] } @@ -574,12 +574,12 @@ impl<'pr> ParseResult<'pr> { /// Returns an optional location of the __END__ marker and the rest of the content of the file. #[must_use] - pub fn data_loc(&self) -> Option> { + pub fn data_loc(&self) -> Option> { let location = unsafe { &(*self.parser.as_ptr()).data_loc }; if location.length == 0 { None } else { - Some(Slice::new(self.parser, location)) + Some(Location::new(self.parser, location)) } } diff --git a/src/prism.c b/src/prism.c index 622a952a15..15f462ffda 100644 --- a/src/prism.c +++ b/src/prism.c @@ -27,8 +27,8 @@ pm_version(void) { #define FL PM_NODE_FLAGS #define UP PM_NODE_UPCAST -#define PM_SLICE_START(slice_) ((slice_)->start) -#define PM_SLICE_END(slice_) ((slice_)->start + (slice_)->length) +#define PM_LOCATION_START(location_) ((location_)->start) +#define PM_LOCATION_END(location_) ((location_)->start + (location_)->length) #define PM_TOKEN_START(parser_, token_) U32((token_)->start - (parser_)->start) #define PM_TOKEN_END(parser_, token_) U32((token_)->end - (parser_)->start) @@ -47,10 +47,10 @@ pm_version(void) { #define PM_NODE_START_SET_TOKEN(parser_, node_, token_) (PM_NODE_START(node_) = PM_TOKEN_START(parser_, token_)) #define PM_NODE_LENGTH_SET_NODE(left_, right_) (PM_NODE_LENGTH(left_) = PM_NODE_END(right_) - PM_NODE_START(left_)) #define PM_NODE_LENGTH_SET_TOKEN(parser_, node_, token_) (PM_NODE_LENGTH(node_) = PM_TOKEN_END(parser_, token_) - PM_NODE_START(node_)) -#define PM_NODE_LENGTH_SET_SLICE(node_, slice_) (PM_NODE_LENGTH(node_) = PM_SLICE_END(slice_) - PM_NODE_START(node_)) +#define PM_NODE_LENGTH_SET_LOCATION(node_, location_) (PM_NODE_LENGTH(node_) = PM_LOCATION_END(location_) - PM_NODE_START(node_)) -#define TOK2SLICE(parser_, token_) ((pm_slice_t) { .start = PM_TOKEN_START(parser_, token_), .length = PM_TOKEN_LENGTH(token_) }) -#define NTOK2SLICE(parser_, token_) ((token_) == NULL ? ((pm_slice_t) { 0 }) : TOK2SLICE(parser_, token_)) +#define TOK2LOC(parser_, token_) ((pm_location_t) { .start = PM_TOKEN_START(parser_, token_), .length = PM_TOKEN_LENGTH(token_) }) +#define NTOK2LOC(parser_, token_) ((token_) == NULL ? ((pm_location_t) { 0 }) : TOK2LOC(parser_, token_)) #define NTOK2PTR(token_) ((token_).start == NULL ? NULL : &(token_)) /******************************************************************************/ @@ -1574,13 +1574,13 @@ pm_conditional_predicate(pm_parser_t *parser, pm_node_t *node, pm_conditional_pr */ typedef struct { /** The optional location of the opening parenthesis or bracket. */ - pm_slice_t opening_loc; + pm_location_t opening_loc; /** The lazily-allocated optional arguments node. */ pm_arguments_node_t *arguments; /** The optional location of the closing parenthesis or bracket. */ - pm_slice_t closing_loc; + pm_location_t closing_loc; /** The optional block attached to the call. */ pm_node_t *block; @@ -1592,13 +1592,13 @@ typedef struct { /** * Retrieve the end location of a `pm_arguments_t` object. */ -static inline const pm_slice_t * +static inline const pm_location_t * pm_arguments_end(pm_arguments_t *arguments) { if (arguments->block != NULL) { uint32_t end = PM_NODE_END(arguments->block); if (arguments->closing_loc.length > 0) { - uint32_t arguments_end = PM_SLICE_END(&arguments->closing_loc); + uint32_t arguments_end = PM_LOCATION_END(&arguments->closing_loc); if (arguments_end > end) { return &arguments->closing_loc; } @@ -1976,7 +1976,7 @@ pm_alias_global_variable_node_create(pm_parser_t *parser, const pm_token_t *keyw .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_ALIAS_GLOBAL_VARIABLE_NODE, 0, keyword, old_name), .new_name = new_name, .old_name = old_name, - .keyword_loc = TOK2SLICE(parser, keyword) + .keyword_loc = TOK2LOC(parser, keyword) }; return node; @@ -1994,7 +1994,7 @@ pm_alias_method_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_n .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_ALIAS_METHOD_NODE, 0, keyword, old_name), .new_name = new_name, .old_name = old_name, - .keyword_loc = TOK2SLICE(parser, keyword) + .keyword_loc = TOK2LOC(parser, keyword) }; return node; @@ -2011,7 +2011,7 @@ pm_alternation_pattern_node_create(pm_parser_t *parser, pm_node_t *left, pm_node .base = PM_NODE_INIT_NODES(parser, PM_ALTERNATION_PATTERN_NODE, 0, left, right), .left = left, .right = right, - .operator_loc = TOK2SLICE(parser, operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -2029,7 +2029,7 @@ pm_and_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *opera *node = (pm_and_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_AND_NODE, 0, left, right), .left = left, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .right = right }; @@ -2097,8 +2097,8 @@ pm_array_node_create(pm_parser_t *parser, const pm_token_t *opening) { } else { *node = (pm_array_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_ARRAY_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, opening), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, opening), .elements = { 0 } }; } @@ -2136,7 +2136,7 @@ static void pm_array_node_close_set(const pm_parser_t *parser, pm_array_node_t *node, const pm_token_t *closing) { assert(closing->type == PM_TOKEN_BRACKET_RIGHT || closing->type == PM_TOKEN_STRING_END || closing->type == 0); PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); - node->closing_loc = TOK2SLICE(parser, closing); + node->closing_loc = TOK2LOC(parser, closing); } /** @@ -2208,8 +2208,8 @@ pm_array_pattern_node_constant_create(pm_parser_t *parser, pm_node_t *constant, .base = PM_NODE_INIT_NODE_TOKEN(parser, PM_ARRAY_PATTERN_NODE, 0, constant, closing), .constant = constant, .rest = NULL, - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, closing), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing), .requireds = { 0 }, .posts = { 0 } }; @@ -2229,8 +2229,8 @@ pm_array_pattern_node_empty_create(pm_parser_t *parser, const pm_token_t *openin .base = PM_NODE_INIT_TOKENS(parser, PM_ARRAY_PATTERN_NODE, 0, opening, closing), .constant = NULL, .rest = NULL, - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, closing), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing), .requireds = { 0 }, .posts = { 0 } }; @@ -2279,7 +2279,7 @@ pm_assoc_node_create(pm_parser_t *parser, pm_node_t *key, const pm_token_t *oper *node = (pm_assoc_node_t) { .base = PM_NODE_INIT(parser, PM_ASSOC_NODE, flags, PM_NODE_START(key), U32(end - PM_NODE_START(key))), .key = key, - .operator_loc = NTOK2SLICE(parser, operator), + .operator_loc = NTOK2LOC(parser, operator), .value = value }; @@ -2301,7 +2301,7 @@ pm_assoc_splat_node_create(pm_parser_t *parser, pm_node_t *value, const pm_token : PM_NODE_INIT_TOKEN_NODE(parser, PM_ASSOC_SPLAT_NODE, 0, operator, value) ), .value = value, - .operator_loc = TOK2SLICE(parser, operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -2335,7 +2335,7 @@ pm_begin_node_create(pm_parser_t *parser, const pm_token_t *begin_keyword, pm_st *node = (pm_begin_node_t) { .base = PM_NODE_INIT(parser, PM_BEGIN_NODE, 0, start, U32(end - start)), - .begin_keyword_loc = NTOK2SLICE(parser, begin_keyword), + .begin_keyword_loc = NTOK2LOC(parser, begin_keyword), .statements = statements, .end_keyword_loc = { 0 } }; @@ -2386,7 +2386,7 @@ static void pm_begin_node_end_keyword_set(const pm_parser_t *parser, pm_begin_node_t *node, const pm_token_t *end_keyword) { assert(end_keyword->type == PM_TOKEN_KEYWORD_END || end_keyword->type == 0); PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); - node->end_keyword_loc = TOK2SLICE(parser, end_keyword); + node->end_keyword_loc = TOK2LOC(parser, end_keyword); } /** @@ -2404,7 +2404,7 @@ pm_block_argument_node_create(pm_parser_t *parser, const pm_token_t *operator, p : PM_NODE_INIT_TOKEN_NODE(parser, PM_BLOCK_ARGUMENT_NODE, 0, operator, expression) ), .expression = expression, - .operator_loc = TOK2SLICE(parser, operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -2422,8 +2422,8 @@ pm_block_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const p .locals = *locals, .parameters = parameters, .body = body, - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, closing) + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing) }; return node; @@ -2444,8 +2444,8 @@ pm_block_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, cons : PM_NODE_INIT_TOKENS(parser, PM_BLOCK_PARAMETER_NODE, 0, operator, name) ), .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), - .name_loc = NTOK2SLICE(parser, name), - .operator_loc = TOK2SLICE(parser, operator) + .name_loc = NTOK2LOC(parser, name), + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -2479,7 +2479,7 @@ pm_block_parameters_node_create(pm_parser_t *parser, pm_parameters_node_t *param *node = (pm_block_parameters_node_t) { .base = PM_NODE_INIT(parser, PM_BLOCK_PARAMETERS_NODE, 0, start, U32(end - start)), .parameters = parameters, - .opening_loc = NTOK2SLICE(parser, opening), + .opening_loc = NTOK2LOC(parser, opening), .closing_loc = { 0 }, .locals = { 0 } }; @@ -2494,7 +2494,7 @@ static void pm_block_parameters_node_closing_set(const pm_parser_t *parser, pm_block_parameters_node_t *node, const pm_token_t *closing) { assert(closing->type == PM_TOKEN_PIPE || closing->type == PM_TOKEN_PARENTHESIS_RIGHT || closing->type == 0); PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); - node->closing_loc = TOK2SLICE(parser, closing); + node->closing_loc = TOK2LOC(parser, closing); } /** @@ -2541,7 +2541,7 @@ pm_break_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument : PM_NODE_INIT_TOKEN_NODE(parser, PM_BREAK_NODE, 0, keyword, arguments) ), .arguments = arguments, - .keyword_loc = TOK2SLICE(parser, keyword) + .keyword_loc = TOK2LOC(parser, keyword) }; return node; @@ -2607,9 +2607,9 @@ pm_call_node_aref_create(pm_parser_t *parser, pm_node_t *receiver, pm_arguments_ PM_NODE_START_SET_NODE(node, receiver); - const pm_slice_t *end = pm_arguments_end(arguments); + const pm_location_t *end = pm_arguments_end(arguments); assert(end != NULL && "unreachable"); - PM_NODE_LENGTH_SET_SLICE(node, end); + PM_NODE_LENGTH_SET_LOCATION(node, end); node->receiver = receiver; node->message_loc.start = arguments->opening_loc.start; @@ -2638,7 +2638,7 @@ pm_call_node_binary_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t PM_NODE_LENGTH_SET_NODE(node, PM_NODE_END(receiver) > PM_NODE_END(argument) ? receiver : argument); node->receiver = receiver; - node->message_loc = TOK2SLICE(parser, operator); + node->message_loc = TOK2LOC(parser, operator); pm_arguments_node_t *arguments = pm_arguments_node_create(parser); pm_arguments_node_arguments_append(arguments, argument); @@ -2660,16 +2660,16 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); PM_NODE_START_SET_NODE(node, receiver); - const pm_slice_t *end = pm_arguments_end(arguments); + const pm_location_t *end = pm_arguments_end(arguments); if (end == NULL) { PM_NODE_LENGTH_SET_TOKEN(parser, node, message); } else { - PM_NODE_LENGTH_SET_SLICE(node, end); + PM_NODE_LENGTH_SET_LOCATION(node, end); } node->receiver = receiver; - node->call_operator_loc = TOK2SLICE(parser, operator); - node->message_loc = TOK2SLICE(parser, message); + node->call_operator_loc = TOK2LOC(parser, operator); + node->message_loc = TOK2LOC(parser, message); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2693,7 +2693,7 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o static pm_call_node_t * pm_call_node_call_synthesized_create(pm_parser_t *parser, pm_node_t *receiver, const char *message, pm_arguments_node_t *arguments) { pm_call_node_t *node = pm_call_node_create(parser, 0); - node->base.location = (pm_slice_t) { .start = 0, .length = U32(parser->end - parser->start) }; + node->base.location = (pm_location_t) { .start = 0, .length = U32(parser->end - parser->start) }; node->receiver = receiver; node->arguments = arguments; @@ -2711,11 +2711,11 @@ pm_call_node_fcall_create(pm_parser_t *parser, pm_token_t *message, pm_arguments pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); PM_NODE_START_SET_TOKEN(parser, node, message); - const pm_slice_t *end = pm_arguments_end(arguments); + const pm_location_t *end = pm_arguments_end(arguments); assert(end != NULL && "unreachable"); - PM_NODE_LENGTH_SET_SLICE(node, end); + PM_NODE_LENGTH_SET_LOCATION(node, end); - node->message_loc = TOK2SLICE(parser, message); + node->message_loc = TOK2LOC(parser, message); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2733,7 +2733,7 @@ static pm_call_node_t * pm_call_node_fcall_synthesized_create(pm_parser_t *parser, pm_arguments_node_t *arguments, pm_constant_id_t name) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location = (pm_slice_t) { 0 }; + node->base.location = (pm_location_t) { 0 }; node->arguments = arguments; node->name = name; @@ -2752,14 +2752,14 @@ pm_call_node_not_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *me PM_NODE_START_SET_TOKEN(parser, node, message); if (arguments->closing_loc.length > 0) { - PM_NODE_LENGTH_SET_SLICE(node, &arguments->closing_loc); + PM_NODE_LENGTH_SET_LOCATION(node, &arguments->closing_loc); } else { assert(receiver != NULL); PM_NODE_LENGTH_SET_NODE(node, receiver); } node->receiver = receiver; - node->message_loc = TOK2SLICE(parser, message); + node->message_loc = TOK2LOC(parser, message); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2778,12 +2778,12 @@ pm_call_node_shorthand_create(pm_parser_t *parser, pm_node_t *receiver, pm_token pm_call_node_t *node = pm_call_node_create(parser, pm_call_node_ignore_visibility_flag(receiver)); PM_NODE_START_SET_NODE(node, receiver); - const pm_slice_t *end = pm_arguments_end(arguments); + const pm_location_t *end = pm_arguments_end(arguments); assert(end != NULL && "unreachable"); - PM_NODE_LENGTH_SET_SLICE(node, end); + PM_NODE_LENGTH_SET_LOCATION(node, end); node->receiver = receiver; - node->call_operator_loc = TOK2SLICE(parser, operator); + node->call_operator_loc = TOK2LOC(parser, operator); node->opening_loc = arguments->opening_loc; node->arguments = arguments->arguments; node->closing_loc = arguments->closing_loc; @@ -2810,7 +2810,7 @@ pm_call_node_unary_create(pm_parser_t *parser, pm_token_t *operator, pm_node_t * PM_NODE_LENGTH_SET_NODE(node, receiver); node->receiver = receiver; - node->message_loc = TOK2SLICE(parser, operator); + node->message_loc = TOK2LOC(parser, operator); node->name = pm_parser_constant_id_constant(parser, name, strlen(name)); return node; @@ -2824,8 +2824,8 @@ static pm_call_node_t * pm_call_node_variable_call_create(pm_parser_t *parser, pm_token_t *message) { pm_call_node_t *node = pm_call_node_create(parser, PM_CALL_NODE_FLAGS_IGNORE_VISIBILITY); - node->base.location = TOK2SLICE(parser, message); - node->message_loc = TOK2SLICE(parser, message); + node->base.location = TOK2LOC(parser, message); + node->message_loc = TOK2LOC(parser, message); node->name = pm_parser_constant_id_token(parser, message); return node; @@ -2884,7 +2884,7 @@ pm_call_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -2940,7 +2940,7 @@ pm_index_and_write_node_create(pm_parser_t *parser, pm_call_node_t *target, cons .arguments = target->arguments, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -2968,7 +2968,7 @@ pm_call_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .read_name = 0, .write_name = target->name, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .binary_operator_loc = TOK2SLICE(parser, operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3001,7 +3001,7 @@ pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), - .binary_operator_loc = TOK2SLICE(parser, operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3029,7 +3029,7 @@ pm_call_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3062,7 +3062,7 @@ pm_index_or_write_node_create(pm_parser_t *parser, pm_call_node_t *target, const .arguments = target->arguments, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3137,7 +3137,7 @@ pm_capture_pattern_node_create(pm_parser_t *parser, pm_node_t *value, pm_local_v .base = PM_NODE_INIT_NODES(parser, PM_CAPTURE_PATTERN_NODE, 0, value, target), .value = value, .target = target, - .operator_loc = TOK2SLICE(parser, operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -3154,8 +3154,8 @@ pm_case_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, pm_node .base = PM_NODE_INIT_TOKENS(parser, PM_CASE_NODE, 0, case_keyword, end_keyword == NULL ? case_keyword : end_keyword), .predicate = predicate, .else_clause = NULL, - .case_keyword_loc = TOK2SLICE(parser, case_keyword), - .end_keyword_loc = NTOK2SLICE(parser, end_keyword), + .case_keyword_loc = TOK2LOC(parser, case_keyword), + .end_keyword_loc = NTOK2LOC(parser, end_keyword), .conditions = { 0 } }; @@ -3188,7 +3188,7 @@ pm_case_node_else_clause_set(pm_case_node_t *node, pm_else_node_t *else_clause) static void pm_case_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_node_t *node, const pm_token_t *end_keyword) { PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); - node->end_keyword_loc = TOK2SLICE(parser, end_keyword); + node->end_keyword_loc = TOK2LOC(parser, end_keyword); } /** @@ -3202,7 +3202,7 @@ pm_case_match_node_create(pm_parser_t *parser, const pm_token_t *case_keyword, p .base = PM_NODE_INIT_TOKEN(parser, PM_CASE_MATCH_NODE, 0, case_keyword), .predicate = predicate, .else_clause = NULL, - .case_keyword_loc = TOK2SLICE(parser, case_keyword), + .case_keyword_loc = TOK2LOC(parser, case_keyword), .end_keyword_loc = { 0 }, .conditions = { 0 } }; @@ -3236,7 +3236,7 @@ pm_case_match_node_else_clause_set(pm_case_match_node_t *node, pm_else_node_t *e static void pm_case_match_node_end_keyword_loc_set(const pm_parser_t *parser, pm_case_match_node_t *node, const pm_token_t *end_keyword) { PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); - node->end_keyword_loc = TOK2SLICE(parser, end_keyword); + node->end_keyword_loc = TOK2LOC(parser, end_keyword); } /** @@ -3249,12 +3249,12 @@ pm_class_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const p *node = (pm_class_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_CLASS_NODE, 0, class_keyword, end_keyword), .locals = *locals, - .class_keyword_loc = TOK2SLICE(parser, class_keyword), + .class_keyword_loc = TOK2LOC(parser, class_keyword), .constant_path = constant_path, - .inheritance_operator_loc = NTOK2SLICE(parser, inheritance_operator), + .inheritance_operator_loc = NTOK2LOC(parser, inheritance_operator), .superclass = superclass, .body = body, - .end_keyword_loc = TOK2SLICE(parser, end_keyword), + .end_keyword_loc = TOK2LOC(parser, end_keyword), .name = pm_parser_constant_id_token(parser, name) }; @@ -3273,7 +3273,7 @@ pm_class_variable_and_write_node_create(pm_parser_t *parser, pm_class_variable_r .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = target->name, .name_loc = target->base.location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3291,7 +3291,7 @@ pm_class_variable_operator_write_node_create(pm_parser_t *parser, pm_class_varia .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, .name_loc = target->base.location, - .binary_operator_loc = TOK2SLICE(parser, operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -3311,7 +3311,7 @@ pm_class_variable_or_write_node_create(pm_parser_t *parser, pm_class_variable_re .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = target->name, .name_loc = target->base.location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3360,7 +3360,7 @@ pm_class_variable_write_node_create(pm_parser_t *parser, pm_class_variable_read_ .base = PM_NODE_INIT_NODES(parser, PM_CLASS_VARIABLE_WRITE_NODE, flags, read_node, value), .name = read_node->name, .name_loc = read_node->base.location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3378,7 +3378,7 @@ pm_constant_path_and_write_node_create(pm_parser_t *parser, pm_constant_path_nod *node = (pm_constant_path_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_AND_WRITE_NODE, 0, target, value), .target = target, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3395,7 +3395,7 @@ pm_constant_path_operator_write_node_create(pm_parser_t *parser, pm_constant_pat *node = (pm_constant_path_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_OPERATOR_WRITE_NODE, 0, target, value), .target = target, - .binary_operator_loc = TOK2SLICE(parser, operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -3414,7 +3414,7 @@ pm_constant_path_or_write_node_create(pm_parser_t *parser, pm_constant_path_node *node = (pm_constant_path_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_OR_WRITE_NODE, 0, target, value), .target = target, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3442,8 +3442,8 @@ pm_constant_path_node_create(pm_parser_t *parser, pm_node_t *parent, const pm_to ), .parent = parent, .name = name, - .delimiter_loc = TOK2SLICE(parser, delimiter), - .name_loc = TOK2SLICE(parser, name_token) + .delimiter_loc = TOK2LOC(parser, delimiter), + .name_loc = TOK2LOC(parser, name_token) }; return node; @@ -3460,7 +3460,7 @@ pm_constant_path_write_node_create(pm_parser_t *parser, pm_constant_path_node_t *node = (pm_constant_path_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_PATH_WRITE_NODE, flags, target, value), .target = target, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3479,7 +3479,7 @@ pm_constant_and_write_node_create(pm_parser_t *parser, pm_constant_read_node_t * .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_AND_WRITE_NODE, 0, target, value), .name = target->name, .name_loc = target->base.location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3497,7 +3497,7 @@ pm_constant_operator_write_node_create(pm_parser_t *parser, pm_constant_read_nod .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, .name_loc = target->base.location, - .binary_operator_loc = TOK2SLICE(parser, operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -3517,7 +3517,7 @@ pm_constant_or_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *t .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_OR_WRITE_NODE, 0, target, value), .name = target->name, .name_loc = target->base.location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3552,7 +3552,7 @@ pm_constant_write_node_create(pm_parser_t *parser, pm_constant_read_node_t *targ .base = PM_NODE_INIT_NODES(parser, PM_CONSTANT_WRITE_NODE, flags, target, value), .name = target->name, .name_loc = target->base.location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3635,17 +3635,17 @@ pm_def_node_create( : PM_NODE_INIT_TOKENS(parser, PM_DEF_NODE, 0, def_keyword, end_keyword) ), .name = name, - .name_loc = TOK2SLICE(parser, name_loc), + .name_loc = TOK2LOC(parser, name_loc), .receiver = receiver, .parameters = parameters, .body = body, .locals = *locals, - .def_keyword_loc = TOK2SLICE(parser, def_keyword), - .operator_loc = NTOK2SLICE(parser, operator), - .lparen_loc = NTOK2SLICE(parser, lparen), - .rparen_loc = NTOK2SLICE(parser, rparen), - .equal_loc = NTOK2SLICE(parser, equal), - .end_keyword_loc = NTOK2SLICE(parser, end_keyword) + .def_keyword_loc = TOK2LOC(parser, def_keyword), + .operator_loc = NTOK2LOC(parser, operator), + .lparen_loc = NTOK2LOC(parser, lparen), + .rparen_loc = NTOK2LOC(parser, rparen), + .equal_loc = NTOK2LOC(parser, equal), + .end_keyword_loc = NTOK2LOC(parser, end_keyword) }; return node; @@ -3664,10 +3664,10 @@ pm_defined_node_create(pm_parser_t *parser, const pm_token_t *lparen, pm_node_t ? PM_NODE_INIT_TOKEN_NODE(parser, PM_DEFINED_NODE, 0, keyword, value) : PM_NODE_INIT_TOKENS(parser, PM_DEFINED_NODE, 0, keyword, rparen) ), - .lparen_loc = NTOK2SLICE(parser, lparen), + .lparen_loc = NTOK2LOC(parser, lparen), .value = value, - .rparen_loc = NTOK2SLICE(parser, rparen), - .keyword_loc = TOK2SLICE(parser, keyword) + .rparen_loc = NTOK2LOC(parser, rparen), + .keyword_loc = TOK2LOC(parser, keyword) }; return node; @@ -3686,9 +3686,9 @@ pm_else_node_create(pm_parser_t *parser, const pm_token_t *else_keyword, pm_stat ? PM_NODE_INIT_TOKEN_NODE(parser, PM_ELSE_NODE, 0, else_keyword, statements) : PM_NODE_INIT_TOKENS(parser, PM_ELSE_NODE, 0, else_keyword, end_keyword) ), - .else_keyword_loc = TOK2SLICE(parser, else_keyword), + .else_keyword_loc = TOK2LOC(parser, else_keyword), .statements = statements, - .end_keyword_loc = NTOK2SLICE(parser, end_keyword) + .end_keyword_loc = NTOK2LOC(parser, end_keyword) }; return node; @@ -3703,9 +3703,9 @@ pm_embedded_statements_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_embedded_statements_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_EMBEDDED_STATEMENTS_NODE, 0, opening, closing), - .opening_loc = TOK2SLICE(parser, opening), + .opening_loc = TOK2LOC(parser, opening), .statements = statements, - .closing_loc = TOK2SLICE(parser, closing) + .closing_loc = TOK2LOC(parser, closing) }; return node; @@ -3720,7 +3720,7 @@ pm_embedded_variable_node_create(pm_parser_t *parser, const pm_token_t *operator *node = (pm_embedded_variable_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_EMBEDDED_VARIABLE_NODE, 0, operator, variable), - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .variable = variable }; @@ -3736,9 +3736,9 @@ pm_ensure_node_create(pm_parser_t *parser, const pm_token_t *ensure_keyword, pm_ *node = (pm_ensure_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_ENSURE_NODE, 0, ensure_keyword, end_keyword), - .ensure_keyword_loc = TOK2SLICE(parser, ensure_keyword), + .ensure_keyword_loc = TOK2LOC(parser, ensure_keyword), .statements = statements, - .end_keyword_loc = TOK2SLICE(parser, end_keyword) + .end_keyword_loc = TOK2LOC(parser, end_keyword) }; return node; @@ -4006,10 +4006,10 @@ pm_for_node_create( .index = index, .collection = collection, .statements = statements, - .for_keyword_loc = TOK2SLICE(parser, for_keyword), - .in_keyword_loc = TOK2SLICE(parser, in_keyword), - .do_keyword_loc = NTOK2SLICE(parser, do_keyword), - .end_keyword_loc = TOK2SLICE(parser, end_keyword) + .for_keyword_loc = TOK2LOC(parser, for_keyword), + .in_keyword_loc = TOK2LOC(parser, in_keyword), + .do_keyword_loc = NTOK2LOC(parser, do_keyword), + .end_keyword_loc = TOK2LOC(parser, end_keyword) }; return node; @@ -4082,8 +4082,8 @@ pm_hash_pattern_node_empty_create(pm_parser_t *parser, const pm_token_t *opening *node = (pm_hash_pattern_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_HASH_PATTERN_NODE, 0, opening, closing), .constant = NULL, - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, closing), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing), .elements = { 0 }, .rest = NULL }; @@ -4164,7 +4164,7 @@ pm_global_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), .name_loc = target->location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4182,7 +4182,7 @@ pm_global_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *ta .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), .name_loc = target->location, - .binary_operator_loc = TOK2SLICE(parser, operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -4202,7 +4202,7 @@ pm_global_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = pm_global_variable_write_name(parser, target), .name_loc = target->location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4251,7 +4251,7 @@ pm_global_variable_write_node_create(pm_parser_t *parser, pm_node_t *target, con .base = PM_NODE_INIT_NODES(parser, PM_GLOBAL_VARIABLE_WRITE_NODE, flags, target, value), .name = pm_global_variable_write_name(parser, target), .name_loc = target->location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4286,7 +4286,7 @@ pm_hash_node_create(pm_parser_t *parser, const pm_token_t *opening) { *node = (pm_hash_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_HASH_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = TOK2SLICE(parser, opening), + .opening_loc = TOK2LOC(parser, opening), .closing_loc = { 0 }, .elements = { 0 } }; @@ -4317,7 +4317,7 @@ pm_hash_node_elements_append(pm_hash_node_t *hash, pm_node_t *element) { static inline void pm_hash_node_closing_loc_set(const pm_parser_t *parser, pm_hash_node_t *hash, pm_token_t *token) { PM_NODE_LENGTH_SET_TOKEN(parser, hash, token); - hash->closing_loc = TOK2SLICE(parser, token); + hash->closing_loc = TOK2LOC(parser, token); } /** @@ -4350,12 +4350,12 @@ pm_if_node_create(pm_parser_t *parser, *node = (pm_if_node_t) { .base = PM_NODE_INIT(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, start, U32(end - start)), - .if_keyword_loc = TOK2SLICE(parser, if_keyword), + .if_keyword_loc = TOK2LOC(parser, if_keyword), .predicate = predicate, - .then_keyword_loc = NTOK2SLICE(parser, then_keyword), + .then_keyword_loc = NTOK2LOC(parser, then_keyword), .statements = statements, .subsequent = subsequent, - .end_keyword_loc = NTOK2SLICE(parser, end_keyword) + .end_keyword_loc = NTOK2LOC(parser, end_keyword) }; return node; @@ -4374,7 +4374,7 @@ pm_if_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const pm_t *node = (pm_if_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, statement, predicate), - .if_keyword_loc = TOK2SLICE(parser, if_keyword), + .if_keyword_loc = TOK2LOC(parser, if_keyword), .predicate = predicate, .then_keyword_loc = { 0 }, .statements = statements, @@ -4406,7 +4406,7 @@ pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_to .base = PM_NODE_INIT_NODES(parser, PM_IF_NODE, PM_NODE_FLAG_NEWLINE, predicate, false_expression), .if_keyword_loc = { 0 }, .predicate = predicate, - .then_keyword_loc = TOK2SLICE(parser, qmark), + .then_keyword_loc = TOK2LOC(parser, qmark), .statements = if_statements, .subsequent = UP(else_node), .end_keyword_loc = { 0 } @@ -4419,13 +4419,13 @@ pm_if_node_ternary_create(pm_parser_t *parser, pm_node_t *predicate, const pm_to static inline void pm_if_node_end_keyword_loc_set(const pm_parser_t *parser, pm_if_node_t *node, const pm_token_t *keyword) { PM_NODE_LENGTH_SET_TOKEN(parser, node, keyword); - node->end_keyword_loc = TOK2SLICE(parser, keyword); + node->end_keyword_loc = TOK2LOC(parser, keyword); } static inline void pm_else_node_end_keyword_loc_set(const pm_parser_t *parser, pm_else_node_t *node, const pm_token_t *keyword) { PM_NODE_LENGTH_SET_TOKEN(parser, node, keyword); - node->end_keyword_loc = TOK2SLICE(parser, keyword); + node->end_keyword_loc = TOK2LOC(parser, keyword); } /** @@ -4578,8 +4578,8 @@ pm_in_node_create(pm_parser_t *parser, pm_node_t *pattern, pm_statements_node_t .base = PM_NODE_INIT(parser, PM_IN_NODE, 0, start, U32(end - start)), .pattern = pattern, .statements = statements, - .in_loc = TOK2SLICE(parser, in_keyword), - .then_loc = NTOK2SLICE(parser, then_keyword) + .in_loc = TOK2LOC(parser, in_keyword), + .then_loc = NTOK2LOC(parser, then_keyword) }; return node; @@ -4597,7 +4597,7 @@ pm_instance_variable_and_write_node_create(pm_parser_t *parser, pm_instance_vari .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_AND_WRITE_NODE, 0, target, value), .name = target->name, .name_loc = target->base.location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4615,7 +4615,7 @@ pm_instance_variable_operator_write_node_create(pm_parser_t *parser, pm_instance .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name = target->name, .name_loc = target->base.location, - .binary_operator_loc = TOK2SLICE(parser, operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) }; @@ -4635,7 +4635,7 @@ pm_instance_variable_or_write_node_create(pm_parser_t *parser, pm_instance_varia .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_OR_WRITE_NODE, 0, target, value), .name = target->name, .name_loc = target->base.location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4671,7 +4671,7 @@ pm_instance_variable_write_node_create(pm_parser_t *parser, pm_instance_variable .base = PM_NODE_INIT_NODES(parser, PM_INSTANCE_VARIABLE_WRITE_NODE, flags, read_node, value), .name = read_node->name, .name_loc = read_node->base.location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -4731,8 +4731,8 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok *node = (pm_interpolated_regular_expression_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening), - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, opening), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, opening), .parts = { 0 } }; @@ -4753,7 +4753,7 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio static inline void pm_interpolated_regular_expression_node_closing_set(pm_parser_t *parser, pm_interpolated_regular_expression_node_t *node, const pm_token_t *closing) { - node->closing_loc = TOK2SLICE(parser, closing); + node->closing_loc = TOK2LOC(parser, closing); PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); pm_node_flag_set(UP(node), pm_regular_expression_flags_create(parser, closing)); } @@ -4895,8 +4895,8 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_string_node_t) { .base = PM_NODE_INIT(parser, PM_INTERPOLATED_STRING_NODE, flags, start, U32(end - start)), - .opening_loc = NTOK2SLICE(parser, opening), - .closing_loc = NTOK2SLICE(parser, closing), + .opening_loc = NTOK2LOC(parser, opening), + .closing_loc = NTOK2LOC(parser, closing), .parts = { 0 } }; @@ -4915,7 +4915,7 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin */ static void pm_interpolated_string_node_closing_set(const pm_parser_t *parser, pm_interpolated_string_node_t *node, const pm_token_t *closing) { - node->closing_loc = TOK2SLICE(parser, closing); + node->closing_loc = TOK2LOC(parser, closing); PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); } @@ -4934,7 +4934,7 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ static void pm_interpolated_symbol_node_closing_loc_set(const pm_parser_t *parser, pm_interpolated_symbol_node_t *node, const pm_token_t *closing) { - node->closing_loc = TOK2SLICE(parser, closing); + node->closing_loc = TOK2LOC(parser, closing); PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); } @@ -4950,8 +4950,8 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_symbol_node_t) { .base = PM_NODE_INIT(parser, PM_INTERPOLATED_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, start, U32(end - start)), - .opening_loc = NTOK2SLICE(parser, opening), - .closing_loc = NTOK2SLICE(parser, closing), + .opening_loc = NTOK2LOC(parser, opening), + .closing_loc = NTOK2LOC(parser, closing), .parts = { 0 } }; @@ -4974,8 +4974,8 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi *node = (pm_interpolated_x_string_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_INTERPOLATED_X_STRING_NODE, 0, opening, closing), - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, closing), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing), .parts = { 0 } }; @@ -4990,7 +4990,7 @@ pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_no static inline void pm_interpolated_xstring_node_closing_set(const pm_parser_t *parser, pm_interpolated_x_string_node_t *node, const pm_token_t *closing) { - node->closing_loc = TOK2SLICE(parser, closing); + node->closing_loc = TOK2LOC(parser, closing); PM_NODE_LENGTH_SET_TOKEN(parser, node, closing); } @@ -5065,7 +5065,7 @@ pm_required_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t *node = (pm_required_keyword_parameter_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_REQUIRED_KEYWORD_PARAMETER_NODE, 0, name), .name = pm_parser_constant_id_location(parser, name->start, name->end - 1), - .name_loc = TOK2SLICE(parser, name), + .name_loc = TOK2LOC(parser, name), }; return node; @@ -5081,7 +5081,7 @@ pm_optional_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t *node = (pm_optional_keyword_parameter_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_OPTIONAL_KEYWORD_PARAMETER_NODE, 0, name, value), .name = pm_parser_constant_id_location(parser, name->start, name->end - 1), - .name_loc = TOK2SLICE(parser, name), + .name_loc = TOK2LOC(parser, name), .value = value }; @@ -5102,8 +5102,8 @@ pm_keyword_rest_parameter_node_create(pm_parser_t *parser, const pm_token_t *ope : PM_NODE_INIT_TOKENS(parser, PM_KEYWORD_REST_PARAMETER_NODE, 0, operator, name) ), .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), - .name_loc = NTOK2SLICE(parser, name), - .operator_loc = TOK2SLICE(parser, operator) + .name_loc = NTOK2LOC(parser, name), + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5127,9 +5127,9 @@ pm_lambda_node_create( *node = (pm_lambda_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_LAMBDA_NODE, 0, operator, closing), .locals = *locals, - .operator_loc = TOK2SLICE(parser, operator), - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, closing), + .operator_loc = TOK2LOC(parser, operator), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing), .parameters = parameters, .body = body }; @@ -5149,7 +5149,7 @@ pm_local_variable_and_write_node_create(pm_parser_t *parser, pm_node_t *target, *node = (pm_local_variable_and_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_AND_WRITE_NODE, 0, target, value), .name_loc = target->location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value, .name = name, .depth = depth @@ -5168,7 +5168,7 @@ pm_local_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *tar *node = (pm_local_variable_operator_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_OPERATOR_WRITE_NODE, 0, target, value), .name_loc = target->location, - .binary_operator_loc = TOK2SLICE(parser, operator), + .binary_operator_loc = TOK2LOC(parser, operator), .value = value, .name = name, .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), @@ -5190,7 +5190,7 @@ pm_local_variable_or_write_node_create(pm_parser_t *parser, pm_node_t *target, c *node = (pm_local_variable_or_write_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_LOCAL_VARIABLE_OR_WRITE_NODE, 0, target, value), .name_loc = target->location, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value, .name = name, .depth = depth @@ -5240,7 +5240,7 @@ pm_local_variable_read_node_missing_create(pm_parser_t *parser, const pm_token_t * Allocate and initialize a new LocalVariableWriteNode node. */ static pm_local_variable_write_node_t * -pm_local_variable_write_node_create(pm_parser_t *parser, pm_constant_id_t name, uint32_t depth, pm_node_t *value, const pm_slice_t *name_loc, const pm_token_t *operator) { +pm_local_variable_write_node_create(pm_parser_t *parser, pm_constant_id_t name, uint32_t depth, pm_node_t *value, const pm_location_t *name_loc, const pm_token_t *operator) { pm_local_variable_write_node_t *node = PM_NODE_ALLOC(parser, pm_local_variable_write_node_t); pm_node_flags_t flags = pm_implicit_array_write_flags(value, PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY); @@ -5250,7 +5250,7 @@ pm_local_variable_write_node_create(pm_parser_t *parser, pm_constant_id_t name, .depth = depth, .value = value, .name_loc = *name_loc, - .operator_loc = TOK2SLICE(parser, operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5294,7 +5294,7 @@ pm_refute_numbered_parameter(pm_parser_t *parser, uint32_t start, uint32_t lengt * name and depth. */ static pm_local_variable_target_node_t * -pm_local_variable_target_node_create(pm_parser_t *parser, const pm_slice_t *location, pm_constant_id_t name, uint32_t depth) { +pm_local_variable_target_node_create(pm_parser_t *parser, const pm_location_t *location, pm_constant_id_t name, uint32_t depth) { pm_refute_numbered_parameter(parser, location->start, location->length); pm_local_variable_target_node_t *node = PM_NODE_ALLOC(parser, pm_local_variable_target_node_t); @@ -5320,7 +5320,7 @@ pm_match_predicate_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t .base = PM_NODE_INIT_NODES(parser, PM_MATCH_PREDICATE_NODE, 0, value, pattern), .value = value, .pattern = pattern, - .operator_loc = TOK2SLICE(parser, operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5339,7 +5339,7 @@ pm_match_required_node_create(pm_parser_t *parser, pm_node_t *value, pm_node_t * .base = PM_NODE_INIT_NODES(parser, PM_MATCH_REQUIRED_NODE, 0, value, pattern), .value = value, .pattern = pattern, - .operator_loc = TOK2SLICE(parser, operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5371,10 +5371,10 @@ pm_module_node_create(pm_parser_t *parser, pm_constant_id_list_t *locals, const *node = (pm_module_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_MODULE_NODE, 0, module_keyword, end_keyword), .locals = (locals == NULL ? ((pm_constant_id_list_t) { .ids = NULL, .size = 0, .capacity = 0 }) : *locals), - .module_keyword_loc = TOK2SLICE(parser, module_keyword), + .module_keyword_loc = TOK2LOC(parser, module_keyword), .constant_path = constant_path, .body = body, - .end_keyword_loc = TOK2SLICE(parser, end_keyword), + .end_keyword_loc = TOK2LOC(parser, end_keyword), .name = pm_parser_constant_id_token(parser, name) }; @@ -5441,7 +5441,7 @@ static void pm_multi_target_node_opening_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *lparen) { PM_NODE_START_SET_TOKEN(parser, node, lparen); PM_NODE_LENGTH_SET_TOKEN(parser, node, lparen); - node->lparen_loc = TOK2SLICE(parser, lparen); + node->lparen_loc = TOK2LOC(parser, lparen); } /** @@ -5450,7 +5450,7 @@ pm_multi_target_node_opening_set(const pm_parser_t *parser, pm_multi_target_node static void pm_multi_target_node_closing_set(const pm_parser_t *parser, pm_multi_target_node_t *node, const pm_token_t *rparen) { PM_NODE_LENGTH_SET_TOKEN(parser, node, rparen); - node->rparen_loc = TOK2SLICE(parser, rparen); + node->rparen_loc = TOK2LOC(parser, rparen); } /** @@ -5468,7 +5468,7 @@ pm_multi_write_node_create(pm_parser_t *parser, pm_multi_target_node_t *target, .rights = target->rights, .lparen_loc = target->lparen_loc, .rparen_loc = target->rparen_loc, - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -5493,7 +5493,7 @@ pm_next_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_arguments ? PM_NODE_INIT_TOKEN(parser, PM_NEXT_NODE, 0, keyword) : PM_NODE_INIT_TOKEN_NODE(parser, PM_NEXT_NODE, 0, keyword, arguments) ), - .keyword_loc = TOK2SLICE(parser, keyword), + .keyword_loc = TOK2LOC(parser, keyword), .arguments = arguments }; @@ -5526,8 +5526,8 @@ pm_no_keywords_parameter_node_create(pm_parser_t *parser, const pm_token_t *oper *node = (pm_no_keywords_parameter_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_NO_KEYWORDS_PARAMETER_NODE, 0, operator, keyword), - .operator_loc = TOK2SLICE(parser, operator), - .keyword_loc = TOK2SLICE(parser, keyword) + .operator_loc = TOK2LOC(parser, operator), + .keyword_loc = TOK2LOC(parser, keyword) }; return node; @@ -5623,8 +5623,8 @@ pm_optional_parameter_node_create(pm_parser_t *parser, const pm_token_t *name, c *node = (pm_optional_parameter_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_OPTIONAL_PARAMETER_NODE, 0, name, value), .name = pm_parser_constant_id_token(parser, name), - .name_loc = TOK2SLICE(parser, name), - .operator_loc = TOK2SLICE(parser, operator), + .name_loc = TOK2LOC(parser, name), + .operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -5644,7 +5644,7 @@ pm_or_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *operat .base = PM_NODE_INIT_NODES(parser, PM_OR_NODE, 0, left, right), .left = left, .right = right, - .operator_loc = TOK2SLICE(parser, operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5776,8 +5776,8 @@ pm_parentheses_node_create(pm_parser_t *parser, const pm_token_t *opening, pm_no *node = (pm_parentheses_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_PARENTHESES_NODE, flags, opening, closing), .body = body, - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, closing) + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing) }; return node; @@ -5793,9 +5793,9 @@ pm_pinned_expression_node_create(pm_parser_t *parser, pm_node_t *expression, con *node = (pm_pinned_expression_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_PINNED_EXPRESSION_NODE, 0, operator, rparen), .expression = expression, - .operator_loc = TOK2SLICE(parser, operator), - .lparen_loc = TOK2SLICE(parser, lparen), - .rparen_loc = TOK2SLICE(parser, rparen) + .operator_loc = TOK2LOC(parser, operator), + .lparen_loc = TOK2LOC(parser, lparen), + .rparen_loc = TOK2LOC(parser, rparen) }; return node; @@ -5811,7 +5811,7 @@ pm_pinned_variable_node_create(pm_parser_t *parser, const pm_token_t *operator, *node = (pm_pinned_variable_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_PINNED_VARIABLE_NODE, 0, operator, variable), .variable = variable, - .operator_loc = TOK2SLICE(parser, operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5827,9 +5827,9 @@ pm_post_execution_node_create(pm_parser_t *parser, const pm_token_t *keyword, co *node = (pm_post_execution_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_POST_EXECUTION_NODE, 0, keyword, closing), .statements = statements, - .keyword_loc = TOK2SLICE(parser, keyword), - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, closing) + .keyword_loc = TOK2LOC(parser, keyword), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing) }; return node; @@ -5845,9 +5845,9 @@ pm_pre_execution_node_create(pm_parser_t *parser, const pm_token_t *keyword, con *node = (pm_pre_execution_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_PRE_EXECUTION_NODE, 0, keyword, closing), .statements = statements, - .keyword_loc = TOK2SLICE(parser, keyword), - .opening_loc = TOK2SLICE(parser, opening), - .closing_loc = TOK2SLICE(parser, closing) + .keyword_loc = TOK2LOC(parser, keyword), + .opening_loc = TOK2LOC(parser, opening), + .closing_loc = TOK2LOC(parser, closing) }; return node; @@ -5886,7 +5886,7 @@ pm_range_node_create(pm_parser_t *parser, pm_node_t *left, const pm_token_t *ope .base = PM_NODE_INIT(parser, PM_RANGE_NODE, flags, start, U32(end - start)), .left = left, .right = right, - .operator_loc = TOK2SLICE(parser, operator) + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -5918,9 +5918,9 @@ pm_regular_expression_node_create_unescaped(pm_parser_t *parser, const pm_token_ *node = (pm_regular_expression_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_REGULAR_EXPRESSION_NODE, flags, opening, closing), - .opening_loc = TOK2SLICE(parser, opening), - .content_loc = TOK2SLICE(parser, content), - .closing_loc = TOK2SLICE(parser, closing), + .opening_loc = TOK2LOC(parser, opening), + .content_loc = TOK2LOC(parser, content), + .closing_loc = TOK2LOC(parser, closing), .unescaped = *unescaped }; @@ -5960,7 +5960,7 @@ pm_rescue_modifier_node_create(pm_parser_t *parser, pm_node_t *expression, const *node = (pm_rescue_modifier_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_RESCUE_MODIFIER_NODE, 0, expression, rescue_expression), .expression = expression, - .keyword_loc = TOK2SLICE(parser, keyword), + .keyword_loc = TOK2LOC(parser, keyword), .rescue_expression = rescue_expression }; @@ -5976,7 +5976,7 @@ pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword) { *node = (pm_rescue_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_RESCUE_NODE, 0, keyword), - .keyword_loc = TOK2SLICE(parser, keyword), + .keyword_loc = TOK2LOC(parser, keyword), .operator_loc = { 0 }, .then_keyword_loc = { 0 }, .reference = NULL, @@ -5990,7 +5990,7 @@ pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword) { static inline void pm_rescue_node_operator_set(const pm_parser_t *parser, pm_rescue_node_t *node, const pm_token_t *operator) { - node->operator_loc = TOK2SLICE(parser, operator); + node->operator_loc = TOK2LOC(parser, operator); } /** @@ -6045,8 +6045,8 @@ pm_rest_parameter_node_create(pm_parser_t *parser, const pm_token_t *operator, c : PM_NODE_INIT_TOKENS(parser, PM_REST_PARAMETER_NODE, 0, operator, name) ), .name = name == NULL ? 0 : pm_parser_constant_id_token(parser, name), - .name_loc = NTOK2SLICE(parser, name), - .operator_loc = TOK2SLICE(parser, operator) + .name_loc = NTOK2LOC(parser, name), + .operator_loc = TOK2LOC(parser, operator) }; return node; @@ -6080,7 +6080,7 @@ pm_return_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argumen ? PM_NODE_INIT_TOKEN(parser, PM_RETURN_NODE, 0, keyword) : PM_NODE_INIT_TOKEN_NODE(parser, PM_RETURN_NODE, 0, keyword, arguments) ), - .keyword_loc = TOK2SLICE(parser, keyword), + .keyword_loc = TOK2LOC(parser, keyword), .arguments = arguments }; @@ -6127,11 +6127,11 @@ pm_singleton_class_node_create(pm_parser_t *parser, pm_constant_id_list_t *local *node = (pm_singleton_class_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_SINGLETON_CLASS_NODE, 0, class_keyword, end_keyword), .locals = *locals, - .class_keyword_loc = TOK2SLICE(parser, class_keyword), - .operator_loc = TOK2SLICE(parser, operator), + .class_keyword_loc = TOK2LOC(parser, class_keyword), + .operator_loc = TOK2LOC(parser, operator), .expression = expression, .body = body, - .end_keyword_loc = TOK2SLICE(parser, end_keyword) + .end_keyword_loc = TOK2LOC(parser, end_keyword) }; return node; @@ -6207,7 +6207,7 @@ pm_splat_node_create(pm_parser_t *parser, const pm_token_t *operator, pm_node_t ? PM_NODE_INIT_TOKEN(parser, PM_SPLAT_NODE, 0, operator) : PM_NODE_INIT_TOKEN_NODE(parser, PM_SPLAT_NODE, 0, operator, expression) ), - .operator_loc = TOK2SLICE(parser, operator), + .operator_loc = TOK2LOC(parser, operator), .expression = expression }; @@ -6311,9 +6311,9 @@ pm_string_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, *node = (pm_string_node_t) { .base = PM_NODE_INIT(parser, PM_STRING_NODE, flags, start, U32(end - start)), - .opening_loc = NTOK2SLICE(parser, opening), - .content_loc = TOK2SLICE(parser, content), - .closing_loc = NTOK2SLICE(parser, closing), + .opening_loc = NTOK2LOC(parser, opening), + .content_loc = TOK2LOC(parser, content), + .closing_loc = NTOK2LOC(parser, closing), .unescaped = *string }; @@ -6347,12 +6347,12 @@ pm_super_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument assert(keyword->type == PM_TOKEN_KEYWORD_SUPER); pm_super_node_t *node = PM_NODE_ALLOC(parser, pm_super_node_t); - const pm_slice_t *end = pm_arguments_end(arguments); + const pm_location_t *end = pm_arguments_end(arguments); assert(end != NULL && "unreachable"); *node = (pm_super_node_t) { - .base = PM_NODE_INIT(parser, PM_SUPER_NODE, 0, PM_TOKEN_START(parser, keyword), PM_SLICE_END(end) - PM_TOKEN_START(parser, keyword)), - .keyword_loc = TOK2SLICE(parser, keyword), + .base = PM_NODE_INIT(parser, PM_SUPER_NODE, 0, PM_TOKEN_START(parser, keyword), PM_LOCATION_END(end) - PM_TOKEN_START(parser, keyword)), + .keyword_loc = TOK2LOC(parser, keyword), .lparen_loc = arguments->opening_loc, .arguments = arguments->arguments, .rparen_loc = arguments->closing_loc, @@ -6585,9 +6585,9 @@ pm_symbol_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, *node = (pm_symbol_node_t) { .base = PM_NODE_INIT(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL | flags, start, U32(end - start)), - .opening_loc = NTOK2SLICE(parser, opening), - .value_loc = NTOK2SLICE(parser, value), - .closing_loc = NTOK2SLICE(parser, closing), + .opening_loc = NTOK2LOC(parser, opening), + .value_loc = NTOK2LOC(parser, value), + .closing_loc = NTOK2LOC(parser, closing), .unescaped = *unescaped }; @@ -6685,9 +6685,9 @@ pm_string_node_to_symbol_node(pm_parser_t *parser, pm_string_node_t *node, const *new_node = (pm_symbol_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_SYMBOL_NODE, PM_NODE_FLAG_STATIC_LITERAL, opening, closing), - .opening_loc = TOK2SLICE(parser, opening), + .opening_loc = TOK2LOC(parser, opening), .value_loc = node->content_loc, - .closing_loc = TOK2SLICE(parser, closing), + .closing_loc = TOK2LOC(parser, closing), .unescaped = node->unescaped }; @@ -6779,7 +6779,7 @@ pm_undef_node_create(pm_parser_t *parser, const pm_token_t *token) { *node = (pm_undef_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_UNDEF_NODE, 0, token), - .keyword_loc = TOK2SLICE(parser, token), + .keyword_loc = TOK2LOC(parser, token), .names = { 0 } }; @@ -6807,9 +6807,9 @@ pm_unless_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_node_t *node = (pm_unless_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_UNLESS_NODE, PM_NODE_FLAG_NEWLINE, keyword, end), - .keyword_loc = TOK2SLICE(parser, keyword), + .keyword_loc = TOK2LOC(parser, keyword), .predicate = predicate, - .then_keyword_loc = NTOK2SLICE(parser, then_keyword), + .then_keyword_loc = NTOK2LOC(parser, then_keyword), .statements = statements, .else_clause = NULL, .end_keyword_loc = { 0 } @@ -6831,7 +6831,7 @@ pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const *node = (pm_unless_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_UNLESS_NODE, PM_NODE_FLAG_NEWLINE, statement, predicate), - .keyword_loc = TOK2SLICE(parser, unless_keyword), + .keyword_loc = TOK2LOC(parser, unless_keyword), .predicate = predicate, .then_keyword_loc = { 0 }, .statements = statements, @@ -6844,7 +6844,7 @@ pm_unless_node_modifier_create(pm_parser_t *parser, pm_node_t *statement, const static inline void pm_unless_node_end_keyword_loc_set(const pm_parser_t *parser, pm_unless_node_t *node, const pm_token_t *end_keyword) { - node->end_keyword_loc = TOK2SLICE(parser, end_keyword); + node->end_keyword_loc = TOK2LOC(parser, end_keyword); PM_NODE_LENGTH_SET_TOKEN(parser, node, end_keyword); } @@ -6881,9 +6881,9 @@ pm_until_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_to *node = (pm_until_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_UNTIL_NODE, flags, keyword, closing), - .keyword_loc = TOK2SLICE(parser, keyword), - .do_keyword_loc = NTOK2SLICE(parser, do_keyword), - .closing_loc = TOK2SLICE(parser, closing), + .keyword_loc = TOK2LOC(parser, keyword), + .do_keyword_loc = NTOK2LOC(parser, do_keyword), + .closing_loc = TOK2LOC(parser, closing), .predicate = predicate, .statements = statements }; @@ -6902,7 +6902,7 @@ pm_until_node_modifier_create(pm_parser_t *parser, const pm_token_t *keyword, pm *node = (pm_until_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_UNTIL_NODE, flags, statements, predicate), - .keyword_loc = TOK2SLICE(parser, keyword), + .keyword_loc = TOK2LOC(parser, keyword), .do_keyword_loc = { 0 }, .closing_loc = { 0 }, .predicate = predicate, @@ -6921,7 +6921,7 @@ pm_when_node_create(pm_parser_t *parser, const pm_token_t *keyword) { *node = (pm_when_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_WHEN_NODE, 0, keyword), - .keyword_loc = TOK2SLICE(parser, keyword), + .keyword_loc = TOK2LOC(parser, keyword), .statements = NULL, .then_keyword_loc = { 0 }, .conditions = { 0 } @@ -6945,7 +6945,7 @@ pm_when_node_conditions_append(pm_when_node_t *node, pm_node_t *condition) { static inline void pm_when_node_then_keyword_loc_set(const pm_parser_t *parser, pm_when_node_t *node, const pm_token_t *then_keyword) { PM_NODE_LENGTH_SET_TOKEN(parser, node, then_keyword); - node->then_keyword_loc = TOK2SLICE(parser, then_keyword); + node->then_keyword_loc = TOK2LOC(parser, then_keyword); } /** @@ -6970,9 +6970,9 @@ pm_while_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_to *node = (pm_while_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_WHILE_NODE, flags, keyword, closing), - .keyword_loc = TOK2SLICE(parser, keyword), - .do_keyword_loc = NTOK2SLICE(parser, do_keyword), - .closing_loc = TOK2SLICE(parser, closing), + .keyword_loc = TOK2LOC(parser, keyword), + .do_keyword_loc = NTOK2LOC(parser, do_keyword), + .closing_loc = TOK2LOC(parser, closing), .predicate = predicate, .statements = statements }; @@ -6991,7 +6991,7 @@ pm_while_node_modifier_create(pm_parser_t *parser, const pm_token_t *keyword, pm *node = (pm_while_node_t) { .base = PM_NODE_INIT_NODES(parser, PM_WHILE_NODE, flags, statements, predicate), - .keyword_loc = TOK2SLICE(parser, keyword), + .keyword_loc = TOK2LOC(parser, keyword), .do_keyword_loc = { 0 }, .closing_loc = { 0 }, .predicate = predicate, @@ -7030,9 +7030,9 @@ pm_xstring_node_create_unescaped(pm_parser_t *parser, const pm_token_t *opening, *node = (pm_x_string_node_t) { .base = PM_NODE_INIT_TOKENS(parser, PM_X_STRING_NODE, PM_STRING_FLAGS_FROZEN, opening, closing), - .opening_loc = TOK2SLICE(parser, opening), - .content_loc = TOK2SLICE(parser, content), - .closing_loc = TOK2SLICE(parser, closing), + .opening_loc = TOK2LOC(parser, opening), + .content_loc = TOK2LOC(parser, content), + .closing_loc = TOK2LOC(parser, closing), .unescaped = *unescaped }; @@ -7051,25 +7051,25 @@ pm_xstring_node_create(pm_parser_t *parser, const pm_token_t *opening, const pm_ * Allocate a new YieldNode node. */ static pm_yield_node_t * -pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_slice_t *lparen_loc, pm_arguments_node_t *arguments, const pm_slice_t *rparen_loc) { +pm_yield_node_create(pm_parser_t *parser, const pm_token_t *keyword, const pm_location_t *lparen_loc, pm_arguments_node_t *arguments, const pm_location_t *rparen_loc) { pm_yield_node_t *node = PM_NODE_ALLOC(parser, pm_yield_node_t); uint32_t start = PM_TOKEN_START(parser, keyword); uint32_t end; if (rparen_loc->length > 0) { - end = PM_SLICE_END(rparen_loc); + end = PM_LOCATION_END(rparen_loc); } else if (arguments != NULL) { end = PM_NODE_END(arguments); } else if (lparen_loc->length > 0) { - end = PM_SLICE_END(lparen_loc); + end = PM_LOCATION_END(lparen_loc); } else { end = PM_TOKEN_END(parser, keyword); } *node = (pm_yield_node_t) { .base = PM_NODE_INIT(parser, PM_YIELD_NODE, 0, start, U32(end - start)), - .keyword_loc = TOK2SLICE(parser, keyword), + .keyword_loc = TOK2LOC(parser, keyword), .lparen_loc = *lparen_loc, .arguments = arguments, .rparen_loc = *rparen_loc @@ -7120,18 +7120,18 @@ pm_parser_local_add(pm_parser_t *parser, pm_constant_id_t constant_id, const uin * Add a local variable from a location to the current scope. */ static pm_constant_id_t -pm_parser_local_add_location(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, uint32_t reads) { +pm_parser_local_add_raw(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, uint32_t reads) { pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, start, end); if (constant_id != 0) pm_parser_local_add(parser, constant_id, start, end, reads); return constant_id; } /** - * Add a local variable from a slice to the current scope. + * Add a local variable from a location to the current scope. */ static pm_constant_id_t -pm_parser_local_add_slice(pm_parser_t *parser, pm_slice_t *slice, uint32_t reads) { - return pm_parser_local_add_location(parser, parser->start + slice->start, parser->start + slice->start + slice->length, reads); +pm_parser_local_add_location(pm_parser_t *parser, pm_location_t *location, uint32_t reads) { + return pm_parser_local_add_raw(parser, parser->start + location->start, parser->start + location->start + location->length, reads); } /** @@ -7139,7 +7139,7 @@ pm_parser_local_add_slice(pm_parser_t *parser, pm_slice_t *slice, uint32_t reads */ static inline pm_constant_id_t pm_parser_local_add_token(pm_parser_t *parser, pm_token_t *token, uint32_t reads) { - return pm_parser_local_add_location(parser, token->start, token->end, reads); + return pm_parser_local_add_raw(parser, token->start, token->end, reads); } /** @@ -7686,8 +7686,8 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { // Allocate a new magic comment node to append to the parser's list. pm_magic_comment_t *magic_comment; if ((magic_comment = (pm_magic_comment_t *) xcalloc(1, sizeof(pm_magic_comment_t))) != NULL) { - magic_comment->key = (pm_slice_t) { .start = U32(key_start - parser->start), .length = U32(key_length) }; - magic_comment->value = (pm_slice_t) { .start = U32(value_start - parser->start), .length = value_length }; + magic_comment->key = (pm_location_t) { .start = U32(key_start - parser->start), .length = U32(key_length) }; + magic_comment->value = (pm_location_t) { .start = U32(value_start - parser->start), .length = value_length }; pm_list_append(&parser->magic_comment_list, (pm_list_node_t *) magic_comment); } } @@ -9313,7 +9313,7 @@ parser_comment(pm_parser_t *parser, pm_comment_type_t type) { *comment = (pm_comment_t) { .type = type, - .location = TOK2SLICE(parser, &parser->current) + .location = TOK2LOC(parser, &parser->current) }; return comment; @@ -12764,8 +12764,8 @@ parse_target(pm_parser_t *parser, pm_node_t *target, bool multiple, bool splat_p // When it was parsed in the prefix position, foo was seen as a // method call with no receiver and no arguments. Now we have an // =, so we know it's a local variable write. - pm_slice_t message_loc = call->message_loc; - pm_constant_id_t name = pm_parser_local_add_slice(parser, &message_loc, 0); + pm_location_t message_loc = call->message_loc; + pm_constant_id_t name = pm_parser_local_add_location(parser, &message_loc, 0); pm_node_destroy(parser, target); return UP(pm_local_variable_target_node_create(parser, &message_loc, name, 0)); @@ -12879,7 +12879,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod case PM_LOCAL_VARIABLE_READ_NODE: { pm_local_variable_read_node_t *local_read = (pm_local_variable_read_node_t *) target; - pm_slice_t location = target->location; + pm_location_t location = target->location; pm_constant_id_t name = local_read->name; uint32_t depth = local_read->depth; pm_scope_t *scope = pm_parser_scope_find(parser, depth); @@ -12947,13 +12947,13 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod // When it was parsed in the prefix position, foo was seen as a // method call with no receiver and no arguments. Now we have an // =, so we know it's a local variable write. - pm_slice_t message_loc = call->message_loc; + pm_location_t message_loc = call->message_loc; pm_refute_numbered_parameter(parser, message_loc.start, message_loc.length); - pm_parser_local_add_slice(parser, &message_loc, 0); + pm_parser_local_add_location(parser, &message_loc, 0); pm_node_destroy(parser, target); - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, parser->start + PM_SLICE_START(&message_loc), parser->start + PM_SLICE_END(&message_loc)); + pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, parser->start + PM_LOCATION_START(&message_loc), parser->start + PM_LOCATION_END(&message_loc)); target = UP(pm_local_variable_write_node_create(parser, constant_id, 0, value, &message_loc, operator)); return target; @@ -12975,7 +12975,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod pm_arguments_node_arguments_append(arguments, value); PM_NODE_LENGTH_SET_NODE(call, arguments); - call->equal_loc = TOK2SLICE(parser, operator); + call->equal_loc = TOK2LOC(parser, operator); parse_write_name(parser, &call->name); pm_node_flag_set(UP(call), PM_CALL_NODE_FLAGS_ATTRIBUTE_WRITE | pm_implicit_array_write_flags(value, PM_CALL_NODE_FLAGS_IMPLICIT_ARRAY)); @@ -12997,7 +12997,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod // Replace the name with "[]=". call->name = pm_parser_constant_id_constant(parser, "[]=", 3); - call->equal_loc = TOK2SLICE(parser, operator); + call->equal_loc = TOK2LOC(parser, operator); // Ensure that the arguments for []= don't contain keywords pm_index_arguments_check(parser, call->arguments, call->block); @@ -13048,7 +13048,7 @@ parse_unwriteable_write(pm_parser_t *parser, pm_node_t *target, const pm_token_t default: break; } - pm_constant_id_t name = pm_parser_local_add_location(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target), 1); + pm_constant_id_t name = pm_parser_local_add_raw(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target), 1); pm_local_variable_write_node_t *result = pm_local_variable_write_node_create(parser, name, 0, value, &target->location, equals); pm_node_destroy(parser, target); @@ -14356,11 +14356,11 @@ parse_rescues(pm_parser_t *parser, size_t opening_newline_index, const pm_token_ if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) { - rescue->then_keyword_loc = TOK2SLICE(parser, &parser->previous); + rescue->then_keyword_loc = TOK2LOC(parser, &parser->previous); } } else { expect1(parser, PM_TOKEN_KEYWORD_THEN, PM_ERR_RESCUE_TERM); - rescue->then_keyword_loc = TOK2SLICE(parser, &parser->previous); + rescue->then_keyword_loc = TOK2LOC(parser, &parser->previous); } if (!match3(parser, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_ENSURE, PM_TOKEN_KEYWORD_END)) { @@ -14756,10 +14756,10 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept if (accept1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { found |= true; - arguments->opening_loc = TOK2SLICE(parser, &parser->previous); + arguments->opening_loc = TOK2LOC(parser, &parser->previous); if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - arguments->closing_loc = TOK2SLICE(parser, &parser->previous); + arguments->closing_loc = TOK2LOC(parser, &parser->previous); } else { pm_accepts_block_stack_push(parser, true); parse_arguments(parser, arguments, accepts_block, PM_TOKEN_PARENTHESIS_RIGHT, (uint16_t) (depth + 1)); @@ -14771,7 +14771,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool accept } pm_accepts_block_stack_pop(parser); - arguments->closing_loc = TOK2SLICE(parser, &parser->previous); + arguments->closing_loc = TOK2LOC(parser, &parser->previous); } } else if (accepts_command_call && (token_begins_expression_p(parser->current.type) || match3(parser, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR, PM_TOKEN_UAMPERSAND)) && !match1(parser, PM_TOKEN_BRACE_LEFT)) { found |= true; @@ -16085,7 +16085,7 @@ parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, uint8_t flag * an error to the parser. */ static void -parse_pattern_capture(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_constant_id_t capture, const pm_slice_t *location) { +parse_pattern_capture(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_constant_id_t capture, const pm_location_t *location) { // Skip this capture if it starts with an underscore. if (parser->start[location->start] == '_') return; @@ -16164,8 +16164,8 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->constant = node; - pattern_node->opening_loc = TOK2SLICE(parser, &opening); - pattern_node->closing_loc = TOK2SLICE(parser, &closing); + pattern_node->opening_loc = TOK2LOC(parser, &opening); + pattern_node->closing_loc = TOK2LOC(parser, &closing); return UP(pattern_node); } @@ -16180,8 +16180,8 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->constant = node; - pattern_node->opening_loc = TOK2SLICE(parser, &opening); - pattern_node->closing_loc = TOK2SLICE(parser, &closing); + pattern_node->opening_loc = TOK2LOC(parser, &opening); + pattern_node->closing_loc = TOK2LOC(parser, &closing); return UP(pattern_node); } @@ -16196,8 +16196,8 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); pattern_node->constant = node; - pattern_node->opening_loc = TOK2SLICE(parser, &opening); - pattern_node->closing_loc = TOK2SLICE(parser, &closing); + pattern_node->opening_loc = TOK2LOC(parser, &opening); + pattern_node->closing_loc = TOK2LOC(parser, &closing); return UP(pattern_node); } @@ -16236,10 +16236,10 @@ parse_pattern_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) { pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &TOK2SLICE(parser, &parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2LOC(parser, &parser->previous)); name = UP(pm_local_variable_target_node_create( parser, - &TOK2SLICE(parser, &parser->previous), + &TOK2LOC(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) )); @@ -16272,10 +16272,10 @@ parse_pattern_keyword_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &TOK2SLICE(parser, &parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2LOC(parser, &parser->previous)); value = UP(pm_local_variable_target_node_create( parser, - &TOK2SLICE(parser, &parser->previous), + &TOK2LOC(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) )); @@ -16317,9 +16317,9 @@ pm_slice_is_valid_local(const pm_parser_t *parser, const uint8_t *start, const u */ static pm_node_t * parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_symbol_node_t *key) { - const pm_slice_t *value_slice = &((pm_symbol_node_t *) key)->value_loc; - const uint8_t *start = parser->start + PM_SLICE_START(value_slice); - const uint8_t *end = parser->start + PM_SLICE_END(value_slice); + const pm_location_t *value_loc = &((pm_symbol_node_t *) key)->value_loc; + const uint8_t *start = parser->start + PM_LOCATION_START(value_loc); + const uint8_t *end = parser->start + PM_LOCATION_END(value_loc); pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, start, end); int depth = -1; @@ -16330,7 +16330,7 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca pm_parser_err(parser, PM_NODE_START(key), PM_NODE_LENGTH(key), PM_ERR_PATTERN_HASH_KEY_LOCALS); if ((end > start) && ((end[-1] == '!') || (end[-1] == '?'))) { - PM_PARSER_ERR_FORMAT(parser, value_slice->start, value_slice->length, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (end - start), (const char *) start); + PM_PARSER_ERR_FORMAT(parser, value_loc->start, value_loc->length, PM_ERR_INVALID_LOCAL_VARIABLE_WRITE, (int) (end - start), (const char *) start); } } @@ -16338,10 +16338,10 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca pm_parser_local_add(parser, constant_id, start, end, 0); } - parse_pattern_capture(parser, captures, constant_id, value_slice); + parse_pattern_capture(parser, captures, constant_id, value_loc); pm_local_variable_target_node_t *target = pm_local_variable_target_node_create( parser, - value_slice, + value_loc, constant_id, (uint32_t) (depth == -1 ? 0 : depth) ); @@ -16497,10 +16497,10 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &TOK2SLICE(parser, &parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2LOC(parser, &parser->previous)); return UP(pm_local_variable_target_node_create( parser, - &TOK2SLICE(parser, &parser->previous), + &TOK2LOC(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) )); @@ -16530,8 +16530,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm PM_NODE_START_SET_TOKEN(parser, pattern_node, &opening); PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); - pattern_node->opening_loc = TOK2SLICE(parser, &opening); - pattern_node->closing_loc = TOK2SLICE(parser, &closing); + pattern_node->opening_loc = TOK2LOC(parser, &opening); + pattern_node->closing_loc = TOK2LOC(parser, &closing); return UP(pattern_node); } @@ -16544,8 +16544,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm PM_NODE_START_SET_TOKEN(parser, pattern_node, &opening); PM_NODE_LENGTH_SET_TOKEN(parser, pattern_node, &closing); - pattern_node->opening_loc = TOK2SLICE(parser, &opening); - pattern_node->closing_loc = TOK2SLICE(parser, &closing); + pattern_node->opening_loc = TOK2LOC(parser, &opening); + pattern_node->closing_loc = TOK2LOC(parser, &closing); return UP(pattern_node); } @@ -16604,8 +16604,8 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm PM_NODE_START_SET_TOKEN(parser, node, &opening); PM_NODE_LENGTH_SET_TOKEN(parser, node, &closing); - node->opening_loc = TOK2SLICE(parser, &opening); - node->closing_loc = TOK2SLICE(parser, &closing); + node->opening_loc = TOK2LOC(parser, &opening); + node->closing_loc = TOK2LOC(parser, &closing); } parser->pattern_matching_newlines = previous_pattern_matching_newlines; @@ -16863,10 +16863,10 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p pm_parser_local_add(parser, constant_id, parser->previous.start, parser->previous.end, 0); } - parse_pattern_capture(parser, captures, constant_id, &TOK2SLICE(parser, &parser->previous)); + parse_pattern_capture(parser, captures, constant_id, &TOK2LOC(parser, &parser->previous)); pm_local_variable_target_node_t *target = pm_local_variable_target_node_create( parser, - &TOK2SLICE(parser, &parser->previous), + &TOK2LOC(parser, &parser->previous), constant_id, (uint32_t) (depth == -1 ? 0 : depth) ); @@ -17531,8 +17531,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_multi_target_node_targets_append(parser, multi_target, statement); } - multi_target->lparen_loc = TOK2SLICE(parser, &opening); - multi_target->rparen_loc = TOK2SLICE(parser, &parser->previous); + multi_target->lparen_loc = TOK2LOC(parser, &opening); + multi_target->rparen_loc = TOK2LOC(parser, &parser->previous); PM_NODE_START_SET_TOKEN(parser, multi_target, &opening); PM_NODE_LENGTH_SET_TOKEN(parser, multi_target, &parser->previous); @@ -17854,11 +17854,11 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b call->closing_loc = arguments.closing_loc; call->block = arguments.block; - const pm_slice_t *end = pm_arguments_end(&arguments); + const pm_location_t *end = pm_arguments_end(&arguments); if (end == NULL) { - PM_NODE_LENGTH_SET_SLICE(call, &call->message_loc); + PM_NODE_LENGTH_SET_LOCATION(call, &call->message_loc); } else { - PM_NODE_LENGTH_SET_SLICE(call, end); + PM_NODE_LENGTH_SET_LOCATION(call, end); } } } else { @@ -17949,8 +17949,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_node_flag_set(part, parse_unescaped_encoding(parser)); pm_string_node_t *cast = (pm_string_node_t *) part; - cast->opening_loc = TOK2SLICE(parser, &opening); - cast->closing_loc = TOK2SLICE(parser, &parser->current); + cast->opening_loc = TOK2LOC(parser, &opening); + cast->closing_loc = TOK2LOC(parser, &parser->current); cast->base.location = cast->opening_loc; if (lex_mode.quote == PM_HEREDOC_QUOTE_BACKTICK) { @@ -19071,13 +19071,13 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { receiver = UP(pm_parentheses_node_create(parser, &lparen, NULL, &parser->previous, 0)); } else { - arguments.opening_loc = TOK2SLICE(parser, &lparen); + arguments.opening_loc = TOK2LOC(parser, &lparen); receiver = parse_expression(parser, PM_BINDING_POWER_COMPOSITION, true, false, PM_ERR_NOT_EXPRESSION, (uint16_t) (depth + 1)); if (!parser->recovering) { accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN); - arguments.closing_loc = TOK2SLICE(parser, &parser->previous); + arguments.closing_loc = TOK2LOC(parser, &parser->previous); } } } else { @@ -20378,7 +20378,7 @@ parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { // Next, create the local variable target and add it to the list of // targets for the match. - pm_node_t *target = UP(pm_local_variable_target_node_create(parser, &TOK2SLICE(parser, &((pm_token_t) { .type = 0, .start = start, .end = end })), name, depth == -1 ? 0 : (uint32_t) depth)); + pm_node_t *target = UP(pm_local_variable_target_node_create(parser, &TOK2LOC(parser, &((pm_token_t) { .type = 0, .start = start, .end = end })), name, depth == -1 ? 0 : (uint32_t) depth)); pm_node_list_append(&callback_data->match->targets, target); } @@ -20429,7 +20429,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // is parsed because it could be referenced in the value. pm_call_node_t *call_node = (pm_call_node_t *) node; if (PM_NODE_FLAG_P(call_node, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { - pm_parser_local_add_slice(parser, &call_node->message_loc, 0); + pm_parser_local_add_location(parser, &call_node->message_loc, 0); } } PRISM_FALLTHROUGH @@ -20438,7 +20438,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // variable before parsing the value, in case the value // references the variable. if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) { - pm_parser_local_add_location(parser, parser->start + PM_NODE_START(node), parser->start + PM_NODE_END(node), 0); + pm_parser_local_add_raw(parser, parser->start + PM_NODE_START(node), parser->start + PM_NODE_END(node), 0); } parser_lex(parser); @@ -20564,7 +20564,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length); - pm_constant_id_t constant_id = pm_parser_local_add_slice(parser, &cast->message_loc, 1); + pm_constant_id_t constant_id = pm_parser_local_add_location(parser, &cast->message_loc, 1); parser_lex(parser); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ, (uint16_t) (depth + 1)); @@ -20696,7 +20696,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length); - pm_constant_id_t constant_id = pm_parser_local_add_slice(parser, &cast->message_loc, 1); + pm_constant_id_t constant_id = pm_parser_local_add_location(parser, &cast->message_loc, 1); parser_lex(parser); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ, (uint16_t) (depth + 1)); @@ -20839,7 +20839,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // will transform it into a local variable write. if (PM_NODE_FLAG_P(cast, PM_CALL_NODE_FLAGS_VARIABLE_CALL)) { pm_refute_numbered_parameter(parser, cast->message_loc.start, cast->message_loc.length); - pm_constant_id_t constant_id = pm_parser_local_add_slice(parser, &cast->message_loc, 1); + pm_constant_id_t constant_id = pm_parser_local_add_location(parser, &cast->message_loc, 1); pm_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, accepts_command_call, PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR, (uint16_t) (depth + 1)); pm_node_t *result = UP(pm_local_variable_operator_write_node_create(parser, UP(cast), &token, value, constant_id, 0)); @@ -21259,7 +21259,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t parser_lex(parser); pm_arguments_t arguments = { 0 }; - arguments.opening_loc = TOK2SLICE(parser, &parser->previous); + arguments.opening_loc = TOK2LOC(parser, &parser->previous); if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { pm_accepts_block_stack_push(parser, true); @@ -21268,7 +21268,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_EXPECT_RBRACKET); } - arguments.closing_loc = TOK2SLICE(parser, &parser->previous); + arguments.closing_loc = TOK2LOC(parser, &parser->previous); // If we have a comma after the closing bracket then this is a multiple // assignment and we should parse the targets. @@ -21676,7 +21676,7 @@ parse_program(pm_parser_t *parser) { // correct the location information. if (statements == NULL) { statements = pm_statements_node_create(parser); - statements->base.location = (pm_slice_t) { 0 }; + statements->base.location = (pm_location_t) { 0 }; } return UP(pm_program_node_create(parser, &locals, statements)); diff --git a/templates/ext/prism/api_node.c.erb b/templates/ext/prism/api_node.c.erb index 2960d3e410..e9c3742085 100644 --- a/templates/ext/prism/api_node.c.erb +++ b/templates/ext/prism/api_node.c.erb @@ -12,15 +12,10 @@ static VALUE rb_cPrism<%= node.name %>; <%- end -%> static VALUE -pm_slice_new(const uint32_t start, const uint32_t length, VALUE source, bool freeze) { +pm_location_new(const uint32_t start, const uint32_t length, VALUE source, bool freeze) { if (freeze) { - VALUE slice_argv[] = { - source, - LONG2FIX(start), - LONG2FIX(length) - }; - - return rb_obj_freeze(rb_class_new_instance(3, slice_argv, rb_cPrismLocation)); + VALUE location_argv[] = { source, LONG2FIX(start), LONG2FIX(length) }; + return rb_obj_freeze(rb_class_new_instance(3, location_argv, rb_cPrismLocation)); } else { uint64_t value = ((((uint64_t) start) << 32) | ((uint64_t) length)); return ULL2NUM(value); @@ -30,7 +25,7 @@ pm_slice_new(const uint32_t start, const uint32_t length, VALUE source, bool fre VALUE pm_token_new(const pm_parser_t *parser, const pm_token_t *token, rb_encoding *encoding, VALUE source, bool freeze) { ID type = rb_intern(pm_token_type_name(token->type)); - VALUE location = pm_slice_new((uint32_t) (token->start - parser->start), (uint32_t) (token->end - token->start), source, freeze); + VALUE location = pm_location_new((uint32_t) (token->start - parser->start), (uint32_t) (token->end - token->start), source, freeze); VALUE slice = rb_enc_str_new((const char *) token->start, token->end - token->start, encoding); if (freeze) rb_obj_freeze(slice); @@ -200,7 +195,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi argv[1] = ULONG2NUM(node->node_id); // location - argv[2] = pm_slice_new(node->location.start, node->location.length, source, freeze); + argv[2] = pm_location_new(node->location.start, node->location.length, source, freeze); // flags argv[3] = ULONG2NUM(node->flags); @@ -237,10 +232,10 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi if (freeze) rb_obj_freeze(argv[<%= index %>]); <%- when Prism::Template::LocationField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" - argv[<%= index %>] = pm_slice_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); + argv[<%= index %>] = pm_location_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::OptionalLocationField -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" - argv[<%= index %>] = cast-><%= field.name %>.length == 0 ? Qnil : pm_slice_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); + argv[<%= index %>] = cast-><%= field.name %>.length == 0 ? Qnil : pm_location_new(cast-><%= field.name %>.start, cast-><%= field.name %>.length, source, freeze); <%- when Prism::Template::UInt8Field -%> #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>" argv[<%= index %>] = UINT2NUM(cast-><%= field.name %>); diff --git a/templates/include/prism/ast.h.erb b/templates/include/prism/ast.h.erb index c96b664531..ded568f33c 100644 --- a/templates/include/prism/ast.h.erb +++ b/templates/include/prism/ast.h.erb @@ -47,19 +47,19 @@ typedef struct { /** * This struct represents a slice in the source code, defined by an offset and - * a length. Note that we have confirmation that we can represent all slices + * a length. Note that we have confirmation that we can represent all locations * within Ruby source files using 32-bit integers per: * * https://bugs.ruby-lang.org/issues/20488#note-1 * */ typedef struct { - /** The offset of the slice from the start of the source. */ + /** The offset of the location from the start of the source. */ uint32_t start; - /** The length of the slice. */ + /** The length of the location. */ uint32_t length; -} pm_slice_t; +} pm_location_t; struct pm_node; @@ -116,7 +116,7 @@ static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = 0x2; typedef struct pm_node { /** * This represents the type of the node. It somewhat maps to the nodes that - * existed in the original grammar and ripper, but it's not a 1:1 mapping. + * existed in the original grammar and ripper, but it is not a 1:1 mapping. */ pm_node_type_t type; @@ -133,10 +133,10 @@ typedef struct pm_node { uint32_t node_id; /** - * This is the location of the node in the source. It's a range of bytes + * This is the location of the node in the source. It is a range of bytes * containing a start and an end. */ - pm_slice_t location; + pm_location_t location; } pm_node_t; /** @@ -204,7 +204,7 @@ typedef struct pm_<%= node.human %> { when Prism::Template::ConstantField, Prism::Template::OptionalConstantField then "pm_constant_id_t #{field.name}" when Prism::Template::ConstantListField then "pm_constant_id_list_t #{field.name}" when Prism::Template::StringField then "pm_string_t #{field.name}" - when Prism::Template::LocationField, Prism::Template::OptionalLocationField then "pm_slice_t #{field.name}" + when Prism::Template::LocationField, Prism::Template::OptionalLocationField then "pm_location_t #{field.name}" when Prism::Template::UInt8Field then "uint8_t #{field.name}" when Prism::Template::UInt32Field then "uint32_t #{field.name}" when Prism::Template::IntegerField then "pm_integer_t #{field.name}" diff --git a/templates/include/prism/diagnostic.h.erb b/templates/include/prism/diagnostic.h.erb index ce4102639d..c1864e6021 100644 --- a/templates/include/prism/diagnostic.h.erb +++ b/templates/include/prism/diagnostic.h.erb @@ -40,7 +40,7 @@ typedef struct { pm_list_node_t node; /** The location of the diagnostic in the source. */ - pm_slice_t location; + pm_location_t location; /** The ID of the diagnostic. */ pm_diagnostic_id_t diag_id; diff --git a/templates/src/node.c.erb b/templates/src/node.c.erb index e642de6958..f1709a0249 100644 --- a/templates/src/node.c.erb +++ b/templates/src/node.c.erb @@ -226,8 +226,8 @@ pm_dump_json_constant(pm_buffer_t *buffer, const pm_parser_t *parser, pm_constan } static void -pm_dump_json_slice(pm_buffer_t *buffer, const pm_slice_t *slice) { - pm_buffer_append_format(buffer, "{\"start\":%" PRIu32 ",\"length\":%" PRIu32 "}", slice->start, slice->length); +pm_dump_json_location(pm_buffer_t *buffer, const pm_location_t *location) { + pm_buffer_append_format(buffer, "{\"start\":%" PRIu32 ",\"length\":%" PRIu32 "}", location->start, location->length); } /** @@ -241,7 +241,7 @@ pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *no pm_buffer_append_string(buffer, "{\"type\":\"<%= node.name %>\",\"location\":", <%= node.name.bytesize + 22 %>); const pm_<%= node.human %>_t *cast = (const pm_<%= node.human %>_t *) node; - pm_dump_json_slice(buffer, &cast->base.location); + pm_dump_json_location(buffer, &cast->base.location); <%- [*node.flags, *node.fields].each_with_index do |field, index| -%> // Dump the <%= field.name %> field @@ -288,10 +288,10 @@ pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *no } pm_buffer_append_byte(buffer, ']'); <%- when Prism::Template::LocationField -%> - pm_dump_json_slice(buffer, &cast-><%= field.name %>); + pm_dump_json_location(buffer, &cast-><%= field.name %>); <%- when Prism::Template::OptionalLocationField -%> if (cast-><%= field.name %>.length != 0) { - pm_dump_json_slice(buffer, &cast-><%= field.name %>); + pm_dump_json_location(buffer, &cast-><%= field.name %>); } else { pm_buffer_append_string(buffer, "null", 4); } diff --git a/templates/src/prettyprint.c.erb b/templates/src/prettyprint.c.erb index 67668ff063..74c0f6dbdf 100644 --- a/templates/src/prettyprint.c.erb +++ b/templates/src/prettyprint.c.erb @@ -11,9 +11,9 @@ void pm_prettyprint(void) {} #else static inline void -prettyprint_slice(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_slice_t *slice) { - pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, slice->start, parser->start_line); - pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, slice->start + slice->length, parser->start_line); +prettyprint_location(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_location_t *location) { + pm_line_column_t start = pm_newline_list_line_column(&parser->newline_list, location->start, parser->start_line); + pm_line_column_t end = pm_newline_list_line_column(&parser->newline_list, location->start + location->length, parser->start_line); pm_buffer_append_format(output_buffer, "(%" PRIi32 ",%" PRIu32 ")-(%" PRIi32 ",%" PRIu32 ")", start.line, start.column, end.line, end.column); } @@ -35,7 +35,7 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node; <%- end -%> pm_buffer_append_string(output_buffer, "@ <%= node.name %> (location: ", <%= node.name.length + 14 %>); - prettyprint_slice(output_buffer, parser, &node->location); + prettyprint_location(output_buffer, parser, &node->location); pm_buffer_append_string(output_buffer, ")\n", 2); <%- (fields = [*node.flags, *node.fields]).each_with_index do |field, index| -%> <%- preadd = index == fields.length - 1 ? " " : "| " -%> @@ -102,21 +102,21 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm } pm_buffer_append_string(output_buffer, "]\n", 2); <%- when Prism::Template::LocationField -%> - pm_slice_t *slice = &cast-><%= field.name %>; + pm_location_t *location = &cast-><%= field.name %>; pm_buffer_append_byte(output_buffer, ' '); - prettyprint_slice(output_buffer, parser, slice); + prettyprint_location(output_buffer, parser, location); pm_buffer_append_string(output_buffer, " = \"", 4); - pm_buffer_append_source(output_buffer, parser->start + slice->start, (size_t) slice->length, PM_BUFFER_ESCAPING_RUBY); + pm_buffer_append_source(output_buffer, parser->start + location->start, (size_t) location->length, PM_BUFFER_ESCAPING_RUBY); pm_buffer_append_string(output_buffer, "\"\n", 2); <%- when Prism::Template::OptionalLocationField -%> - pm_slice_t *slice = &cast-><%= field.name %>; - if (slice->length == 0) { + pm_location_t *location = &cast-><%= field.name %>; + if (location->length == 0) { pm_buffer_append_string(output_buffer, " nil\n", 5); } else { pm_buffer_append_byte(output_buffer, ' '); - prettyprint_slice(output_buffer, parser, slice); + prettyprint_location(output_buffer, parser, location); pm_buffer_append_string(output_buffer, " = \"", 4); - pm_buffer_append_source(output_buffer, parser->start + slice->start, (size_t) slice->length, PM_BUFFER_ESCAPING_RUBY); + pm_buffer_append_source(output_buffer, parser->start + location->start, (size_t) location->length, PM_BUFFER_ESCAPING_RUBY); pm_buffer_append_string(output_buffer, "\"\n", 2); } <%- when Prism::Template::UInt8Field -%> diff --git a/templates/src/serialize.c.erb b/templates/src/serialize.c.erb index 3dd28fc83a..10e73dfbea 100644 --- a/templates/src/serialize.c.erb +++ b/templates/src/serialize.c.erb @@ -20,9 +20,9 @@ pm_sizet_to_u32(size_t value) { } static void -pm_serialize_slice(const pm_slice_t *slice, pm_buffer_t *buffer) { - pm_buffer_append_varuint(buffer, slice->start); - pm_buffer_append_varuint(buffer, slice->length); +pm_serialize_location(const pm_location_t *location, pm_buffer_t *buffer) { + pm_buffer_append_varuint(buffer, location->start); + pm_buffer_append_varuint(buffer, location->length); } static void @@ -73,7 +73,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { <%- if Prism::Template::INCLUDE_NODE_ID -%> pm_buffer_append_varuint(buffer, node->node_id); <%- end -%> - pm_serialize_slice(&node->location, buffer); + pm_serialize_location(&node->location, buffer); switch (PM_NODE_TYPE(node)) { // We do not need to serialize a ScopeNode ever as @@ -119,7 +119,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { } <%- when Prism::Template::LocationField -%> <%- if field.should_be_serialized? -%> - pm_serialize_slice(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + pm_serialize_location(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); <%- end -%> <%- when Prism::Template::OptionalLocationField -%> <%- if field.should_be_serialized? -%> @@ -127,7 +127,7 @@ pm_serialize_node(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { pm_buffer_append_byte(buffer, 0); } else { pm_buffer_append_byte(buffer, 1); - pm_serialize_slice(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + pm_serialize_location(&((pm_<%= node.human %>_t *)node)-><%= field.name %>, buffer); } <%- end -%> <%- when Prism::Template::UInt8Field -%> @@ -170,7 +170,7 @@ pm_serialize_comment(pm_comment_t *comment, pm_buffer_t *buffer) { pm_buffer_append_byte(buffer, (uint8_t) comment->type); // serialize location - pm_serialize_slice(&comment->location, buffer); + pm_serialize_location(&comment->location, buffer); } /** @@ -213,7 +213,7 @@ pm_serialize_data_loc(const pm_parser_t *parser, pm_buffer_t *buffer) { pm_buffer_append_byte(buffer, 0); } else { pm_buffer_append_byte(buffer, 1); - pm_serialize_slice(&parser->data_loc, buffer); + pm_serialize_location(&parser->data_loc, buffer); } } @@ -228,7 +228,7 @@ pm_serialize_diagnostic(pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) { pm_buffer_append_string(buffer, diagnostic->message, message_length); // serialize location - pm_serialize_slice(&diagnostic->location, buffer); + pm_serialize_location(&diagnostic->location, buffer); pm_buffer_append_byte(buffer, diagnostic->level); } From 1714fdc27207089b3477bec796c246b1a63fad72 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 3 Dec 2025 11:33:25 -0500 Subject: [PATCH 22/22] More location cleanup --- include/prism/defines.h | 33 ++++++++++++++++++ rust/ruby-prism/build.rs | 4 +-- rust/ruby-prism/src/lib.rs | 4 +-- src/prism.c | 58 +++++++++++++++---------------- src/util/pm_char.c | 2 +- templates/include/prism/ast.h.erb | 12 +++++-- 6 files changed, 77 insertions(+), 36 deletions(-) diff --git a/include/prism/defines.h b/include/prism/defines.h index e31429c789..c41e6031a3 100644 --- a/include/prism/defines.h +++ b/include/prism/defines.h @@ -257,4 +257,37 @@ #define PRISM_FALLTHROUGH #endif +/** + * We need to align nodes in the AST to a pointer boundary so that it can be + * safely cast to different node types. Use PRISM_ALIGNAS/PRISM_ALIGNOF to + * specify alignment in a compiler-agnostic way. + */ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L /* C11 or later */ + #include + + /** Specify alignment for a type or variable. */ + #define PRISM_ALIGNAS(size) alignas(size) + + /** Get the alignment requirement of a type. */ + #define PRISM_ALIGNOF(type) alignof(type) +#elif defined(__GNUC__) || defined(__clang__) + /** Specify alignment for a type or variable. */ + #define PRISM_ALIGNAS(size) __attribute__((aligned(size))) + + /** Get the alignment requirement of a type. */ + #define PRISM_ALIGNOF(type) __alignof__(type) +#elif defined(_MSC_VER) + /** Specify alignment for a type or variable. */ + #define PRISM_ALIGNAS(size) __declspec(align(size)) + + /** Get the alignment requirement of a type. */ + #define PRISM_ALIGNOF(type) __alignof(type) +#else + /** Void because this platform does not support specifying alignment. */ + #define PRISM_ALIGNAS(size) + + /** Fallback to sizeof as alignment requirement of a type. */ + #define PRISM_ALIGNOF(type) sizeof(type) +#endif + #endif diff --git a/rust/ruby-prism/build.rs b/rust/ruby-prism/build.rs index 042a123dd4..8faad957ab 100644 --- a/rust/ruby-prism/build.rs +++ b/rust/ruby-prism/build.rs @@ -605,9 +605,9 @@ impl<'pr> Node<'pr> {{ /// Panics if the node type cannot be read. /// #[allow(clippy::not_unsafe_ptr_arg_deref)] + #[allow(clippy::cast_ptr_alignment)] pub(crate) fn new(parser: NonNull, node: *mut pm_node_t) -> Self {{ - match unsafe {{ (*node).type_ }} {{ -" + match unsafe {{ (*node).type_ }} {{" )?; for node in &config.nodes { diff --git a/rust/ruby-prism/src/lib.rs b/rust/ruby-prism/src/lib.rs index f54437abca..46499a63e3 100644 --- a/rust/ruby-prism/src/lib.rs +++ b/rust/ruby-prism/src/lib.rs @@ -19,7 +19,7 @@ use std::mem::MaybeUninit; use std::ptr::NonNull; pub use self::bindings::*; -use ruby_prism_sys::{pm_comment_t, pm_comment_type_t, pm_constant_id_list_t, pm_constant_id_t, pm_diagnostic_t, pm_integer_t, pm_magic_comment_t, pm_node_destroy, pm_node_list, pm_node_t, pm_parse, pm_parser_free, pm_parser_init, pm_parser_t, pm_location_t}; +use ruby_prism_sys::{pm_comment_t, pm_comment_type_t, pm_constant_id_list_t, pm_constant_id_t, pm_diagnostic_t, pm_integer_t, pm_location_t, pm_magic_comment_t, pm_node_destroy, pm_node_list, pm_node_t, pm_parse, pm_parser_free, pm_parser_init, pm_parser_t}; /// A range in the source file, represented as a start offset and length. pub struct Location<'pr> { @@ -52,7 +52,7 @@ impl<'pr> Location<'pr> { /// Returns the end offset from the beginning of the parsed source. #[must_use] - pub fn end(&self) -> u32 { + pub const fn end(&self) -> u32 { self.start + self.length } diff --git a/src/prism.c b/src/prism.c index 15f462ffda..9bfac79ca7 100644 --- a/src/prism.c +++ b/src/prism.c @@ -1016,7 +1016,7 @@ pm_locals_order(PRISM_ATTRIBUTE_UNUSED pm_parser_t *parser, pm_locals_t *locals, * Retrieve the constant pool id for the given location. */ static inline pm_constant_id_t -pm_parser_constant_id_location(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { +pm_parser_constant_id_raw(pm_parser_t *parser, const uint8_t *start, const uint8_t *end) { return pm_constant_pool_insert_shared(&parser->constant_pool, start, (size_t) (end - start)); } @@ -1041,7 +1041,7 @@ pm_parser_constant_id_constant(pm_parser_t *parser, const char *start, size_t le */ static inline pm_constant_id_t pm_parser_constant_id_token(pm_parser_t *parser, const pm_token_t *token) { - return pm_parser_constant_id_location(parser, token->start, token->end); + return pm_parser_constant_id_raw(parser, token->start, token->end); } /** @@ -2683,7 +2683,7 @@ pm_call_node_call_create(pm_parser_t *parser, pm_node_t *receiver, pm_token_t *o * If the final character is `@` as is the case for `foo.~@`, * we should ignore the @ in the same way we do for symbols. */ - node->name = pm_parser_constant_id_location(parser, message->start, parse_operator_symbol_name(message)); + node->name = pm_parser_constant_id_raw(parser, message->start, parse_operator_symbol_name(message)); return node; } @@ -2967,7 +2967,7 @@ pm_call_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .message_loc = target->message_loc, .read_name = 0, .write_name = target->name, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1), .binary_operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3000,7 +3000,7 @@ pm_index_operator_write_node_create(pm_parser_t *parser, pm_call_node_t *target, .arguments = target->arguments, .closing_loc = target->closing_loc, .block = (pm_block_argument_node_t *) target->block, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1), .binary_operator_loc = TOK2LOC(parser, operator), .value = value }; @@ -3293,7 +3293,7 @@ pm_class_variable_operator_write_node_create(pm_parser_t *parser, pm_class_varia .name_loc = target->base.location, .binary_operator_loc = TOK2LOC(parser, operator), .value = value, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1) }; return node; @@ -3397,7 +3397,7 @@ pm_constant_path_operator_write_node_create(pm_parser_t *parser, pm_constant_pat .target = target, .binary_operator_loc = TOK2LOC(parser, operator), .value = value, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1) }; return node; @@ -3499,7 +3499,7 @@ pm_constant_operator_write_node_create(pm_parser_t *parser, pm_constant_read_nod .name_loc = target->base.location, .binary_operator_loc = TOK2LOC(parser, operator), .value = value, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1) }; return node; @@ -4145,7 +4145,7 @@ pm_global_variable_write_name(pm_parser_t *parser, const pm_node_t *target) { case PM_NUMBERED_REFERENCE_READ_NODE: // This will only ever happen in the event of a syntax error, but we // still need to provide something for the node. - return pm_parser_constant_id_location(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target)); + return pm_parser_constant_id_raw(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target)); default: assert(false && "unreachable"); return (pm_constant_id_t) -1; @@ -4184,7 +4184,7 @@ pm_global_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *ta .name_loc = target->location, .binary_operator_loc = TOK2LOC(parser, operator), .value = value, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1) }; return node; @@ -4617,7 +4617,7 @@ pm_instance_variable_operator_write_node_create(pm_parser_t *parser, pm_instance .name_loc = target->base.location, .binary_operator_loc = TOK2LOC(parser, operator), .value = value, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1) + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1) }; return node; @@ -5064,7 +5064,7 @@ pm_required_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t *node = (pm_required_keyword_parameter_node_t) { .base = PM_NODE_INIT_TOKEN(parser, PM_REQUIRED_KEYWORD_PARAMETER_NODE, 0, name), - .name = pm_parser_constant_id_location(parser, name->start, name->end - 1), + .name = pm_parser_constant_id_raw(parser, name->start, name->end - 1), .name_loc = TOK2LOC(parser, name), }; @@ -5080,7 +5080,7 @@ pm_optional_keyword_parameter_node_create(pm_parser_t *parser, const pm_token_t *node = (pm_optional_keyword_parameter_node_t) { .base = PM_NODE_INIT_TOKEN_NODE(parser, PM_OPTIONAL_KEYWORD_PARAMETER_NODE, 0, name, value), - .name = pm_parser_constant_id_location(parser, name->start, name->end - 1), + .name = pm_parser_constant_id_raw(parser, name->start, name->end - 1), .name_loc = TOK2LOC(parser, name), .value = value }; @@ -5171,7 +5171,7 @@ pm_local_variable_operator_write_node_create(pm_parser_t *parser, pm_node_t *tar .binary_operator_loc = TOK2LOC(parser, operator), .value = value, .name = name, - .binary_operator = pm_parser_constant_id_location(parser, operator->start, operator->end - 1), + .binary_operator = pm_parser_constant_id_raw(parser, operator->start, operator->end - 1), .depth = depth }; @@ -6652,20 +6652,20 @@ pm_symbol_node_synthesized_create(pm_parser_t *parser, const char *content) { */ static bool pm_symbol_node_label_p(const pm_parser_t *parser, const pm_node_t *node) { - const uint8_t *end = NULL; + const pm_location_t *location = NULL; switch (PM_NODE_TYPE(node)) { case PM_SYMBOL_NODE: { const pm_symbol_node_t *cast = (pm_symbol_node_t *) node; - if (cast->closing_loc.length != 0) { - end = parser->start + cast->closing_loc.start + cast->closing_loc.length; + if (cast->closing_loc.length > 0) { + location = &cast->closing_loc; } break; } case PM_INTERPOLATED_SYMBOL_NODE: { const pm_interpolated_symbol_node_t *cast = (pm_interpolated_symbol_node_t *) node; - if (cast->closing_loc.length != 0) { - end = parser->start + cast->closing_loc.start + cast->closing_loc.length; + if (cast->closing_loc.length > 0) { + location = &cast->closing_loc; } break; } @@ -6673,7 +6673,7 @@ pm_symbol_node_label_p(const pm_parser_t *parser, const pm_node_t *node) { return false; } - return (end != NULL) && (end[-1] == ':'); + return (location != NULL) && (parser->start[PM_LOCATION_END(location) - 1] == ':'); } /** @@ -7121,7 +7121,7 @@ pm_parser_local_add(pm_parser_t *parser, pm_constant_id_t constant_id, const uin */ static pm_constant_id_t pm_parser_local_add_raw(pm_parser_t *parser, const uint8_t *start, const uint8_t *end, uint32_t reads) { - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, start, end); + pm_constant_id_t constant_id = pm_parser_constant_id_raw(parser, start, end); if (constant_id != 0) pm_parser_local_add(parser, constant_id, start, end, reads); return constant_id; } @@ -7129,7 +7129,7 @@ pm_parser_local_add_raw(pm_parser_t *parser, const uint8_t *start, const uint8_t /** * Add a local variable from a location to the current scope. */ -static pm_constant_id_t +static inline pm_constant_id_t pm_parser_local_add_location(pm_parser_t *parser, pm_location_t *location, uint32_t reads) { return pm_parser_local_add_raw(parser, parser->start + location->start, parser->start + location->start + location->length, reads); } @@ -12633,7 +12633,7 @@ parse_unwriteable_target(pm_parser_t *parser, pm_node_t *target) { default: break; } - pm_constant_id_t name = pm_parser_constant_id_location(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target)); + pm_constant_id_t name = pm_parser_constant_id_raw(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target)); pm_local_variable_target_node_t *result = pm_local_variable_target_node_create(parser, &target->location, name, 0); pm_node_destroy(parser, target); @@ -12953,7 +12953,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod pm_parser_local_add_location(parser, &message_loc, 0); pm_node_destroy(parser, target); - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, parser->start + PM_LOCATION_START(&message_loc), parser->start + PM_LOCATION_END(&message_loc)); + pm_constant_id_t constant_id = pm_parser_constant_id_raw(parser, parser->start + PM_LOCATION_START(&message_loc), parser->start + PM_LOCATION_END(&message_loc)); target = UP(pm_local_variable_write_node_create(parser, constant_id, 0, value, &message_loc, operator)); return target; @@ -13048,7 +13048,7 @@ parse_unwriteable_write(pm_parser_t *parser, pm_node_t *target, const pm_token_t default: break; } - pm_constant_id_t name = pm_parser_local_add_raw(parser, parser->start + PM_NODE_START(target), parser->start + PM_NODE_END(target), 1); + pm_constant_id_t name = pm_parser_local_add_location(parser, &target->location, 1); pm_local_variable_write_node_t *result = pm_local_variable_write_node_create(parser, name, 0, value, &target->location, equals); pm_node_destroy(parser, target); @@ -16321,7 +16321,7 @@ parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *ca const uint8_t *start = parser->start + PM_LOCATION_START(value_loc); const uint8_t *end = parser->start + PM_LOCATION_END(value_loc); - pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, start, end); + pm_constant_id_t constant_id = pm_parser_constant_id_raw(parser, start, end); int depth = -1; if (pm_slice_is_valid_local(parser, start, end)) { @@ -18859,7 +18859,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b * methods to override the unary operators, we should ignore * the @ in the same way we do for symbols. */ - pm_constant_id_t name_id = pm_parser_constant_id_location(parser, name.start, parse_operator_symbol_name(&name)); + pm_constant_id_t name_id = pm_parser_constant_id_raw(parser, name.start, parse_operator_symbol_name(&name)); flush_block_exits(parser, previous_block_exits); pm_node_list_free(¤t_block_exits); @@ -20337,7 +20337,7 @@ parse_regular_expression_named_capture(const pm_string_t *capture, void *data) { // copy the names directly. The pointers will line up. start = source; end = source + length; - name = pm_parser_constant_id_location(parser, start, end); + name = pm_parser_constant_id_raw(parser, start, end); } else { // Otherwise, the name is a slice of the malloc-ed owned string, // in which case we need to copy it out into a new string. @@ -20438,7 +20438,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t // variable before parsing the value, in case the value // references the variable. if (PM_NODE_TYPE_P(node, PM_IT_LOCAL_VARIABLE_READ_NODE)) { - pm_parser_local_add_raw(parser, parser->start + PM_NODE_START(node), parser->start + PM_NODE_END(node), 0); + pm_parser_local_add_location(parser, &node->location, 0); } parser_lex(parser); diff --git a/src/util/pm_char.c b/src/util/pm_char.c index 05fadf9353..748582b7fe 100644 --- a/src/util/pm_char.c +++ b/src/util/pm_char.c @@ -84,7 +84,7 @@ pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length) { */ size_t pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list, uint32_t start_offset) { - if (length <= 0 || length > UINT32_MAX) return 0; + if (length <= 0) return 0; uint32_t size = 0; uint32_t maximum = (uint32_t) length; diff --git a/templates/include/prism/ast.h.erb b/templates/include/prism/ast.h.erb index ded568f33c..9115f20eaa 100644 --- a/templates/include/prism/ast.h.erb +++ b/templates/include/prism/ast.h.erb @@ -164,6 +164,15 @@ typedef struct pm_node { * Return true if the given flag is set on the given node. */ #define PM_NODE_FLAG_P(node_, flag_) ((PM_NODE_FLAGS(node_) & (flag_)) != 0) + +/** + * The alignment required for a child node within a parent node. + */ +#ifdef _MSC_VER +#define PM_NODE_ALIGNAS __declspec(align(8)) +#else +#define PM_NODE_ALIGNAS PRISM_ALIGNAS(PRISM_ALIGNOF(void *)) +#endif <%- nodes.each do |node| -%> /** @@ -186,7 +195,6 @@ typedef struct pm_node { typedef struct pm_<%= node.human %> { /** The embedded base node. */ pm_node_t base; - <%- node.fields.each do |field| -%> /** @@ -199,7 +207,7 @@ typedef struct pm_<%= node.human %> { <%- end -%> */ <%= case field - when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "struct #{field.c_type} *#{field.name}" + when Prism::Template::NodeField, Prism::Template::OptionalNodeField then "PM_NODE_ALIGNAS struct #{field.c_type} *#{field.name}" when Prism::Template::NodeListField then "struct pm_node_list #{field.name}" when Prism::Template::ConstantField, Prism::Template::OptionalConstantField then "pm_constant_id_t #{field.name}" when Prism::Template::ConstantListField then "pm_constant_id_list_t #{field.name}"