Update codegen/grammar

Author: aibaars

Cargo.lock

@@ -6,6 +6,8 @@ edition = "2021"
 # When updating these dependencies, run `rust/update_cargo_deps.sh`
 [dependencies]
 ungrammar = "1.16.1"
-proc-macro2 = "1.0.33"
-quote = "1.0.12"
-itertools = "0.10.1"
+proc-macro2 = "1.0.47"
+quote = "1.0.20"
+either = "1.9.0"
+ra_ap_stdx = "0.0.248"
+itertools = "0.12.0"

rust/ast-generator/Cargo.toml

@@ -11,9 +11,11 @@ use std::{
     fs,
 };
 
+use either::Either;
 use itertools::Itertools;
 use proc_macro2::{Punct, Spacing};
 use quote::{format_ident, quote};
+use ra_ap_stdx::panic_context;
 use ungrammar::{Grammar, Rule};
 
 use crate::{
@@ -495,6 +497,7 @@ fn generate_syntax_kinds(grammar: KindsSrc) -> String {
         .map(|name| format_ident!("{}", name))
         .collect::<Vec<_>>();
 
+    // FIXME: This generates enum kinds?
     let nodes = grammar
         .nodes
         .iter()
@@ -757,6 +760,7 @@ pub(crate) fn lower(grammar: &Grammar) -> AstSrc {
     for &node in &nodes {
         let name = grammar[node].name.clone();
         let rule = &grammar[node].rule;
+        let _g = panic_context::enter(name.clone());
         match lower_enum(grammar, rule) {
             Some(variants) => {
                 let enum_src = AstEnumSrc {
@@ -904,11 +908,16 @@ fn lower_separated_list(
         Rule::Seq(it) => it,
         _ => return false,
     };
-    let (node, repeat, trailing_sep) = match rule.as_slice() {
+
+    let (nt, repeat, trailing_sep) = match rule.as_slice() {
         [Rule::Node(node), Rule::Rep(repeat), Rule::Opt(trailing_sep)] => {
-            (node, repeat, Some(trailing_sep))
+            (Either::Left(node), repeat, Some(trailing_sep))
+        }
+        [Rule::Node(node), Rule::Rep(repeat)] => (Either::Left(node), repeat, None),
+        [Rule::Token(token), Rule::Rep(repeat), Rule::Opt(trailing_sep)] => {
+            (Either::Right(token), repeat, Some(trailing_sep))
         }
-        [Rule::Node(node), Rule::Rep(repeat)] => (node, repeat, None),
+        [Rule::Token(token), Rule::Rep(repeat)] => (Either::Right(token), repeat, None),
         _ => return false,
     };
     let repeat = match &**repeat {
@@ -917,21 +926,34 @@ fn lower_separated_list(
     };
     if !matches!(
         repeat.as_slice(),
-        [comma, Rule::Node(n)]
-            if trailing_sep.map_or(true, |it| comma == &**it) && n == node
+        [comma, nt_]
+            if trailing_sep.map_or(true, |it| comma == &**it) && match (nt, nt_) {
+                (Either::Left(node), Rule::Node(nt_)) => node == nt_,
+                (Either::Right(token), Rule::Token(nt_)) => token == nt_,
+                _ => false,
+            }
     ) {
         return false;
     }
-    let ty = grammar[*node].name.clone();
-    let name = label
-        .cloned()
-        .unwrap_or_else(|| pluralize(&to_lower_snake_case(&ty)));
-    let field = Field::Node {
-        name,
-        ty,
-        cardinality: Cardinality::Many,
-    };
-    acc.push(field);
+    match nt {
+        Either::Right(token) => {
+            let name = clean_token_name(&grammar[*token].name);
+            let field = Field::Token(name);
+            acc.push(field);
+        }
+        Either::Left(node) => {
+            let ty = grammar[*node].name.clone();
+            let name = label
+                .cloned()
+                .unwrap_or_else(|| pluralize(&to_lower_snake_case(&ty)));
+            let field = Field::Node {
+                name,
+                ty,
+                cardinality: Cardinality::Many,
+            };
+            acc.push(field);
+        }
+    }
     true
 }
 

rust/ast-generator/src/codegen/grammar.rs

@@ -118,9 +118,34 @@ const CONTEXTUAL_KEYWORDS: &[&str] = &[
     "dyn",
     "auto",
     "yeet",
+    "safe",
 ];
 // keywords we use for special macro expansions
-const CONTEXTUAL_BUILTIN_KEYWORDS: &[&str] = &["builtin", "offset_of", "format_args", "asm"];
+const CONTEXTUAL_BUILTIN_KEYWORDS: &[&str] = &[
+    "asm",
+    "att_syntax",
+    "builtin",
+    "clobber_abi",
+    "format_args",
+    // "in",
+    "inlateout",
+    "inout",
+    "label",
+    "lateout",
+    "may_unwind",
+    "nomem",
+    "noreturn",
+    "nostack",
+    "offset_of",
+    "options",
+    "out",
+    "preserves_flags",
+    "pure",
+    // "raw",
+    "readonly",
+    "sym",
+];
+
 // keywords that are keywords depending on the edition
 const EDITION_DEPENDENT_KEYWORDS: &[(&str, Edition)] = &[
     ("try", Edition::Edition2018),