diff --git a/Cargo.toml b/Cargo.toml index a6aab6f..0228595 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "eventql-parser" -version = "0.1.13" +version = "0.1.14" authors = ["Yorick Laupa "] description = "EventQL Lexer and Parser" homepage = "https://github.com/YoEight/eventql-parser" diff --git a/src/lexer.rs b/src/lexer.rs index 51ed282..2b4ece7 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -11,14 +11,14 @@ use crate::error::LexerError; use crate::token::{Operator, Sym, Symbol, Text, Token}; use nom::branch::alt; use nom::bytes::complete::{tag, take_while}; -use nom::character::complete::{alpha1, alphanumeric0, char, multispace1}; +use nom::character::complete::{alpha1, char, multispace1, satisfy}; use nom::character::one_of; use nom::combinator::{eof, opt, recognize}; use nom::error::{Error, context}; use nom::multi::many0; use nom::number::complete::double; use nom::sequence::{delimited, pair}; -use nom::{IResult, Parser}; +use nom::{AsChar, IResult, Parser}; /// Tokenize an EventQL query string. /// @@ -165,31 +165,34 @@ fn operator_2(input: Text) -> IResult { } fn ident(input: Text) -> IResult { - recognize(pair(alpha1, alphanumeric0)) - .map(|value: Text| { - let sym = if value.fragment().eq_ignore_ascii_case("and") { - Sym::Operator(Operator::And) - } else if value.fragment().eq_ignore_ascii_case("or") { - Sym::Operator(Operator::Or) - } else if value.fragment().eq_ignore_ascii_case("xor") { - Sym::Operator(Operator::Xor) - } else if value.fragment().eq_ignore_ascii_case("not") { - Sym::Operator(Operator::Not) - } else if value.fragment().eq_ignore_ascii_case("contains") { - Sym::Operator(Operator::Contains) - } else if value.fragment().eq_ignore_ascii_case("as") { - Sym::Operator(Operator::As) - } else { - Sym::Id(value.fragment()) - }; - - Token { - sym, - line: value.location_line(), - col: value.get_column() as u32, - } - }) - .parse(input) + recognize(pair( + alpha1, + many0(satisfy(|c| AsChar::is_alphanum(c) || c == '_')), + )) + .map(|value: Text| { + let sym = if value.fragment().eq_ignore_ascii_case("and") { + Sym::Operator(Operator::And) + } else if value.fragment().eq_ignore_ascii_case("or") { + Sym::Operator(Operator::Or) + } else if value.fragment().eq_ignore_ascii_case("xor") { + Sym::Operator(Operator::Xor) + } else if value.fragment().eq_ignore_ascii_case("not") { + Sym::Operator(Operator::Not) + } else if value.fragment().eq_ignore_ascii_case("contains") { + Sym::Operator(Operator::Contains) + } else if value.fragment().eq_ignore_ascii_case("as") { + Sym::Operator(Operator::As) + } else { + Sym::Id(value.fragment()) + }; + + Token { + sym, + line: value.location_line(), + col: value.get_column() as u32, + } + }) + .parse(input) } fn number(input: Text) -> IResult { diff --git a/src/tests/lexer.rs b/src/tests/lexer.rs index 2a6ca58..68e4d5f 100644 --- a/src/tests/lexer.rs +++ b/src/tests/lexer.rs @@ -17,3 +17,9 @@ fn test_lexer_comment() { let session = Session::builder().build(); insta::assert_yaml_snapshot!(session.tokenize("// useless comment\n ")); } + +#[test] +fn test_lexer_id_with_underscore() { + let session = Session::builder().build(); + insta::assert_yaml_snapshot!(session.tokenize("event_type")); +} diff --git a/src/tests/snapshots/eventql_parser__tests__lexer__lexer_id_with_underscore.snap b/src/tests/snapshots/eventql_parser__tests__lexer__lexer_id_with_underscore.snap new file mode 100644 index 0000000..1933ca6 --- /dev/null +++ b/src/tests/snapshots/eventql_parser__tests__lexer__lexer_id_with_underscore.snap @@ -0,0 +1,12 @@ +--- +source: src/tests/lexer.rs +expression: "session.tokenize(\"event_type\")" +--- +Ok: + - sym: + Id: event_type + line: 1 + col: 1 + - sym: Eof + line: 1 + col: 11