additional letters token rule included, to expand range of acceptable identifiers

Author: wumpz

change.log

@@ -161,4 +161,7 @@ Added connect visitor
 TableNamesFinder included in source 
 Added proper support for sets (union, intersect) 
 Added support for extract(year from datetime-expr)
-Start implementation of analytical expressions
+Start implementation of analytical expressions
+
+0.8.1-SNAPSHOT
+Expansionpoint for additional letters included

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>net.sf.jsqlparser</groupId>
 	<artifactId>jsqlparser</artifactId>
-	<version>0.8.0-SNAPSHOT</version>
+	<version>0.8.1-SNAPSHOT</version>
 	<name>clone of the jsqlparser library</name>
 	<dependencies>
 		<dependency>

src/main/javacc/net/sf/jsqlparser/parser/JSqlParserCC.jj

@@ -26,6 +26,7 @@ options{
 //  DEBUG_LOOKAHEAD= true ;
 //  FORCE_LA_CHECK=true;
 //  DEBUG_TOKEN_MANAGER=true;
+//  UNICODE_INPUT=true;
 }
 
 PARSER_BEGIN(CCJSqlParser)
@@ -253,11 +254,21 @@ SPECIAL_TOKEN:
 
 TOKEN:
 {
-	< S_IDENTIFIER: ( <LETTER> )+ ( <DIGIT> | <LETTER> |<SPECIAL_CHARS>)* >
+	< S_IDENTIFIER: ( <LETTER> | <ADDITIONAL_LETTERS> )+ ( <DIGIT> | <LETTER> | <ADDITIONAL_LETTERS> | <SPECIAL_CHARS>)* >
 | 	< #LETTER: ["a"-"z", "A"-"Z", "_"] >
 |   < #SPECIAL_CHARS: "$" | "_">
 |   < S_CHAR_LITERAL: "'" (~["'"])* "'" ("'" (~["'"])* "'")*>
 |   < S_QUOTED_IDENTIFIER: "\"" (~["\n","\r","\""])* "\"" | ("`" (~["\n","\r","`"])* "`") | ("[" (~["\n","\r","]"])* "]") >
+
+/*
+To deal with database names (columns, tables) using not only latin base characters, one
+can expand the following rule to accept additional letters. Here is the
+addition of german umlauts.
+
+There seems to be no way to recognize letters by an external function to allow
+a configurable addition. One must rebuild JSqlParser with this new "Letterset".
+*/
+|   < #ADDITIONAL_LETTERS: ["�","�","�","�","�","�","�"] >
 }
 
 

src/test/java/net/sf/jsqlparser/test/select/SelectTest.java

@@ -737,6 +737,17 @@ public void testProblemFunction2() throws JSQLParserException {
 		String stmt = "SELECT sysdate FROM testtable";
 		assertSqlCanBeParsedAndDeparsed(stmt);
 	}
+	
+	public void testAdditionalLettersGerman() throws JSQLParserException {
+		String stmt = "SELECT colä, colü, colö FROM testtableöäü";
+		assertSqlCanBeParsedAndDeparsed(stmt);
+		
+		stmt = "SELECT colÄ, colÜ, colÖ FROM testtableÖÜÄ";
+		assertSqlCanBeParsedAndDeparsed(stmt);
+		
+		stmt = "SELECT Äcol FROM testtableÖÜÄ";
+		assertSqlCanBeParsedAndDeparsed(stmt);
+	}
 
 	private void assertSqlCanBeParsedAndDeparsed(String statement) throws JSQLParserException {
 		Statement parsed = parserManager.parse(new StringReader(statement));