Lexer performance improvement

Author: docd27

.gitignore

@@ -2,6 +2,7 @@ node_modules/
 .npm
 .eslintcache
 .antlr
+.vscode
 
 bin/
 build/

.npmignore

@@ -5,6 +5,7 @@ node_modules/
 .github/
 .gitignore
 .npmignore
+.vscode
 
 # Build
 bin/
@@ -38,3 +39,4 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
+*.cpuprofile

src/lib/parse.js

@@ -1,4 +1,3 @@
-
 /**
  * @internal
  * Very basic GLSL parsing handling #version and #extension preprocessor directives
@@ -8,25 +7,6 @@ export function* simpleParse(input) {
   yield* parser(lexer(input));
 }
 
-/**
- * @internal
- * @param {string} input
- * @return {Generator<[string,string],void,void>}
- */
-function* iterateStringLookahead(input) {
-  const inputGenerator = input[Symbol.iterator]();
-  let cur = inputGenerator.next();
-  let next = inputGenerator.next();
-  while (!cur.done && !next.done) {
-    yield [cur.value, next.value];
-    cur = next;
-    next = inputGenerator.next();
-  }
-  if (!cur.done) {
-    yield [cur.value, undefined];
-  }
-}
-
 /**
  * @typedef {1|2|3|4|5|6|7|8} TokenID Token ID
  * @return {TokenID}
@@ -107,7 +87,13 @@ function* lexer(input) {
     line++; col = 0;
   };
 
-  for (const [cur, next] of iterateStringLookahead(input)) {
+  let next = input[0];
+  let cur;
+  /* Even though GLSL uses UTF-8 encoding, the actual syntactic charset is ASCII
+     So we can use JS indexed iteration (UTF-16) for a speedup compared to full Unicode iteration (for of) */
+  for (let i = 1; i <= input.length; i++) {
+    cur = next;
+    next = i < input.length ? input[i] : undefined;
     col++;
     if (skipOne) {
       skipOne = false;
@@ -325,7 +311,6 @@ const printToken = (t) => {
 };
 
 export const test = {
-  iterateStringLookahead,
   lexer,
   parser,
   printToken,

test/tests/parser.js

@@ -18,13 +18,6 @@ function unMockConsoleError() {
 
 export function parserTests() {
   describe('Parser', function() {
-    describe('#iterateStringLookahead()', function() {
-      it('should correctly yield lookahead for a string', function() {
-        const input = 'test';
-        const result = [...parse.test.iterateStringLookahead(input)];
-        assert.deepStrictEqual(result, [['t', 'e'], ['e', 's'], ['s', 't'], ['t', undefined]]);
-      });
-    });
     describe('#lexer()', function() {
       const lexerTest = (input) => [...parse.test.lexer(input)].map(parse.test.printToken);
       it('should lex empty input', function() {