JavaScript: Show character code when reporting unexpected character.

Author: Max Schaefer

javascript/extractor/src/com/semmle/jcorn/Parser.java

@@ -721,7 +721,8 @@ protected Token getTokenFromCode(int code) {
         return this.finishOp(TokenType.prefix, 1);
     }
 
-    this.raise(this.pos, "Unexpected character '" + codePointToString(code) + "'");
+    String msg = String.format("Unexpected character '%s' (U+%04X)", codePointToString(code), code);
+    this.raise(this.pos, msg);
     return null;
   }
 

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -163,9 +163,9 @@
  * following environment variables are available:
  *
  * <ul>
- *   <li><code>LGTM_THREADS</code> determines how many threads are used for parallel
- *       extraction of JavaScript files (TypeScript files cannot currently be extracted in
- *       parallel). If left unspecified, the extractor uses a single thread.
+ *   <li><code>LGTM_THREADS</code> determines how many threads are used for parallel extraction of
+ *       JavaScript files (TypeScript files cannot currently be extracted in parallel). If left
+ *       unspecified, the extractor uses a single thread.
  *   <li><code>LGTM_TRAP_CACHE</code> and <code>LGTM_TRAP_CACHE_BOUND</code> can be used to specify
  *       the location and size of a trap cache to be used during extraction.
  * </ul>

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -37,7 +37,7 @@ public class Main {
    * A version identifier that should be updated every time the extractor changes in such a way that
    * it may produce different tuples for the same file under the same {@link ExtractorConfig}.
    */
-  public static final String EXTRACTOR_VERSION = "2019-03-13";
+  public static final String EXTRACTOR_VERSION = "2019-03-21";
 
   public static final Pattern NEWLINE = Pattern.compile("\n");
 

javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -84,9 +84,7 @@ private Path addFile(boolean extracted, FileType fileType, Path root, String...
     return f;
   }
 
-  /**
-   * Add a file with default file type.
-   */
+  /** Add a file with default file type. */
   private Path addFile(boolean extracted, Path root, String... components) throws IOException {
     return addFile(extracted, null, root, components);
   }

javascript/extractor/tests/encoding/input/zwsp.js

@@ -0,0 +1 @@
+​

javascript/extractor/tests/encoding/output/trap/zwsp.js.trap

@@ -0,0 +1,26 @@
+#10000=@"/zwsp.js;sourcefile"
+files(#10000,"/zwsp.js","zwsp","js",0)
+#10001=@"/;folder"
+folders(#10001,"/","")
+containerparent(#10001,#10000)
+#10002=@"loc,{#10000},0,0,0,0"
+locations_default(#10002,#10000,0,0,0,0)
+hasLocation(#10000,#10002)
+#20000=@"global_scope"
+scopes(#20000,0)
+#20001=@"script;{#10000},1,1"
+toplevels(#20001,0)
+#20002=@"loc,{#10000},1,1,1,1"
+locations_default(#20002,#10000,1,1,1,1)
+hasLocation(#20001,#20002)
+#20003=*
+jsParseErrors(#20003,#20001,"Error: Unexpected character '​' (U+200B)","​
+")
+hasLocation(#20003,#20002)
+#20004=*
+lines(#20004,#20001,"​","
+")
+hasLocation(#20004,#20002)
+numlines(#20001,1,0,0)
+numlines(#10000,1,0,0)
+filetype(#10000,"javascript")

javascript/extractor/tests/shebang/output/trap/tst.html.trap

@@ -14,7 +14,7 @@ toplevels(#20001,1)
 locations_default(#20002,#10000,3,17,3,17)
 hasLocation(#20001,#20002)
 #20003=*
-jsParseErrors(#20003,#20001,"Error: Unexpected character '#'","#!/usr/bin/node
+jsParseErrors(#20003,#20001,"Error: Unexpected character '#' (U+0023)","#!/usr/bin/node
 ")
 #20004=@"loc,{#10000},4,1,4,1"
 locations_default(#20004,#10000,4,1,4,1)