TS: fix extraction of symbols with unusual names

Author: asger-semmle

javascript/extractor/lib/typescript/src/type_table.ts

@@ -251,8 +251,8 @@ export class TypeTable {
    * A symbol string is a `;`-separated string consisting of:
    * - a tag string, `root`, `member`, or `other`,
    * - an empty string or a `file:pos` string to distinguish this from symbols with other lexical roots,
-   * - the unqualified name of the symbol,
-   * - for non-root symbols, the ID of the parent symbol.
+   * - the ID of the parent symbol, or an empty string if this is a root symbol,
+   * - the unqualified name of the symbol.
    *
    * Symbol strings serve the same dual purpose as type strings (see `typeIds`).
    */
@@ -667,11 +667,11 @@ export class TypeTable {
   private getSymbolString(symbol: AugmentedSymbol): string {
     let parent = symbol.parent;
     if (parent == null || parent.escapedName === ts.InternalSymbolName.Global) {
-      return "root;" + this.getSymbolDeclarationString(symbol) + ";" + symbol.name;
+      return "root;" + this.getSymbolDeclarationString(symbol) + ";;" + symbol.name;
     } else if (parent.exports != null && parent.exports.get(symbol.escapedName) === symbol) {
-      return "member;;" + symbol.name + ";" + this.getSymbolId(parent);
+      return "member;;" + this.getSymbolId(parent) + ";" + symbol.name;
     } else {
-      return "other;" + this.getSymbolDeclarationString(symbol) + ";" + symbol.name + ";" + this.getSymbolId(parent);
+      return "other;" + this.getSymbolDeclarationString(symbol) + ";" + this.getSymbolId(parent) + ";" + symbol.name;
     }
   }
 

javascript/extractor/src/com/semmle/ts/extractor/TypeExtractor.java

@@ -4,7 +4,11 @@
 import com.google.gson.JsonObject;
 import com.semmle.util.trap.TrapWriter;
 import com.semmle.util.trap.TrapWriter.Label;
+
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -97,7 +101,7 @@ public void extract() {
   private void extractType(int id) {
     Label lbl = trapWriter.globalID("type;" + id);
     String contents = table.getTypeString(id);
-    String[] parts = contents.split(";");
+    String[] parts = split(contents);
     int kind = tagToKind.get(parts[0]);
     trapWriter.addTuple("types", lbl, kind, table.getTypeToStringValue(id));
     int firstChild = 1;
@@ -160,14 +164,15 @@ private void extractPropertyLookups(JsonObject lookups) {
   }
 
   private void extractSymbol(int index) {
-    // Format is: kind;decl;name[;parent]
-    String[] parts = table.getSymbolString(index).split(";");
+    // Format is: kind;decl;parent;name
+    String[] parts = split(table.getSymbolString(index), 4);
     int kind = symbolKind.get(parts[0]);
-    String name = parts[2];
+    String name = parts[3];
     Label label = trapWriter.globalID("symbol;" + index);
     trapWriter.addTuple("symbols", label, kind, name);
-    if (parts.length == 4) {
-      Label parentLabel = trapWriter.globalID("symbol;" + parts[3]);
+    String parentStr = parts[2];
+    if (parentStr.length() > 0) {
+      Label parentLabel = trapWriter.globalID("symbol;" + parentStr);
       trapWriter.addTuple("symbol_parent", label, parentLabel);
     }
   }
@@ -185,7 +190,7 @@ private void extractSymbolNameMapping(String relationName, JsonObject mappings)
   private void extractSignature(int index) {
     // Format is:
     // kind;numTypeParams;requiredParams;returnType(;paramName;paramType)*
-    String[] parts = table.getSignatureString(index).split(";");
+    String[] parts = split(table.getSignatureString(index));
     Label label = trapWriter.globalID("signature;" + index);
     int kind = Integer.parseInt(parts[0]);
     int numberOfTypeParameters = Integer.parseInt(parts[1]);
@@ -269,4 +274,35 @@ private void extractSelfTypes(JsonObject table) {
           trapWriter.globalID("type;" + typeId));
     }
   }
+
+  /** Like {@link #split(String)} without a limit. */
+  private static String[] split(String input) {
+    return split(input, -1);
+  }
+
+  /**
+   * Splits the input around the semicolon (<code>;</code>) character, preserving all empty
+   * substrings.
+   *
+   * <p>At most <code>limit</code> substrings will be extracted. If the limit is reached, the last
+   * substring will extend to the end of the string, possibly itself containing semicolons.
+   *
+   * <p>Note that the {@link String#split(String)} method does not preserve empty substrings at the
+   * end of the string in case the string ends with a semicolon.
+   */
+  private static String[] split(String input, int limit) {
+    List<String> result = new ArrayList<String>();
+    int lastPos = 0;
+    for (int i = 0; i < input.length(); ++i) {
+      if (input.charAt(i) == ';') {
+        result.add(input.substring(lastPos, i));
+        lastPos = i + 1;
+        if (result.size() == limit - 1) break;
+      }
+    }
+    result.add(input.substring(lastPos));
+    return result.toArray(EMPTY_STRING_ARRAY);
+  }
+
+  private static final String[] EMPTY_STRING_ARRAY = new String[0];
 }

javascript/ql/test/library-tests/TypeScript/RegressionTests/EmptyName/test.expected

@@ -0,0 +1,11 @@
+|  in unknown scope |
+| Array in global scope |
+| Intl in global scope |
+| Intl.CollatorOptions in global scope |
+| Intl.NumberFormatOptions in global scope |
+| MK in unknown scope |
+| Mapped in test.ts |
+| RegExp in global scope |
+| RegExpMatchArray in global scope |
+| fn in test.ts |
+| test.ts |

javascript/ql/test/library-tests/TypeScript/RegressionTests/EmptyName/test.ql

@@ -0,0 +1,4 @@
+import javascript
+
+from CanonicalName name
+select name

javascript/ql/test/library-tests/TypeScript/RegressionTests/EmptyName/test.ts

@@ -0,0 +1,9 @@
+type Mapped<MK extends string = ''> = {
+        [mk in MK]: string
+    };
+
+export function fn(ev: Mapped) {
+    const props: Mapped = {
+        ...ev
+    };
+}

javascript/ql/test/library-tests/TypeScript/RegressionTests/EmptyName/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "include": ["."]
+}

javascript/ql/test/library-tests/TypeScript/RegressionTests/SemicolonInName/test.expected

@@ -0,0 +1,4 @@
+| Bar.Foo in global scope | Bar in global scope |
+| Intl.CollatorOptions in global scope | Intl in global scope |
+| Intl.NumberFormatOptions in global scope | Intl in global scope |
+| fn in test.ts | test.ts |

javascript/ql/test/library-tests/TypeScript/RegressionTests/SemicolonInName/test.ql

@@ -0,0 +1,4 @@
+import javascript
+
+from CanonicalName typename
+select typename, typename.getParent()

javascript/ql/test/library-tests/TypeScript/RegressionTests/SemicolonInName/test.ts

@@ -0,0 +1,9 @@
+type Mapped<MK extends string = ';'> = {
+        [mk in MK]: string
+    };
+
+export function fn(ev: Mapped) {
+    const props: Mapped = {
+        ...ev
+    };
+}

javascript/ql/test/library-tests/TypeScript/RegressionTests/SemicolonInName/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "include": ["."]
+}