Merge pull request #2379 from asger-semmle/typescript-fixes

TS: A bunch of TypeScript fixes

Author: esbena

javascript/extractor/lib/typescript/package.json

@@ -8,7 +8,8 @@
         "build": "tsc --project tsconfig.json",
         "check": "tsc --noEmit --project . && tslint --project .",
         "lint": "tslint --project .",
-        "lint-fix": "tslint --project . --fix"
+        "lint-fix": "tslint --project . --fix",
+        "watch": "tsc -p . -w --sourceMap"
     },
     "devDependencies": {
         "@types/node": "12.7.11",

javascript/extractor/lib/typescript/src/ast_extractor.ts

@@ -198,7 +198,9 @@ export function augmentAst(ast: AugmentedSourceFile, code: string, project: Proj
                     : null;
                 let type = contextualType || typeChecker.getTypeAtLocation(node);
                 if (type != null) {
-                    let id = typeTable.buildType(type);
+                    let parent = node.parent;
+                    let unfoldAlias = ts.isTypeAliasDeclaration(parent) && node === parent.type;
+                    let id = typeTable.buildType(type, unfoldAlias);
                     if (id != null) {
                         node.$type = id;
                     }

javascript/extractor/lib/typescript/src/main.ts

@@ -253,7 +253,7 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
         fileExists: (path: string) => fs.existsSync(path),
         readFile: ts.sys.readFile,
     };
-    let config = ts.parseJsonConfigFileContent(tsConfig, parseConfigHost, basePath);
+    let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath);
     let project = new Project(tsConfigFilename, config, state.typeTable);
     project.load();
 
@@ -272,7 +272,9 @@ function handleOpenProjectCommand(command: OpenProjectCommand) {
     });
 
     for (let typeRoot of typeRoots || []) {
-        traverseTypeRoot(typeRoot, "");
+        if (fs.existsSync(typeRoot) && fs.statSync(typeRoot).isDirectory()) {
+            traverseTypeRoot(typeRoot, "");
+        }
     }
 
     for (let sourceFile of program.getSourceFiles()) {

javascript/extractor/lib/typescript/src/type_table.ts

@@ -92,6 +92,20 @@ function isTypeofCandidateSymbol(symbol: ts.Symbol) {
 
 const signatureKinds = [ts.SignatureKind.Call, ts.SignatureKind.Construct];
 
+/**
+ * Bitmask of flags set on a signature, but not exposed in the public API.
+ */
+const enum InternalSignatureFlags {
+  HasRestParameter = 1
+}
+
+/**
+ * Signature interface with some internal properties exposed.
+ */
+interface AugmentedSignature extends ts.Signature {
+  flags?: InternalSignatureFlags;
+}
+
 /**
  * Encodes property lookup tuples `(baseType, name, property)` as three
  * staggered arrays.
@@ -102,6 +116,16 @@ interface PropertyLookupTable {
   propertyTypes: number[];
 }
 
+/**
+ * Encodes `(aliasType, underlyingType)` tuples as two staggered arrays.
+ *
+ * Such a tuple denotes that `aliasType` is an alias for `underlyingType`.
+ */
+interface TypeAliasTable {
+  aliasTypes: number[];
+  underlyingTypes: number[];
+}
+
 /**
  * Encodes type signature tuples `(baseType, kind, index, signature)` as four
  * staggered arrays.
@@ -287,6 +311,11 @@ export class TypeTable {
     propertyTypes: [],
   };
 
+  private typeAliases: TypeAliasTable = {
+    aliasTypes: [],
+    underlyingTypes: [],
+  };
+
   private signatureMappings: SignatureTable = {
     baseTypes: [],
     kinds: [],
@@ -304,7 +333,7 @@ export class TypeTable {
     propertyTypes: [],
   };
 
-  private buildTypeWorklist: [ts.Type, number][] = [];
+  private buildTypeWorklist: [ts.Type, number, boolean][] = [];
 
   private expansiveTypes: Map<number, boolean> = new Map();
 
@@ -372,9 +401,9 @@ export class TypeTable {
   /**
    * Gets the canonical ID for the given type, generating a fresh ID if necessary.
    */
-  public buildType(type: ts.Type): number | null {
+  public buildType(type: ts.Type, unfoldAlias: boolean): number | null {
     this.isInShallowTypeContext = false;
-    let id = this.getId(type);
+    let id = this.getId(type, unfoldAlias);
     this.iterateBuildTypeWorklist();
     if (id == null) return null;
     return id;
@@ -385,7 +414,7 @@ export class TypeTable {
    *
    * Returns `null` if we do not support extraction of this type.
    */
-  public getId(type: ts.Type): number | null {
+  public getId(type: ts.Type, unfoldAlias: boolean): number | null {
     if (this.typeRecursionDepth > 100) {
       // Ignore infinitely nested anonymous types, such as `{x: {x: {x: ... }}}`.
       // Such a type can't be written directly with TypeScript syntax (as it would need to be named),
@@ -397,19 +426,19 @@ export class TypeTable {
       type = this.typeChecker.getBaseTypeOfLiteralType(type);
     }
     ++this.typeRecursionDepth;
-    let content = this.getTypeString(type);
+    let content = this.getTypeString(type, unfoldAlias);
     --this.typeRecursionDepth;
     if (content == null) return null; // Type not supported.
     let id = this.typeIds.get(content);
     if (id == null) {
-      let stringValue = this.stringifyType(type);
+      let stringValue = this.stringifyType(type, unfoldAlias);
       if (stringValue == null) {
         return null; // Type not supported.
       }
       id = this.typeIds.size;
       this.typeIds.set(content, id);
       this.typeToStringValues.push(stringValue);
-      this.buildTypeWorklist.push([type, id]);
+      this.buildTypeWorklist.push([type, id, unfoldAlias]);
       this.typeExtractionState.push(
         this.isInShallowTypeContext ? TypeExtractionState.PendingShallow : TypeExtractionState.PendingFull);
       // If the type is the self-type for a named type (not a generic instantiation of it),
@@ -426,20 +455,20 @@ export class TypeTable {
         this.typeExtractionState[id] = TypeExtractionState.PendingFull;
       } else if (state === TypeExtractionState.DoneShallow) {
         this.typeExtractionState[id] = TypeExtractionState.PendingFull;
-        this.buildTypeWorklist.push([type, id]);
+        this.buildTypeWorklist.push([type, id, unfoldAlias]);
       }
     }
     return id;
   }
 
-  private stringifyType(type: ts.Type): string {
+  private stringifyType(type: ts.Type, unfoldAlias: boolean): string {
+    let formatFlags = unfoldAlias
+        ? ts.TypeFormatFlags.InTypeAlias
+        : ts.TypeFormatFlags.UseAliasDefinedOutsideCurrentScope;
     let toStringValue: string;
     // Some types can't be stringified. Just discard the type if we can't stringify it.
     try {
-      toStringValue = this.typeChecker.typeToString(
-            type,
-            undefined,
-            ts.TypeFormatFlags.UseAliasDefinedOutsideCurrentScope);
+      toStringValue = this.typeChecker.typeToString(type, undefined, formatFlags);
     } catch (e) {
       console.warn("Recovered from a compiler crash while stringifying a type. Discarding the type.");
       console.warn(e.stack);
@@ -477,9 +506,9 @@ export class TypeTable {
   /**
    * Gets a string representing the kind and contents of the given type.
    */
-  private getTypeString(type: AugmentedType): string | null {
+  private getTypeString(type: AugmentedType, unfoldAlias: boolean): string | null {
     // Reference to a type alias.
-    if (type.aliasSymbol != null) {
+    if (!unfoldAlias && type.aliasSymbol != null) {
       let tag = "reference;" + this.getSymbolId(type.aliasSymbol);
       return type.aliasTypeArguments == null
           ? tag
@@ -499,7 +528,7 @@ export class TypeTable {
       if (flags & ts.TypeFlags.TypeVariable) {
         let enclosingType = getEnclosingTypeOfThisType(type);
         if (enclosingType != null) {
-          return "this;" + this.getId(enclosingType);
+          return "this;" + this.getId(enclosingType, false);
         } else if (symbol.parent == null) {
           // The type variable is bound on a call signature. Only extract it by name.
           return "lextypevar;" + symbol.name;
@@ -730,7 +759,7 @@ export class TypeTable {
   private makeTypeStringVector(tag: string, types: ReadonlyArray<ts.Type>, length = types.length): string | null {
     let hash = tag;
     for (let i = 0; i < length; ++i) {
-      let id = this.getId(types[i]);
+      let id = this.getId(types[i], false);
       if (id == null) return null;
       hash += ";" + id;
     }
@@ -748,7 +777,7 @@ export class TypeTable {
     for (let property of type.getProperties()) {
       let propertyType = this.typeChecker.getTypeOfSymbolAtLocation(property, this.arbitraryAstNode);
       if (propertyType == null) return null;
-      let propertyTypeId = this.getId(propertyType);
+      let propertyTypeId = this.getId(propertyType, false);
       if (propertyTypeId == null) return null;
       hash += ";p" + this.getSymbolId(property) + ';' + propertyTypeId;
     }
@@ -761,13 +790,13 @@ export class TypeTable {
     }
     let indexType = type.getStringIndexType();
     if (indexType != null) {
-      let indexTypeId = this.getId(indexType);
+      let indexTypeId = this.getId(indexType, false);
       if (indexTypeId == null) return null;
       hash += ";s" + indexTypeId;
     }
     indexType = type.getNumberIndexType();
     if (indexType != null) {
-      let indexTypeId = this.getId(indexType);
+      let indexTypeId = this.getId(indexType, false);
       if (indexTypeId == null) return null;
       hash += ";i" + indexTypeId;
     }
@@ -789,6 +818,7 @@ export class TypeTable {
       typeStrings: Array.from(this.typeIds.keys()),
       typeToStringValues: this.typeToStringValues,
       propertyLookups: this.propertyLookups,
+      typeAliases: this.typeAliases,
       symbolStrings: Array.from(this.symbolIds.keys()),
       moduleMappings: this.moduleMappings,
       globalMappings: this.globalMappings,
@@ -812,10 +842,17 @@ export class TypeTable {
     let worklist = this.buildTypeWorklist;
     let typeExtractionState = this.typeExtractionState;
     while (worklist.length > 0) {
-      let [type, id] = worklist.pop();
+      let [type, id, unfoldAlias] = worklist.pop();
       let isShallowContext = typeExtractionState[id] === TypeExtractionState.PendingShallow;
       if (isShallowContext && !isTypeAlwaysSafeToExpand(type)) {
         typeExtractionState[id] = TypeExtractionState.DoneShallow;
+      } else if (type.aliasSymbol != null && !unfoldAlias) {
+        typeExtractionState[id] = TypeExtractionState.DoneFull;
+        let underlyingTypeId = this.getId(type, true);
+        if (underlyingTypeId != null) {
+          this.typeAliases.aliasTypes.push(id);
+          this.typeAliases.underlyingTypes.push(underlyingTypeId);
+        }
       } else {
         typeExtractionState[id] = TypeExtractionState.DoneFull;
         this.isInShallowTypeContext = isShallowContext || this.isExpansiveTypeReference(type);
@@ -847,7 +884,7 @@ export class TypeTable {
     for (let symbol of props) {
       let propertyType = this.typeChecker.getTypeOfSymbolAtLocation(symbol, this.arbitraryAstNode);
       if (propertyType == null) continue;
-      let propertyTypeId = this.getId(propertyType);
+      let propertyTypeId = this.getId(propertyType, false);
       if (propertyTypeId == null) continue;
       this.propertyLookups.baseTypes.push(id);
       this.propertyLookups.names.push(symbol.name);
@@ -879,7 +916,7 @@ export class TypeTable {
   /**
    * Returns a unique string for the given call/constructor signature.
    */
-  private getSignatureString(kind: ts.SignatureKind, signature: ts.Signature): string {
+  private getSignatureString(kind: ts.SignatureKind, signature: AugmentedSignature): string {
     let parameters = signature.getParameters();
     let numberOfTypeParameters = signature.typeParameters == null
         ? 0
@@ -892,27 +929,51 @@ export class TypeTable {
         break;
       }
     }
-    let returnTypeId = this.getId(signature.getReturnType());
+    let hasRestParam = (signature.flags & InternalSignatureFlags.HasRestParameter) !== 0;
+    let restParameterTag = '';
+    if (hasRestParam) {
+      if (requiredParameters === parameters.length) {
+        // Do not count the rest parameter as a required parameter
+        requiredParameters = parameters.length - 1;
+      }
+      if (parameters.length === 0) return null;
+      let restParameter = parameters[parameters.length - 1];
+      let restParameterType = this.typeChecker.getTypeOfSymbolAtLocation(restParameter, this.arbitraryAstNode);
+      if (restParameterType == null) return null;
+      let restParameterTypeId = this.getId(restParameterType, false);
+      if (restParameterTypeId == null) return null;
+      restParameterTag = '' + restParameterTypeId;
+    }
+    let returnTypeId = this.getId(signature.getReturnType(), false);
     if (returnTypeId == null) {
       return null;
     }
-    let tag = `${kind};${numberOfTypeParameters};${requiredParameters};${returnTypeId}`;
+    let tag = `${kind};${numberOfTypeParameters};${requiredParameters};${restParameterTag};${returnTypeId}`;
     for (let typeParameter of signature.typeParameters || []) {
       tag += ";" + typeParameter.symbol.name;
       let constraint = typeParameter.getConstraint();
       let constraintId: number;
-      if (constraint == null || (constraintId = this.getId(constraint)) == null) {
+      if (constraint == null || (constraintId = this.getId(constraint, false)) == null) {
         tag += ";";
       } else {
         tag += ";" + constraintId;
       }
     }
-    for (let parameter of parameters) {
+    for (let paramIndex = 0; paramIndex < parameters.length; ++paramIndex) {
+      let parameter = parameters[paramIndex];
       let parameterType = this.typeChecker.getTypeOfSymbolAtLocation(parameter, this.arbitraryAstNode);
       if (parameterType == null) {
         return null;
       }
-      let parameterTypeId = this.getId(parameterType);
+      let isRestParameter = hasRestParam && (paramIndex === parameters.length - 1);
+      if (isRestParameter) {
+        // The type of the rest parameter is the array type, but we wish to extract the non-array type.
+        if (!isTypeReference(parameterType)) return null;
+        let typeArguments = parameterType.typeArguments;
+        if (typeArguments == null || typeArguments.length === 0) return null;
+        parameterType = typeArguments[0];
+      }
+      let parameterTypeId = this.getId(parameterType, false);
       if (parameterTypeId == null) {
         return null;
       }
@@ -946,7 +1007,7 @@ export class TypeTable {
 
   private extractIndexer(baseType: number, indexType: ts.Type, table: IndexerTable) {
     if (indexType == null) return;
-    let indexTypeId = this.getId(indexType);
+    let indexTypeId = this.getId(indexType, false);
     if (indexTypeId == null) return;
     table.baseTypes.push(baseType);
     table.propertyTypes.push(indexTypeId);
@@ -1017,7 +1078,7 @@ export class TypeTable {
     let selfType = this.getSelfType(type);
     if (selfType != null) {
       this.checkExpansiveness(selfType);
-      let id = this.getId(selfType);
+      let id = this.getId(selfType, false);
       return this.expansiveTypes.get(id);
     }
     return false;
@@ -1086,7 +1147,7 @@ export class TypeTable {
     search(type, 0);
 
     function search(type: ts.TypeReference, expansionDepth: number): number | null {
-      let id = typeTable.getId(type);
+      let id = typeTable.getId(type, false);
       if (id == null) return null;
 
       let index = indexTable.get(id);

javascript/extractor/src/com/semmle/js/ast/AFunction.java

@@ -3,6 +3,7 @@
 import com.semmle.ts.ast.DecoratorList;
 import com.semmle.ts.ast.ITypeExpression;
 import com.semmle.ts.ast.TypeParameter;
+import com.semmle.util.data.IntList;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -18,6 +19,9 @@ public class AFunction<B> {
   private final List<ITypeExpression> parameterTypes;
   private final ITypeExpression thisParameterType;
   private final List<DecoratorList> parameterDecorators;
+  private final IntList optionalParameterIndices;
+
+  public static final IntList noOptionalParams = IntList.create(0, 0);
 
   public AFunction(
       Identifier id,
@@ -29,7 +33,8 @@ public AFunction(
       List<ITypeExpression> parameterTypes,
       List<DecoratorList> parameterDecorators,
       ITypeExpression returnType,
-      ITypeExpression thisParameterType) {
+      ITypeExpression thisParameterType,
+      IntList optionalParameterIndices) {
     this.id = id;
     this.params = new ArrayList<IPattern>(params.size());
     this.defaults = new ArrayList<Expression>(params.size());
@@ -42,6 +47,7 @@ public AFunction(
     this.returnType = returnType;
     this.thisParameterType = thisParameterType;
     this.parameterDecorators = parameterDecorators;
+    this.optionalParameterIndices = optionalParameterIndices;
 
     IPattern rest = null;
     for (Expression param : params) {
@@ -143,4 +149,8 @@ public ITypeExpression getThisParameterType() {
   public List<DecoratorList> getParameterDecorators() {
     return parameterDecorators;
   }
+
+  public IntList getOptionalParmaeterIndices() {
+    return optionalParameterIndices;
+  }
 }