Merge pull request #907 from asger-semmle/reapply-closure-modules

Approved by xiemaisi

Author: semmle-qlci

change-notes/1.20/analysis-javascript.md

@@ -13,6 +13,8 @@
 
 * Type inference for function calls has been improved. This may give additional results for queries that rely on type inference.
 
+* The [Closure-Library](https://github.com/google/closure-library/wiki/goog.module:-an-ES6-module-like-alternative-to-goog.provide) module system is now supported.
+
 ## New queries
 
 | **Query**                                     | **Tags**                                             | **Purpose**                                                                                                                                                                 |

javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java

@@ -306,7 +306,7 @@ private class V extends DefaultVisitor<Context, Label> {
 		public V(Platform platform, SourceType sourceType) {
 			this.platform = platform;
 			this.sourceType = sourceType;
-			this.isStrict = sourceType == SourceType.MODULE;
+			this.isStrict = sourceType.isStrictMode();
 		}
 
 		private Label visit(INode child, Label parent, int childIndex) {
@@ -546,31 +546,37 @@ public Label visit(Program nd, Context c) {
 
 			isStrict = hasUseStrict(nd.getBody());
 
-			// if we're extracting a Node.js/ES2015 module, introduce module scope
-			if (platform == Platform.NODE) {
-				// add node.js-specific globals
-				scopeManager.addVariables("global", "process", "console", "Buffer");
+			// Add platform-specific globals.
+			scopeManager.addVariables(platform.getPredefinedGlobals());
 
+			// Introduce local scope if there is one.
+			if (sourceType.hasLocalScope()) {
 				Label moduleScopeKey = trapwriter.globalID("module;{" + locationManager.getFileLabel() + "}," + locationManager.getStartLine() + "," + locationManager.getStartColumn());
 				scopeManager.enterScope(3, moduleScopeKey, toplevelLabel);
-				// special variables aren't available in `.mjs` modules
-				if (!".mjs".equals(locationManager.getSourceFileExtension()))
-					scopeManager.addVariables("require", "module", "exports", "__filename", "__dirname", "arguments");
-				trapwriter.addTuple("isModule", toplevelLabel);
-			} else if (sourceType == SourceType.MODULE) {
-				Label moduleScopeKey = trapwriter.globalID("module;{" + locationManager.getFileLabel() + "}," + locationManager.getStartLine() + "," + locationManager.getStartColumn());
-				scopeManager.enterScope(3, moduleScopeKey, toplevelLabel);
+				scopeManager.addVariables(sourceType.getPredefinedLocals(platform, locationManager.getSourceFileExtension()));
 				trapwriter.addTuple("isModule", toplevelLabel);
 			}
 
+			// Emit the specific source type.
+			switch (sourceType) {
+			case CLOSURE_MODULE:
+				trapwriter.addTuple("isClosureModule", toplevelLabel);
+				break;
+			case MODULE:
+				trapwriter.addTuple("isES2015Module", toplevelLabel);
+				break;
+			default:
+				break;
+			}
+
 			// add all declared global (or module-scoped) names, both non-lexical and lexical
 			scopeManager.addNames(scopeManager.collectDeclaredNames(nd, isStrict, false, DeclKind.none));
 			scopeManager.addNames(scopeManager.collectDeclaredNames(nd, isStrict, true, DeclKind.none));
 
 			visitAll(nd.getBody(), toplevelLabel);
 
-			// if we're extracting a Node.js/ES2015 module, leave its scope
-			if (platform == Platform.NODE || sourceType == SourceType.MODULE)
+			// Leave the local scope again.
+			if (sourceType.hasLocalScope())
 				scopeManager.leaveScope();
 
 			contextManager.leaveContainer();

javascript/extractor/src/com/semmle/js/extractor/ExtractorConfig.java

@@ -4,7 +4,12 @@
 import java.nio.charset.IllegalCharsetNameException;
 import java.nio.charset.StandardCharsets;
 import java.nio.charset.UnsupportedCharsetException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Set;
 
+import com.semmle.js.parser.JcornWrapper;
 import com.semmle.util.data.StringUtil;
 import com.semmle.util.exception.UserError;
 
@@ -50,13 +55,75 @@ public static ECMAVersion of(int version) {
         }
     };
 
+    /**
+     * The type of a source file, which together with the {@link Platform}
+     * determines how the top-level scope of the file behaves, and whether ES2015
+     * module syntax should be allowed.
+     * <p>
+     * Note that the names of these enum members are depended on by {@link Main},
+     * {@link AutoBuild}, and {@link JcornWrapper}.
+     */
     public static enum SourceType {
-        SCRIPT, MODULE, AUTO;
+        /** A script executed in the global scope. */
+        SCRIPT,
+
+        /** An ES2015 module. */
+        MODULE,
+
+        /** A Closure-Library module, defined using `goog.module()`. */
+        CLOSURE_MODULE,
+
+        /** A CommonJS module that is not also an ES2015 module. */
+        COMMONJS_MODULE,
+
+        /** Automatically determined source type. */
+        AUTO;
 
         @Override
         public String toString() {
             return StringUtil.lc(name());
         }
+
+        /**
+         * Returns true if this source is executed directly in the global scope,
+         * or false if it has its own local scope.
+         */
+        public boolean hasLocalScope() {
+            return this != SCRIPT;
+        }
+
+        /**
+         * Returns true if this source is implicitly in strict mode.
+         */
+        public boolean isStrictMode() {
+            return this == MODULE;
+        }
+
+        private static final Set<String> closureLocals = Collections.singleton("exports");
+        private static final Set<String> commonJsLocals = new LinkedHashSet<>(Arrays.asList("require", "module", "exports", "__filename", "__dirname", "arguments"));
+
+        /**
+         * Returns the set of local variables in scope at the top-level of this module.
+         * <p>
+         * If this source type has no local scope, the empty set is returned.
+         */
+        public Set<String> getPredefinedLocals(Platform platform, String extension) {
+            switch (this) {
+            case CLOSURE_MODULE:
+                return closureLocals;
+            case COMMONJS_MODULE:
+                return commonJsLocals;
+            case MODULE:
+                if (platform == Platform.NODE && !extension.equals(".mjs")) {
+                    // An ES2015 module that is compiled to a Node.js module effectively has the locals
+                    // from Node.js even if they are not part of the ES2015 standard.
+                    return commonJsLocals;
+                }
+                return Collections.emptySet();
+            default:
+                return Collections.emptySet();
+            }
+        }
     };
 
     public static enum Platform {
@@ -66,6 +133,15 @@ public static enum Platform {
         public String toString() {
             return StringUtil.lc(name());
         }
+
+        private static final Set<String> nodejsGlobals = new LinkedHashSet<>(Arrays.asList("global", "process", "console", "Buffer"));
+
+        /**
+         * Gets the set of predefined globals for this platform.
+         */
+        public Set<String> getPredefinedGlobals() {
+            return this == NODE ? nodejsGlobals : Collections.emptySet();
+        }
     }
 
     /**

javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java

@@ -29,8 +29,8 @@ public JSExtractor(ExtractorConfig config) {
 		this.config = config;
 	}
 
-	// heuristic: if `import` or `export` appears at the beginning of a line, it's probably a module
-	private static final Pattern containsImportOrExport = Pattern.compile("(?m)^([ \t]*)(import|export)\\b");
+	// heuristic: if `import`, `export`, or `goog.module` appears at the beginning of a line, it's probably a module
+	private static final Pattern containsModuleIndicator = Pattern.compile("(?m)^([ \t]*)(import|export|goog\\.module)\\b");
 
 	public Pair<Label, LoCInfo> extract(TextualExtractor textualExtractor, String source, int toplevelKind, ScopeManager scopeManager) throws ParseError {
 		// if the file starts with `{ "<string>":` it won't parse as JavaScript; try parsing as JSON instead
@@ -69,9 +69,10 @@ public SourceType establishSourceType(String source, boolean allowLeadingWS) {
 		if (sourceType != SourceType.AUTO)
 			return sourceType;
 		if (config.getEcmaVersion().compareTo(ECMAVersion.ECMA2015) >= 0) {
-			Matcher m = containsImportOrExport.matcher(source);
-			if (m.find() && (allowLeadingWS || m.group(1).isEmpty()))
-				return SourceType.MODULE;
+			Matcher m = containsModuleIndicator.matcher(source);
+			if (m.find() && (allowLeadingWS || m.group(1).isEmpty())) {
+				return m.group(2).startsWith("goog") ? SourceType.CLOSURE_MODULE : SourceType.MODULE;
+			}
 		}
 		return SourceType.SCRIPT;
 	}
@@ -89,6 +90,9 @@ public Pair<Label, LoCInfo> extract(TextualExtractor textualExtractor,
 		LoCInfo loc;
 		if (ast != null) {
 			platform = getPlatform(platform, ast);
+			if (sourceType == SourceType.SCRIPT && platform == Platform.NODE) {
+				sourceType = SourceType.COMMONJS_MODULE;
+			}
 
 			lexicalExtractor = new LexicalExtractor(textualExtractor, parserRes.getTokens(), parserRes.getComments());
 			ASTExtractor scriptExtractor = new ASTExtractor(lexicalExtractor, scopeManager);
@@ -125,7 +129,7 @@ public Pair<Label, LoCInfo> extract(TextualExtractor textualExtractor,
 
 		if (config.isExterns())
 			textualExtractor.getTrapwriter().addTuple("isExterns", toplevelLabel);
-		if (platform == Platform.NODE && sourceType != SourceType.MODULE)
+		if (platform == Platform.NODE && sourceType == SourceType.COMMONJS_MODULE)
 			textualExtractor.getTrapwriter().addTuple("isNodejs", toplevelLabel);
 
 		return Pair.make(toplevelLabel, loc);

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -41,7 +41,7 @@ public class Main {
 	 * such a way that it may produce different tuples for the same file under the same
 	 * {@link ExtractorConfig}.
 	 */
-	public static final String EXTRACTOR_VERSION = "2019-01-29";
+	public static final String EXTRACTOR_VERSION = "2019-02-04";
 
 	public static final Pattern NEWLINE = Pattern.compile("\n");
 

javascript/extractor/tests/closure/input/googDotDeclareModuleId.js

@@ -0,0 +1,3 @@
+goog.declareModuleId('test');
+
+export let x = 5;

javascript/extractor/tests/closure/input/googDotModule.js

@@ -0,0 +1,3 @@
+goog.module('test');
+var x = 5;
+exports = { x: x };

javascript/extractor/tests/closure/input/googDotProvide.js

@@ -0,0 +1,3 @@
+goog.provide('test.x');
+
+test.x = 5;