Merge pull request #868 from xiemaisi/js/discard-tokens-early

Approved by esben-semmle

Author: semmle-qlci

javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java

@@ -823,7 +823,6 @@ private void extractFunction(IFunction nd, Label key) {
 
 			contextManager.leaveContainer();
 			scopeManager.leaveScope();
-			lexicalExtractor.emitNumlines(key, nd.getLoc().getStart(), nd.getLoc().getEnd());
 		}
 
 		private void extractParameterDefaultsAndTypes(IFunction nd, Label key, int paramCount) {

javascript/extractor/src/com/semmle/js/extractor/CFGExtractor.java

@@ -773,14 +773,18 @@ public Finally(SourceLocation loc, BlockStatement body) {
 			public <Q, A> A accept(Visitor<Q, A> v, Q q) { return null; }
 		}
 
-		// associate statements with their (direct or indirect) labels
-		private final Map<Statement, Set<String>> loopLabels = new LinkedHashMap<Statement, Set<String>>();
+		// associate statements with their (direct or indirect) labels;
+		// per-function cache, cleared after each function
+		private Map<Statement, Set<String>> loopLabels = new LinkedHashMap<Statement, Set<String>>();
 
-		// cache the set of normal control flow successors
-		private final Map<Node, Object> followingCache = new LinkedHashMap<Node, Object>();
+		// cache the set of normal control flow successors;
+		// per-function cache, cleared after each function
+		private Map<Node, Object> followingCache = new LinkedHashMap<Node, Object>();
 
-		// map from a node in a chain of property accesses or calls to the successor info for the first node in the chain
-		private final Map<Chainable, SuccessorInfo> chainRootSuccessors = new LinkedHashMap<Chainable, SuccessorInfo>();
+		// map from a node in a chain of property accesses or calls to the successor info
+		// for the first node in the chain;
+		// per-function cache, cleared after each function
+		private Map<Chainable, SuccessorInfo> chainRootSuccessors = new LinkedHashMap<Chainable, SuccessorInfo>();
 
 		/**
 		 * Generate entry node.
@@ -1031,6 +1035,16 @@ private void buildFunctionBody(IFunction nd) {
 
 		@Override
 		public Void visit(IFunction nd, SuccessorInfo i) {
+			// save per-function caches
+			Map<Statement, Set<String>> oldLoopLabels = loopLabels;
+			Map<Node, Object> oldFollowingCache = followingCache;
+			Map<Chainable, SuccessorInfo> oldChainRootSuccessors = chainRootSuccessors;
+
+			// clear caches
+			loopLabels = new LinkedHashMap<>();
+			followingCache = new LinkedHashMap<>();
+			chainRootSuccessors = new LinkedHashMap<>();
+
 			if (nd instanceof FunctionDeclaration && nd.hasDeclareKeyword()) {
 				// All 'declared' statements have a no-op CFG node, but their children should
 				// not be processed.
@@ -1039,6 +1053,12 @@ public Void visit(IFunction nd, SuccessorInfo i) {
 			}
 			buildFunctionCreation(nd, i);
 			buildFunctionBody(nd);
+
+			// restore caches
+			loopLabels = oldLoopLabels;
+			followingCache = oldFollowingCache;
+			chainRootSuccessors = oldChainRootSuccessors;
+
 			return null;
 		}
 

javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java

@@ -94,12 +94,14 @@ public Pair<Label, LoCInfo> extract(TextualExtractor textualExtractor,
 			ASTExtractor scriptExtractor = new ASTExtractor(lexicalExtractor, scopeManager);
 			toplevelLabel = scriptExtractor.getToplevelLabel();
 
-			scriptExtractor.extract(ast, platform, sourceType, toplevelKind);
 			lexicalExtractor.extractComments(toplevelLabel);
 			loc = lexicalExtractor.extractLines(parserRes.getSource(), toplevelLabel);
 			lexicalExtractor.extractTokens(toplevelLabel);
-			new CFGExtractor(scriptExtractor).extract(ast);
 			new JSDocExtractor(textualExtractor).extract(lexicalExtractor.getComments());
+			lexicalExtractor.purge();
+
+			scriptExtractor.extract(ast, platform, sourceType, toplevelKind);
+			new CFGExtractor(scriptExtractor).extract(ast);
 		} else {
 			lexicalExtractor = new LexicalExtractor(textualExtractor, new ArrayList<Token>(), new ArrayList<Comment>());
 			ASTExtractor scriptExtractor = new ASTExtractor(lexicalExtractor, null);

javascript/extractor/src/com/semmle/js/extractor/LexicalExtractor.java

@@ -133,7 +133,7 @@ public void extractTokens(Label toplevelKey) {
 			locationManager.emitNodeLocation(token, key);
 
 			// fill in next_token relation
-			while (j < comments.size() && comments.get(j).getLoc().getEnd().compareTo(token.getLoc().getStart()) < 0)
+			while (j < comments.size() && comments.get(j).getLoc().getEnd().compareTo(token.getLoc().getStart()) <= 0)
 				trapwriter.addTuple("next_token", this.trapwriter.localID(comments.get(j++)), key);
 
 			// the parser sometimes duplicates tokens; skip the second one by nulling it out
@@ -173,4 +173,12 @@ public void extractComments(Label toplevelKey) {
 	public String mkToString(SourceElement nd) {
 		return textualExtractor.mkToString(nd);
 	}
+
+	/**
+	 * Purge token and comments information to reduce heap pressure.
+	 */
+	public void purge() {
+		this.tokens.clear();
+		this.comments.clear();
+	}
 }

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -41,7 +41,7 @@ public class Main {
 	 * such a way that it may produce different tuples for the same file under the same
 	 * {@link ExtractorConfig}.
 	 */
-	public static final String EXTRACTOR_VERSION = "2019-01-17";
+	public static final String EXTRACTOR_VERSION = "2019-01-29";
 
 	public static final Pattern NEWLINE = Pattern.compile("\n");
 

javascript/extractor/src/com/semmle/js/extractor/test/TrapTests.java

@@ -5,7 +5,6 @@
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map.Entry;

javascript/extractor/tests/cfg/output/trap/classexpr1.js.trap

@@ -9,113 +9,112 @@ hasLocation(#10000,#10002)
 #20000=@"global_scope"
 scopes(#20000,0)
 #20001=@"script;{#10000},1,1"
-toplevels(#20001,0)
-#20002=@"loc,{#10000},1,1,1,10"
-locations_default(#20002,#10000,1,1,1,10)
-hasLocation(#20001,#20002)
-#20003=*
-stmts(#20003,2,#20001,0,"!class {};")
-hasLocation(#20003,#20002)
-stmtContainers(#20003,#20001)
+#20002=*
+lines(#20002,#20001,"!class {};","")
+#20003=@"loc,{#10000},1,1,1,10"
+locations_default(#20003,#10000,1,1,1,10)
+hasLocation(#20002,#20003)
+numlines(#20001,1,1,0)
 #20004=*
-exprs(#20004,18,#20003,0,"!class {}")
-#20005=@"loc,{#10000},1,1,1,9"
-locations_default(#20005,#10000,1,1,1,9)
+tokeninfo(#20004,8,#20001,0,"!")
+#20005=@"loc,{#10000},1,1,1,1"
+locations_default(#20005,#10000,1,1,1,1)
 hasLocation(#20004,#20005)
-enclosingStmt(#20004,#20003)
-exprContainers(#20004,#20001)
 #20006=*
-exprs(#20006,80,#20004,0,"class {}")
-#20007=@"loc,{#10000},1,2,1,9"
-locations_default(#20007,#10000,1,2,1,9)
+tokeninfo(#20006,7,#20001,1,"class")
+#20007=@"loc,{#10000},1,2,1,6"
+locations_default(#20007,#10000,1,2,1,6)
 hasLocation(#20006,#20007)
-enclosingStmt(#20006,#20003)
-exprContainers(#20006,#20001)
 #20008=*
-properties(#20008,#20006,2,0,"constructor() {}")
-#20009=@"loc,{#10000},1,8,1,7"
-locations_default(#20009,#10000,1,8,1,7)
+tokeninfo(#20008,8,#20001,2,"{")
+#20009=@"loc,{#10000},1,8,1,8"
+locations_default(#20009,#10000,1,8,1,8)
 hasLocation(#20008,#20009)
 #20010=*
-exprs(#20010,0,#20008,0,"constructor")
-hasLocation(#20010,#20009)
-enclosingStmt(#20010,#20003)
-exprContainers(#20010,#20001)
-literals("constructor","constructor",#20010)
-#20011=*
-exprs(#20011,9,#20008,1,"() {}")
-hasLocation(#20011,#20009)
-enclosingStmt(#20011,#20003)
-exprContainers(#20011,#20001)
+tokeninfo(#20010,8,#20001,3,"}")
+#20011=@"loc,{#10000},1,9,1,9"
+locations_default(#20011,#10000,1,9,1,9)
+hasLocation(#20010,#20011)
 #20012=*
-scopes(#20012,1)
-scopenodes(#20011,#20012)
-scopenesting(#20012,#20000)
-#20013=@"var;{arguments};{#20012}"
-variables(#20013,"arguments",#20012)
-isArgumentsObject(#20013)
+tokeninfo(#20012,8,#20001,4,";")
+#20013=@"loc,{#10000},1,10,1,10"
+locations_default(#20013,#10000,1,10,1,10)
+hasLocation(#20012,#20013)
 #20014=*
-stmts(#20014,1,#20011,-2,"{}")
-hasLocation(#20014,#20009)
-stmtContainers(#20014,#20011)
-numlines(#20011,1,0,0)
-isMethod(#20008)
-#20015=*
-lines(#20015,#20001,"!class {};","")
-hasLocation(#20015,#20002)
-numlines(#20001,1,1,0)
+tokeninfo(#20014,0,#20001,5,"")
+#20015=@"loc,{#10000},1,11,1,10"
+locations_default(#20015,#10000,1,11,1,10)
+hasLocation(#20014,#20015)
+toplevels(#20001,0)
+hasLocation(#20001,#20003)
 #20016=*
-tokeninfo(#20016,8,#20001,0,"!")
-#20017=@"loc,{#10000},1,1,1,1"
-locations_default(#20017,#10000,1,1,1,1)
-hasLocation(#20016,#20017)
-#20018=*
-tokeninfo(#20018,7,#20001,1,"class")
-#20019=@"loc,{#10000},1,2,1,6"
-locations_default(#20019,#10000,1,2,1,6)
-hasLocation(#20018,#20019)
-#20020=*
-tokeninfo(#20020,8,#20001,2,"{")
-#20021=@"loc,{#10000},1,8,1,8"
-locations_default(#20021,#10000,1,8,1,8)
-hasLocation(#20020,#20021)
-#20022=*
-tokeninfo(#20022,8,#20001,3,"}")
-#20023=@"loc,{#10000},1,9,1,9"
-locations_default(#20023,#10000,1,9,1,9)
-hasLocation(#20022,#20023)
+stmts(#20016,2,#20001,0,"!class {};")
+hasLocation(#20016,#20003)
+stmtContainers(#20016,#20001)
+#20017=*
+exprs(#20017,18,#20016,0,"!class {}")
+#20018=@"loc,{#10000},1,1,1,9"
+locations_default(#20018,#10000,1,1,1,9)
+hasLocation(#20017,#20018)
+enclosingStmt(#20017,#20016)
+exprContainers(#20017,#20001)
+#20019=*
+exprs(#20019,80,#20017,0,"class {}")
+#20020=@"loc,{#10000},1,2,1,9"
+locations_default(#20020,#10000,1,2,1,9)
+hasLocation(#20019,#20020)
+enclosingStmt(#20019,#20016)
+exprContainers(#20019,#20001)
+#20021=*
+properties(#20021,#20019,2,0,"constructor() {}")
+#20022=@"loc,{#10000},1,8,1,7"
+locations_default(#20022,#10000,1,8,1,7)
+hasLocation(#20021,#20022)
+#20023=*
+exprs(#20023,0,#20021,0,"constructor")
+hasLocation(#20023,#20022)
+enclosingStmt(#20023,#20016)
+exprContainers(#20023,#20001)
+literals("constructor","constructor",#20023)
 #20024=*
-tokeninfo(#20024,8,#20001,4,";")
-#20025=@"loc,{#10000},1,10,1,10"
-locations_default(#20025,#10000,1,10,1,10)
-hasLocation(#20024,#20025)
-#20026=*
-tokeninfo(#20026,0,#20001,5,"")
-#20027=@"loc,{#10000},1,11,1,10"
-locations_default(#20027,#10000,1,11,1,10)
-hasLocation(#20026,#20027)
+exprs(#20024,9,#20021,1,"() {}")
+hasLocation(#20024,#20022)
+enclosingStmt(#20024,#20016)
+exprContainers(#20024,#20001)
+#20025=*
+scopes(#20025,1)
+scopenodes(#20024,#20025)
+scopenesting(#20025,#20000)
+#20026=@"var;{arguments};{#20025}"
+variables(#20026,"arguments",#20025)
+isArgumentsObject(#20026)
+#20027=*
+stmts(#20027,1,#20024,-2,"{}")
+hasLocation(#20027,#20022)
+stmtContainers(#20027,#20024)
+isMethod(#20021)
 #20028=*
 entry_cfg_node(#20028,#20001)
 #20029=@"loc,{#10000},1,1,1,0"
 locations_default(#20029,#10000,1,1,1,0)
 hasLocation(#20028,#20029)
 #20030=*
 exit_cfg_node(#20030,#20001)
-hasLocation(#20030,#20027)
-successor(#20003,#20010)
-successor(#20011,#20008)
+hasLocation(#20030,#20015)
+successor(#20016,#20023)
+successor(#20024,#20021)
 #20031=*
-entry_cfg_node(#20031,#20011)
-hasLocation(#20031,#20009)
+entry_cfg_node(#20031,#20024)
+hasLocation(#20031,#20022)
 #20032=*
-exit_cfg_node(#20032,#20011)
-hasLocation(#20032,#20009)
-successor(#20014,#20032)
-successor(#20031,#20014)
-successor(#20010,#20011)
-successor(#20008,#20006)
-successor(#20006,#20004)
-successor(#20004,#20030)
-successor(#20028,#20003)
+exit_cfg_node(#20032,#20024)
+hasLocation(#20032,#20022)
+successor(#20027,#20032)
+successor(#20031,#20027)
+successor(#20023,#20024)
+successor(#20021,#20019)
+successor(#20019,#20017)
+successor(#20017,#20030)
+successor(#20028,#20016)
 numlines(#10000,1,1,0)
 filetype(#10000,"javascript")