JavaScript: Extract tokens and comments before AST.

This allows us to discard token/comment information sooner, thereby reducing heap pressure for very large files.

Author: Max Schaefer

javascript/extractor/src/com/semmle/js/extractor/JSExtractor.java

@@ -94,12 +94,14 @@ public Pair<Label, LoCInfo> extract(TextualExtractor textualExtractor,
 			ASTExtractor scriptExtractor = new ASTExtractor(lexicalExtractor, scopeManager);
 			toplevelLabel = scriptExtractor.getToplevelLabel();
 
-			scriptExtractor.extract(ast, platform, sourceType, toplevelKind);
 			lexicalExtractor.extractComments(toplevelLabel);
 			loc = lexicalExtractor.extractLines(parserRes.getSource(), toplevelLabel);
 			lexicalExtractor.extractTokens(toplevelLabel);
-			new CFGExtractor(scriptExtractor).extract(ast);
 			new JSDocExtractor(textualExtractor).extract(lexicalExtractor.getComments());
+			lexicalExtractor.purge();
+
+			scriptExtractor.extract(ast, platform, sourceType, toplevelKind);
+			new CFGExtractor(scriptExtractor).extract(ast);
 		} else {
 			lexicalExtractor = new LexicalExtractor(textualExtractor, new ArrayList<Token>(), new ArrayList<Comment>());
 			ASTExtractor scriptExtractor = new ASTExtractor(lexicalExtractor, null);

javascript/extractor/src/com/semmle/js/extractor/LexicalExtractor.java

@@ -173,4 +173,12 @@ public void extractComments(Label toplevelKey) {
 	public String mkToString(SourceElement nd) {
 		return textualExtractor.mkToString(nd);
 	}
+
+	/**
+	 * Purge token and comments information to reduce heap pressure.
+	 */
+	public void purge() {
+		this.tokens.clear();
+		this.comments.clear();
+	}
 }