Skip to content

Commit 57afde0

Browse files
semmle-qlcisemmle-qlci
authored
Merge pull request #1872 from esben-semmle/js/extraction_metrics
Approved by xiemaisi
2 parents 77d7db3 + 2a22471 commit 57afde0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+3442
-155
lines changed

‎javascript/extractor/src/com/semmle/js/ast/DefaultVisitor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import com.semmle.ts.ast.InterfaceTypeExpr;
3636
import com.semmle.ts.ast.IntersectionTypeExpr;
3737
import com.semmle.ts.ast.IsTypeExpr;
38-
import com.semmle.ts.ast.UnaryTypeExpr;
3938
import com.semmle.ts.ast.KeywordTypeExpr;
4039
import com.semmle.ts.ast.MappedTypeExpr;
4140
import com.semmle.ts.ast.NamespaceDeclaration;
@@ -49,6 +48,7 @@
4948
import com.semmle.ts.ast.TypeExpression;
5049
import com.semmle.ts.ast.TypeParameter;
5150
import com.semmle.ts.ast.TypeofTypeExpr;
51+
import com.semmle.ts.ast.UnaryTypeExpr;
5252
import com.semmle.ts.ast.UnionTypeExpr;
5353
import com.semmle.util.exception.CatastrophicError;
5454

‎javascript/extractor/src/com/semmle/js/ast/Visitor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import com.semmle.ts.ast.InterfaceTypeExpr;
3232
import com.semmle.ts.ast.IntersectionTypeExpr;
3333
import com.semmle.ts.ast.IsTypeExpr;
34-
import com.semmle.ts.ast.UnaryTypeExpr;
3534
import com.semmle.ts.ast.KeywordTypeExpr;
3635
import com.semmle.ts.ast.MappedTypeExpr;
3736
import com.semmle.ts.ast.NamespaceDeclaration;
@@ -44,6 +43,7 @@
4443
import com.semmle.ts.ast.TypeAssertion;
4544
import com.semmle.ts.ast.TypeParameter;
4645
import com.semmle.ts.ast.TypeofTypeExpr;
46+
import com.semmle.ts.ast.UnaryTypeExpr;
4747
import com.semmle.ts.ast.UnionTypeExpr;
4848

4949
/**

‎javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,5 @@
11
package com.semmle.js.extractor;
22

3-
import java.util.ArrayList;
4-
import java.util.Collections;
5-
import java.util.List;
6-
import java.util.Set;
7-
import java.util.Stack;
8-
93
import com.semmle.js.ast.AClass;
104
import com.semmle.js.ast.AFunction;
115
import com.semmle.js.ast.AFunctionExpression;
@@ -104,6 +98,7 @@
10498
import com.semmle.js.ast.jsx.JSXNamespacedName;
10599
import com.semmle.js.ast.jsx.JSXOpeningElement;
106100
import com.semmle.js.ast.jsx.JSXSpreadAttribute;
101+
import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
107102
import com.semmle.js.extractor.ExtractorConfig.Platform;
108103
import com.semmle.js.extractor.ExtractorConfig.SourceType;
109104
import com.semmle.js.extractor.ScopeManager.DeclKind;
@@ -149,6 +144,11 @@
149144
import com.semmle.util.collections.CollectionUtil;
150145
import com.semmle.util.trap.TrapWriter;
151146
import com.semmle.util.trap.TrapWriter.Label;
147+
import java.util.ArrayList;
148+
import java.util.Collections;
149+
import java.util.List;
150+
import java.util.Set;
151+
import java.util.Stack;
152152

153153
/** Extractor for AST-based information; invoked by the {@link JSExtractor}. */
154154
public class ASTExtractor {
@@ -193,6 +193,10 @@ public ScopeManager getScopeManager() {
193193
return scopeManager;
194194
}
195195

196+
public ExtractionMetrics getMetrics() {
197+
return lexicalExtractor.getMetrics();
198+
}
199+
196200
/**
197201
* The binding semantics for an identifier.
198202
*
@@ -1946,9 +1950,11 @@ public Label visit(XMLDotDotExpression nd, Context c) {
19461950
}
19471951

19481952
public void extract(Node root, Platform platform, SourceType sourceType, int toplevelKind) {
1953+
lexicalExtractor.getMetrics().startPhase(ExtractionPhase.ASTExtractor_extract);
19491954
trapwriter.addTuple("toplevels", toplevelLabel, toplevelKind);
19501955
locationManager.emitNodeLocation(root, toplevelLabel);
19511956

19521957
root.accept(new V(platform, sourceType), null);
1958+
lexicalExtractor.getMetrics().stopPhase(ExtractionPhase.ASTExtractor_extract);
19531959
}
19541960
}

‎javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java‎

Lines changed: 29 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,26 @@
11
package com.semmle.js.extractor;
22

3+
import com.semmle.js.extractor.ExtractorConfig.SourceType;
4+
import com.semmle.js.extractor.FileExtractor.FileType;
5+
import com.semmle.js.extractor.trapcache.DefaultTrapCache;
6+
import com.semmle.js.extractor.trapcache.DummyTrapCache;
7+
import com.semmle.js.extractor.trapcache.ITrapCache;
8+
import com.semmle.js.parser.ParsedProject;
9+
import com.semmle.js.parser.TypeScriptParser;
10+
import com.semmle.ts.extractor.TypeExtractor;
11+
import com.semmle.ts.extractor.TypeTable;
12+
import com.semmle.util.data.StringUtil;
13+
import com.semmle.util.exception.CatastrophicError;
14+
import com.semmle.util.exception.Exceptions;
15+
import com.semmle.util.exception.ResourceError;
16+
import com.semmle.util.exception.UserError;
17+
import com.semmle.util.extraction.ExtractorOutputConfig;
18+
import com.semmle.util.files.FileUtil;
19+
import com.semmle.util.io.csv.CSVReader;
20+
import com.semmle.util.language.LegacyLanguage;
21+
import com.semmle.util.process.Env;
22+
import com.semmle.util.projectstructure.ProjectLayout;
23+
import com.semmle.util.trap.TrapWriter;
324
import java.io.File;
425
import java.io.IOException;
526
import java.io.Reader;
@@ -27,28 +48,6 @@
2748
import java.util.concurrent.TimeUnit;
2849
import java.util.stream.Stream;
2950

30-
import com.semmle.js.extractor.ExtractorConfig.SourceType;
31-
import com.semmle.js.extractor.FileExtractor.FileType;
32-
import com.semmle.js.extractor.trapcache.DefaultTrapCache;
33-
import com.semmle.js.extractor.trapcache.DummyTrapCache;
34-
import com.semmle.js.extractor.trapcache.ITrapCache;
35-
import com.semmle.js.parser.ParsedProject;
36-
import com.semmle.js.parser.TypeScriptParser;
37-
import com.semmle.ts.extractor.TypeExtractor;
38-
import com.semmle.ts.extractor.TypeTable;
39-
import com.semmle.util.data.StringUtil;
40-
import com.semmle.util.exception.CatastrophicError;
41-
import com.semmle.util.exception.Exceptions;
42-
import com.semmle.util.exception.ResourceError;
43-
import com.semmle.util.exception.UserError;
44-
import com.semmle.util.extraction.ExtractorOutputConfig;
45-
import com.semmle.util.files.FileUtil;
46-
import com.semmle.util.io.csv.CSVReader;
47-
import com.semmle.util.language.LegacyLanguage;
48-
import com.semmle.util.process.Env;
49-
import com.semmle.util.projectstructure.ProjectLayout;
50-
import com.semmle.util.trap.TrapWriter;
51-
5251
/**
5352
* An alternative entry point to the JavaScript extractor.
5453
*
@@ -71,8 +70,8 @@
7170
* patterns that can be used to refine the list of files to include and exclude
7271
* <li><code>LGTM_INDEX_TYPESCRIPT</code>: whether to extract TypeScript
7372
* <li><code>LGTM_INDEX_FILETYPES</code>: a newline-separated list of ".extension:filetype" pairs
74-
* specifying which {@link FileType} to use for the given extension; the additional file
75-
* type <code>XML</code> is also supported
73+
* specifying which {@link FileType} to use for the given extension; the additional file type
74+
* <code>XML</code> is also supported
7675
* <li><code>LGTM_INDEX_XML_MODE</code>: whether to extract XML files
7776
* <li><code>LGTM_THREADS</code>: the maximum number of files to extract in parallel
7877
* <li><code>LGTM_TRAP_CACHE</code>: the path of a directory to use for trap caching
@@ -166,8 +165,8 @@
166165
* <p>If <code>LGTM_INDEX_XML_MODE</code> is set to <code>ALL</code>, then all files with extension
167166
* <code>.xml</code> under <code>LGTM_SRC</code> are extracted as XML (in addition to any files
168167
* whose file type is specified to be <code>XML</code> via <code>LGTM_INDEX_SOURCE_TYPE</code>).
169-
* Currently XML extraction does not respect inclusion and exclusion filters, but this is a bug,
170-
* not a feature, and hence will change eventually.
168+
* Currently XML extraction does not respect inclusion and exclusion filters, but this is a bug, not
169+
* a feature, and hence will change eventually.
171170
*
172171
* <p>Note that all these customisations only apply to <code>LGTM_SRC</code>. Extraction of externs
173172
* is not customisable.
@@ -288,8 +287,7 @@ private void setupFileTypes() {
288287
try {
289288
fileType = StringUtil.uc(fileType);
290289
if ("XML".equals(fileType)) {
291-
if (extension.length() < 2)
292-
throw new UserError("Invalid extension '" + extension + "'.");
290+
if (extension.length() < 2) throw new UserError("Invalid extension '" + extension + "'.");
293291
xmlExtensions.add(extension.substring(1));
294292
} else {
295293
fileTypes.put(extension, FileType.valueOf(fileType));
@@ -304,8 +302,7 @@ private void setupFileTypes() {
304302
private void setupXmlMode() {
305303
String xmlMode = getEnvVar("LGTM_INDEX_XML_MODE", "DISABLED");
306304
xmlMode = StringUtil.uc(xmlMode.trim());
307-
if ("ALL".equals(xmlMode))
308-
xmlExtensions.add("xml");
305+
if ("ALL".equals(xmlMode)) xmlExtensions.add("xml");
309306
else if (!"DISABLED".equals(xmlMode))
310307
throw new UserError("Invalid XML mode '" + xmlMode + "' (should be either ALL or DISABLED).");
311308
}
@@ -744,8 +741,7 @@ private void doExtract(FileExtractor extractor, Path file, ExtractorState state)
744741
try {
745742
long start = logBeginProcess("Extracting " + file);
746743
Integer loc = extractor.extract(f, state);
747-
if (!extractor.getConfig().isExterns() && (loc == null || loc != 0))
748-
seenCode = true;
744+
if (!extractor.getConfig().isExterns() && (loc == null || loc != 0)) seenCode = true;
749745
logEndProcess(start, "Done extracting " + file);
750746
} catch (Throwable t) {
751747
System.err.println("Exception while extracting " + file + ".");
@@ -776,8 +772,7 @@ public Set<String> getXmlExtensions() {
776772
}
777773

778774
protected void extractXml() throws IOException {
779-
if (xmlExtensions.isEmpty())
780-
return;
775+
if (xmlExtensions.isEmpty()) return;
781776
List<String> cmd = new ArrayList<>();
782777
cmd.add("odasa");
783778
cmd.add("index");

‎javascript/extractor/src/com/semmle/js/extractor/CFGExtractor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@
9393
import com.semmle.js.ast.jsx.JSXNamespacedName;
9494
import com.semmle.js.ast.jsx.JSXOpeningElement;
9595
import com.semmle.js.ast.jsx.JSXSpreadAttribute;
96+
import com.semmle.js.extractor.ExtractionMetrics.ExtractionPhase;
9697
import com.semmle.ts.ast.DecoratorList;
9798
import com.semmle.ts.ast.EnumDeclaration;
9899
import com.semmle.ts.ast.EnumMember;
@@ -171,11 +172,13 @@ public class CFGExtractor {
171172
private final TrapWriter trapwriter;
172173
private final Label toplevelLabel;
173174
private final LocationManager locationManager;
175+
private final ExtractionMetrics metrics;
174176

175177
public CFGExtractor(ASTExtractor astExtractor) {
176178
this.trapwriter = astExtractor.getTrapwriter();
177179
this.toplevelLabel = astExtractor.getToplevelLabel();
178180
this.locationManager = astExtractor.getLocationManager();
181+
this.metrics = astExtractor.getMetrics();
179182
}
180183

181184
@SuppressWarnings("unchecked")
@@ -1955,6 +1958,8 @@ public Void visit(XMLDotDotExpression nd, SuccessorInfo c) {
19551958
}
19561959

19571960
public void extract(Node nd) {
1961+
metrics.startPhase(ExtractionPhase.CFGExtractor_extract);
19581962
nd.accept(new V(), new SimpleSuccessorInfo(null));
1963+
metrics.stopPhase(ExtractionPhase.CFGExtractor_extract);
19591964
}
19601965
}

0 commit comments

Comments
 (0)