Skip to content

Commit 30a9078

Browse files
semmle-qlcisemmle-qlci
authored
Merge pull request #2193 from max-schaefer/js/autobuilder-exclude-node_modules
Approved by asger-semmle
2 parents 8839bdd + d4b9beb commit 30a9078

File tree

3 files changed

+80
-31
lines changed

3 files changed

+80
-31
lines changed

‎change-notes/1.23/extractor-javascript.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,17 @@
55
## Changes to code extraction
66

77
* Asynchronous generator methods are now parsed correctly and no longer cause a spurious syntax error.
8+
* Files in `node_modules` and `bower_components` folders are no longer extracted by default. If you still want to extract files from these folders, you can add the following filters to your `lgtm.yml` file (or add them to existing filters):
9+
10+
```yaml
11+
extraction:
12+
javascript:
13+
index:
14+
filters:
15+
- include: "**/node_modules"
16+
- include: "**/bower_components"
17+
```
18+
819
* Recognition of CommonJS modules has improved. As a result, some files that were previously extracted as
920
global scripts are now extracted as modules.
1021
* Top-level `await` is now supported.

‎javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,10 @@ private void setupFilters() {
387387
patterns.add("-**/*.min.js");
388388
patterns.add("-**/*-min.js");
389389

390+
// exclude `node_modules` and `bower_components`
391+
patterns.add("-**/node_modules");
392+
patterns.add("-**/bower_components");
393+
390394
String base = LGTM_SRC.toString().replace('\\', '/');
391395
// process `$LGTM_INDEX_FILTERS`
392396
for (String pattern : Main.NEWLINE.split(getEnvVar("LGTM_INDEX_FILTERS", ""))) {

‎javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java‎

Lines changed: 65 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ public void basicTest() throws IOException {
160160
addFile(false, LGTM_SRC, "tst.json");
161161
addFile(true, LGTM_SRC, "package.json");
162162
addFile(true, LGTM_SRC, ".eslintrc.yml");
163-
addFile(true, LGTM_SRC, "node_modules", "leftpad", "index.js");
163+
addFile(true, LGTM_SRC, "vendor", "leftpad", "index.js");
164164
runTest();
165165
}
166166

@@ -184,72 +184,72 @@ public void typescriptWrongConfig() throws IOException {
184184
public void includeFile() throws IOException {
185185
envVars.put("LGTM_INDEX_INCLUDE", "tst.js");
186186
addFile(true, LGTM_SRC, "tst.js");
187-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "index.js");
187+
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
188188
runTest();
189189
}
190190

191191
@Test
192192
public void excludeFile() throws IOException {
193-
envVars.put("LGTM_INDEX_EXCLUDE", "node_modules/leftpad/index.js");
193+
envVars.put("LGTM_INDEX_EXCLUDE", "vendor/leftpad/index.js");
194194
addFile(true, LGTM_SRC, "tst.js");
195-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "index.js");
195+
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
196196
runTest();
197197
}
198198

199199
@Test
200200
public void excludeFolderByPattern() throws IOException {
201-
envVars.put("LGTM_INDEX_FILTERS", "exclude:**/node_modules");
201+
envVars.put("LGTM_INDEX_FILTERS", "exclude:**/vendor");
202202
addFile(true, LGTM_SRC, "tst.js");
203-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "index.js");
203+
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
204204
runTest();
205205
}
206206

207207
@Test
208208
public void excludeFolderByPattern2() throws IOException {
209-
envVars.put("LGTM_INDEX_FILTERS", "exclude:*/**/node_modules");
209+
envVars.put("LGTM_INDEX_FILTERS", "exclude:*/**/vendor");
210210
addFile(true, LGTM_SRC, "tst.js");
211-
addFile(true, LGTM_SRC, "node_modules", "dep", "index.js");
212-
addFile(false, LGTM_SRC, "node_modules", "dep", "node_modules", "depdep", "index.js");
211+
addFile(true, LGTM_SRC, "vendor", "dep", "index.js");
212+
addFile(false, LGTM_SRC, "vendor", "dep", "vendor", "depdep", "index.js");
213213
runTest();
214214
}
215215

216216
@Test
217217
public void excludeFolderByPattern3() throws IOException {
218-
envVars.put("LGTM_INDEX_FILTERS", "exclude:**/node_modules\n");
218+
envVars.put("LGTM_INDEX_FILTERS", "exclude:**/vendor\n");
219219
addFile(true, LGTM_SRC, "tst.js");
220-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "index.js");
220+
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
221221
runTest();
222222
}
223223

224224
@Test
225225
public void excludeFolderByPatterns() throws IOException {
226-
envVars.put("LGTM_INDEX_FILTERS", "exclude:foo\nexclude:**/node_modules");
226+
envVars.put("LGTM_INDEX_FILTERS", "exclude:foo\nexclude:**/vendor");
227227
addFile(true, LGTM_SRC, "tst.js");
228-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "index.js");
228+
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
229229
runTest();
230230
}
231231

232232
@Test
233233
public void excludeFolderByName() throws IOException {
234-
envVars.put("LGTM_INDEX_EXCLUDE", "node_modules");
234+
envVars.put("LGTM_INDEX_EXCLUDE", "vendor");
235235
addFile(true, LGTM_SRC, "tst.js");
236-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "index.js");
236+
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
237237
runTest();
238238
}
239239

240240
@Test
241241
public void excludeFolderByName2() throws IOException {
242-
envVars.put("LGTM_INDEX_EXCLUDE", "node_modules\n");
242+
envVars.put("LGTM_INDEX_EXCLUDE", "vendor\n");
243243
addFile(true, LGTM_SRC, "tst.js");
244-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "index.js");
244+
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
245245
runTest();
246246
}
247247

248248
@Test
249249
public void excludeFolderByName3() throws IOException {
250-
envVars.put("LGTM_INDEX_EXCLUDE", "./node_modules\n");
250+
envVars.put("LGTM_INDEX_EXCLUDE", "./vendor\n");
251251
addFile(true, LGTM_SRC, "tst.js");
252-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "index.js");
252+
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
253253
runTest();
254254
}
255255

@@ -258,8 +258,8 @@ public void excludeByExtension() throws IOException {
258258
envVars.put("LGTM_INDEX_FILTERS", "exclude:**/*.js");
259259
addFile(false, LGTM_SRC, "tst.js");
260260
addFile(true, LGTM_SRC, "tst.html");
261-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "index.js");
262-
addFile(true, LGTM_SRC, "node_modules", "leftpad", "index.html");
261+
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
262+
addFile(true, LGTM_SRC, "vendor", "leftpad", "index.html");
263263
runTest();
264264
}
265265

@@ -268,7 +268,7 @@ public void includeByExtension() throws IOException {
268268
envVars.put("LGTM_INDEX_FILTERS", "include:**/*.json");
269269
addFile(true, LGTM_SRC, "tst.js");
270270
addFile(true, LGTM_SRC, "tst.json");
271-
addFile(true, LGTM_SRC, "node_modules", "leftpad", "tst.json");
271+
addFile(true, LGTM_SRC, "vendor", "leftpad", "tst.json");
272272
runTest();
273273
}
274274

@@ -277,16 +277,16 @@ public void includeByExtensionInRootOnly() throws IOException {
277277
envVars.put("LGTM_INDEX_FILTERS", "include:*.json");
278278
addFile(true, LGTM_SRC, "tst.js");
279279
addFile(true, LGTM_SRC, "tst.json");
280-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "tst.json");
280+
addFile(false, LGTM_SRC, "vendor", "leftpad", "tst.json");
281281
runTest();
282282
}
283283

284284
@Test
285285
public void includeAndExclude() throws IOException {
286-
envVars.put("LGTM_INDEX_FILTERS", "include:**/*.json\n" + "exclude:**/node_modules");
286+
envVars.put("LGTM_INDEX_FILTERS", "include:**/*.json\n" + "exclude:**/vendor");
287287
addFile(true, LGTM_SRC, "tst.js");
288288
addFile(true, LGTM_SRC, "tst.json");
289-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "tst.json");
289+
addFile(false, LGTM_SRC, "vendor", "leftpad", "tst.json");
290290
runTest();
291291
}
292292

@@ -295,15 +295,15 @@ public void excludeByClassification() throws IOException {
295295
Path repositoryFolders = Files.createFile(SEMMLE_DIST.resolve("repositoryFolders.csv"));
296296
List<String> csvLines = new ArrayList<>();
297297
csvLines.add("classification,path");
298-
csvLines.add("thirdparty," + LGTM_SRC.resolve("node_modules"));
298+
csvLines.add("thirdparty," + LGTM_SRC.resolve("vendor"));
299299
csvLines.add("external," + LGTM_SRC.resolve("foo").resolve("bar").toUri());
300300
csvLines.add("metadata," + LGTM_SRC.resolve(".git"));
301301
Files.write(repositoryFolders, csvLines, StandardCharsets.UTF_8);
302302
envVars.put("LGTM_REPOSITORY_FOLDERS_CSV", repositoryFolders.toString());
303303
addFile(true, LGTM_SRC, "tst.js");
304304
addFile(false, LGTM_SRC, "foo", "bar", "tst.js");
305305
addFile(false, LGTM_SRC, ".git", "tst.js");
306-
addFile(true, LGTM_SRC, "node_modules", "leftpad", "tst.js");
306+
addFile(true, LGTM_SRC, "vendor", "leftpad", "tst.js");
307307
runTest();
308308
}
309309

@@ -324,7 +324,7 @@ public void includeImplicitlyExcluded() throws IOException {
324324
Path repositoryFolders = Files.createFile(SEMMLE_DIST.resolve("repositoryFolders.csv"));
325325
List<String> csvLines = new ArrayList<>();
326326
csvLines.add("classification,path");
327-
csvLines.add("thirdparty," + LGTM_SRC.resolve("node_modules"));
327+
csvLines.add("thirdparty," + LGTM_SRC.resolve("vendor"));
328328
csvLines.add("external," + LGTM_SRC.resolve("foo").resolve("bar"));
329329
csvLines.add("metadata," + LGTM_SRC.resolve(".git"));
330330
Files.write(repositoryFolders, csvLines, StandardCharsets.UTF_8);
@@ -333,7 +333,7 @@ public void includeImplicitlyExcluded() throws IOException {
333333
addFile(true, LGTM_SRC, "tst.js");
334334
addFile(true, LGTM_SRC, "foo", "bar", "tst.js");
335335
addFile(false, LGTM_SRC, ".git", "tst.js");
336-
addFile(true, LGTM_SRC, "node_modules", "leftpad", "tst.js");
336+
addFile(true, LGTM_SRC, "vendor", "leftpad", "tst.js");
337337
runTest();
338338
}
339339

@@ -453,13 +453,13 @@ public void noTypescriptExtraction() throws IOException {
453453
@Test
454454
public void includeNonExistentFile() throws IOException {
455455
envVars.put("LGTM_INDEX_INCLUDE", "tst.js");
456-
addFile(false, LGTM_SRC, "node_modules", "leftpad", "index.js");
456+
addFile(false, LGTM_SRC, "vendor", "leftpad", "index.js");
457457
runTest();
458458
}
459459

460460
@Test
461461
public void excludeNonExistentFile() throws IOException {
462-
envVars.put("LGTM_INDEX_EXCLUDE", "node_modules/leftpad/index.js");
462+
envVars.put("LGTM_INDEX_EXCLUDE", "vendor/leftpad/index.js");
463463
addFile(true, LGTM_SRC, "tst.js");
464464
runTest();
465465
}
@@ -483,6 +483,40 @@ public void minifiedFilesCanBeReIncluded() throws IOException {
483483
runTest();
484484
}
485485

486+
@Test
487+
public void nodeModulesAreExcluded() throws IOException {
488+
addFile(true, LGTM_SRC, "index.js");
489+
addFile(false, LGTM_SRC, "node_modules", "dep", "main.js");
490+
addFile(false, LGTM_SRC, "node_modules", "dep", "node_modules", "leftpad", "index.js");
491+
runTest();
492+
}
493+
494+
@Test
495+
public void nodeModulesCanBeReincluded() throws IOException {
496+
envVars.put("LGTM_INDEX_FILTERS", "include:**/node_modules");
497+
addFile(true, LGTM_SRC, "index.js");
498+
addFile(true, LGTM_SRC, "node_modules", "dep", "main.js");
499+
addFile(true, LGTM_SRC, "node_modules", "dep", "node_modules", "leftpad", "index.js");
500+
runTest();
501+
}
502+
503+
@Test
504+
public void bowerComponentsAreExcluded() throws IOException {
505+
addFile(true, LGTM_SRC, "index.js");
506+
addFile(false, LGTM_SRC, "bower_components", "dep", "main.js");
507+
addFile(false, LGTM_SRC, "bower_components", "dep", "bower_components", "leftpad", "index.js");
508+
runTest();
509+
}
510+
511+
@Test
512+
public void bowerComponentsCanBeReincluded() throws IOException {
513+
envVars.put("LGTM_INDEX_FILTERS", "include:**/bower_components");
514+
addFile(true, LGTM_SRC, "index.js");
515+
addFile(true, LGTM_SRC, "bower_components", "dep", "main.js");
516+
addFile(true, LGTM_SRC, "bower_components", "dep", "bower_components", "leftpad", "index.js");
517+
runTest();
518+
}
519+
486520
@Test
487521
public void customExtensions() throws IOException {
488522
envVars.put("LGTM_INDEX_FILETYPES", ".jsm:js\n.soy:html");

0 commit comments

Comments
 (0)