JavaScript: Teach autobuilder not to extract node_modules and bower_components folders.

Author: Max Schaefer

change-notes/1.23/extractor-javascript.md

@@ -5,6 +5,17 @@
 ## Changes to code extraction
 
 * Asynchronous generator methods are now parsed correctly and no longer cause a spurious syntax error.
+* Files in `node_modules` and `bower_components` folders are no longer extracted by default. If you still want to extract files from these folders, you can add the following filters to your `lgtm.yml` file (or add them to existing filters):
+
+```yaml
+extraction:
+  javascript:
+    index:
+      filters:
+        - include: "**/node_modules"
+        - include: "**/bower_components"
+```
+
 * Recognition of CommonJS modules has improved. As a result, some files that were previously extracted as
   global scripts are now extracted as modules.
 * Top-level `await` is now supported.

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -387,6 +387,10 @@ private void setupFilters() {
     patterns.add("-**/*.min.js");
     patterns.add("-**/*-min.js");
 
+    // exclude `node_modules` and `bower_components`
+    patterns.add("-**/node_modules");
+    patterns.add("-**/bower_components");
+
     String base = LGTM_SRC.toString().replace('\\', '/');
     // process `$LGTM_INDEX_FILTERS`
     for (String pattern : Main.NEWLINE.split(getEnvVar("LGTM_INDEX_FILTERS", ""))) {

javascript/extractor/src/com/semmle/js/extractor/test/AutoBuildTests.java

@@ -483,6 +483,40 @@ public void minifiedFilesCanBeReIncluded() throws IOException {
     runTest();
   }
 
+  @Test
+  public void nodeModulesAreExcluded() throws IOException {
+    addFile(true, LGTM_SRC, "index.js");
+    addFile(false, LGTM_SRC, "node_modules", "dep", "main.js");
+    addFile(false, LGTM_SRC, "node_modules", "dep", "node_modules", "leftpad", "index.js");
+    runTest();
+  }
+
+  @Test
+  public void nodeModulesCanBeReincluded() throws IOException {
+    envVars.put("LGTM_INDEX_FILTERS", "include:**/node_modules");
+    addFile(true, LGTM_SRC, "index.js");
+    addFile(true, LGTM_SRC, "node_modules", "dep", "main.js");
+    addFile(true, LGTM_SRC, "node_modules", "dep", "node_modules", "leftpad", "index.js");
+    runTest();
+  }
+
+  @Test
+  public void bowerComponentsAreExcluded() throws IOException {
+    addFile(true, LGTM_SRC, "index.js");
+    addFile(false, LGTM_SRC, "bower_components", "dep", "main.js");
+    addFile(false, LGTM_SRC, "bower_components", "dep", "bower_components", "leftpad", "index.js");
+    runTest();
+  }
+
+  @Test
+  public void bowerComponentsCanBeReincluded() throws IOException {
+    envVars.put("LGTM_INDEX_FILTERS", "include:**/bower_components");
+    addFile(true, LGTM_SRC, "index.js");
+    addFile(true, LGTM_SRC, "bower_components", "dep", "main.js");
+    addFile(true, LGTM_SRC, "bower_components", "dep", "bower_components", "leftpad", "index.js");
+    runTest();
+  }
+
   @Test
   public void customExtensions() throws IOException {
     envVars.put("LGTM_INDEX_FILETYPES", ".jsm:js\n.soy:html");