JavaScript: Make autobuilder fail if no JS/TS code was seen.

In particular, the autobuilder will no longer succeed for projects that
contain HTML or YAML files but no JS/TS code. Further down the line,
this prevents LGTM.com from classifying such projects as "JavaScript"
projects.

Author: Max Schaefer

change-notes/1.21/extractor-javascript.md

@@ -5,5 +5,5 @@
 ## Changes to code extraction
 
 * ECMAScript 2019 support is now enabled by default.
-
+* On LGTM, JavaScript extraction for projects that do not contain any JavaScript or TypeScript code will now fail, even if the project contains other file types (such as HTML or YAML) recognized by the JavaScript extractor.
 * YAML files are now extracted by default on LGTM. You can specify exclusion filters in your `lgtm.yml` file to override this behavior.

javascript/extractor/src/com/semmle/js/extractor/AutoBuild.java

@@ -1,26 +1,5 @@
 package com.semmle.js.extractor;
 
-import com.semmle.js.extractor.ExtractorConfig.SourceType;
-import com.semmle.js.extractor.FileExtractor.FileType;
-import com.semmle.js.extractor.trapcache.DefaultTrapCache;
-import com.semmle.js.extractor.trapcache.DummyTrapCache;
-import com.semmle.js.extractor.trapcache.ITrapCache;
-import com.semmle.js.parser.ParsedProject;
-import com.semmle.js.parser.TypeScriptParser;
-import com.semmle.ts.extractor.TypeExtractor;
-import com.semmle.ts.extractor.TypeTable;
-import com.semmle.util.data.StringUtil;
-import com.semmle.util.exception.CatastrophicError;
-import com.semmle.util.exception.Exceptions;
-import com.semmle.util.exception.ResourceError;
-import com.semmle.util.exception.UserError;
-import com.semmle.util.extraction.ExtractorOutputConfig;
-import com.semmle.util.files.FileUtil;
-import com.semmle.util.io.csv.CSVReader;
-import com.semmle.util.language.LegacyLanguage;
-import com.semmle.util.process.Env;
-import com.semmle.util.projectstructure.ProjectLayout;
-import com.semmle.util.trap.TrapWriter;
 import java.io.File;
 import java.io.IOException;
 import java.io.Reader;
@@ -48,6 +27,28 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
+import com.semmle.js.extractor.ExtractorConfig.SourceType;
+import com.semmle.js.extractor.FileExtractor.FileType;
+import com.semmle.js.extractor.trapcache.DefaultTrapCache;
+import com.semmle.js.extractor.trapcache.DummyTrapCache;
+import com.semmle.js.extractor.trapcache.ITrapCache;
+import com.semmle.js.parser.ParsedProject;
+import com.semmle.js.parser.TypeScriptParser;
+import com.semmle.ts.extractor.TypeExtractor;
+import com.semmle.ts.extractor.TypeTable;
+import com.semmle.util.data.StringUtil;
+import com.semmle.util.exception.CatastrophicError;
+import com.semmle.util.exception.Exceptions;
+import com.semmle.util.exception.ResourceError;
+import com.semmle.util.exception.UserError;
+import com.semmle.util.extraction.ExtractorOutputConfig;
+import com.semmle.util.files.FileUtil;
+import com.semmle.util.io.csv.CSVReader;
+import com.semmle.util.language.LegacyLanguage;
+import com.semmle.util.process.Env;
+import com.semmle.util.projectstructure.ProjectLayout;
+import com.semmle.util.trap.TrapWriter;
+
 /**
  * An alternative entry point to the JavaScript extractor.
  *
@@ -194,6 +195,7 @@ public class AutoBuild {
   private final TypeScriptMode typeScriptMode;
   private final String defaultEncoding;
   private ExecutorService threadPool;
+  private volatile boolean seenCode = false;
 
   public AutoBuild() {
     this.LGTM_SRC = toRealPath(getPathFromEnvVar("LGTM_SRC"));
@@ -425,7 +427,7 @@ private boolean addPathPattern(Set<Path> patterns, Path base, String pattern) {
   }
 
   /** Perform extraction. */
-  public void run() throws IOException {
+  public int run() throws IOException {
     startThreadPool();
     try {
       extractSource();
@@ -434,6 +436,11 @@ public void run() throws IOException {
     } finally {
       shutdownThreadPool();
     }
+    if (!seenCode) {
+      warn("No JavaScript or TypeScript code found.");
+      return -1;
+    }
+    return 0;
   }
 
   private void startThreadPool() {
@@ -736,7 +743,9 @@ private void doExtract(FileExtractor extractor, Path file, ExtractorState state)
 
     try {
       long start = logBeginProcess("Extracting " + file);
-      extractor.extract(f, state);
+      Integer loc = extractor.extract(f, state);
+      if (!extractor.getConfig().isExterns() && (loc == null || loc != 0))
+        seenCode = true;
       logEndProcess(start, "Done extracting " + file);
     } catch (Throwable t) {
       System.err.println("Exception while extracting " + file + ".");
@@ -787,7 +796,7 @@ protected void extractXml() throws IOException {
 
   public static void main(String[] args) {
     try {
-      new AutoBuild().run();
+      System.exit(new AutoBuild().run());
     } catch (IOException | UserError | CatastrophicError e) {
       System.err.println(e.toString());
       System.exit(1);

javascript/extractor/src/com/semmle/js/extractor/FileExtractor.java

@@ -384,7 +384,10 @@ public boolean supports(File f) {
     return config.hasFileType() || FileType.forFile(f, config) != null;
   }
 
-  public void extract(File f, ExtractorState state) throws IOException {
+  /**
+   * @return the number of lines of code extracted, or {@code null} if the file was cached
+   */
+  public Integer extract(File f, ExtractorState state) throws IOException {
     // populate source archive
     String source = new WholeIO(config.getDefaultEncoding()).strictread(f);
     outputConfig.getSourceArchive().add(f, source);
@@ -396,7 +399,7 @@ public void extract(File f, ExtractorState state) throws IOException {
     locationManager.emitFileLocation(fileLabel, 0, 0, 0, 0);
 
     // now extract the contents
-    extractContents(f, fileLabel, source, locationManager, state);
+    return extractContents(f, fileLabel, source, locationManager, state);
   }
 
   /**
@@ -420,7 +423,7 @@ public void extract(File f, ExtractorState state) throws IOException {
    * <p>Also note that we support extraction with TRAP writer factories that are not file-backed;
    * obviously, no caching is done in that scenario.
    */
-  private void extractContents(
+  private Integer extractContents(
       File f, Label fileLabel, String source, LocationManager locationManager, ExtractorState state)
       throws IOException {
     TrapWriter trapwriter = locationManager.getTrapWriter();
@@ -440,7 +443,7 @@ private void extractContents(
 
       if (cacheFile.exists()) {
         FileUtil.append(cacheFile, resultFile);
-        return;
+        return null;
       }
 
       // not in the cache yet, so use a caching TRAP writer to
@@ -463,6 +466,7 @@ private void extractContents(
       trapwriter.addTuple("numlines", fileLabel, numLines, linesOfCode, linesOfComments);
       trapwriter.addTuple("filetype", fileLabel, fileType.toString());
       successful = true;
+      return linesOfCode;
     } finally {
       if (!successful && trapwriter instanceof CachingTrapWriter)
         ((CachingTrapWriter) trapwriter).discard();