Merge pull request #16631 from hvitved/tree-sitter/multi-file-lists

Tree-sitter: Allow for multiple file lists in simple extractor

Author: hvitved

ql/extractor/src/extractor.rs

@@ -53,7 +53,7 @@ pub fn run(options: Options) -> std::io::Result<()> {
         trap_dir: options.output_dir,
         trap_compression: trap::Compression::from_env("CODEQL_QL_TRAP_COMPRESSION"),
         source_archive_dir: options.source_archive_dir,
-        file_list: options.file_list,
+        file_lists: vec![options.file_list],
     };
 
     extractor.run()

shared/tree-sitter-extractor/src/extractor/simple.rs

@@ -20,7 +20,7 @@ pub struct Extractor {
     pub languages: Vec<LanguageSpec>,
     pub trap_dir: PathBuf,
     pub source_archive_dir: PathBuf,
-    pub file_list: PathBuf,
+    pub file_lists: Vec<PathBuf>,
     // Typically constructed via `trap::Compression::from_env`.
     // This allow us to report the error using our diagnostics system
     // without exposing it to consumers.
@@ -74,7 +74,14 @@ impl Extractor {
             .build_global()
             .unwrap();
 
-        let file_list = File::open(&self.file_list)?;
+        let file_lists: Vec<File> = self
+            .file_lists
+            .iter()
+            .map(|file_list| {
+                File::open(file_list)
+                    .unwrap_or_else(|_| panic!("Unable to open file list at {:?}", file_list))
+            })
+            .collect();
 
         let mut schemas = vec![];
         for lang in &self.languages {
@@ -103,8 +110,10 @@ impl Extractor {
             )
         };
 
-        let lines: std::io::Result<Vec<String>> =
-            std::io::BufReader::new(file_list).lines().collect();
+        let lines: std::io::Result<Vec<String>> = file_lists
+            .iter()
+            .flat_map(|file_list| std::io::BufReader::new(file_list).lines())
+            .collect();
         let lines = lines?;
 
         lines

shared/tree-sitter-extractor/tests/integration_test.rs

@@ -30,7 +30,7 @@ fn simple_extractor() {
         languages: vec![language],
         trap_dir,
         source_archive_dir,
-        file_list,
+        file_lists: vec![file_list],
         trap_compression: Ok(trap::Compression::Gzip),
     };
 

shared/tree-sitter-extractor/tests/multiple_languages.rs

@@ -39,7 +39,7 @@ fn multiple_language_extractor() {
         languages: vec![lang_ql, lang_json],
         trap_dir,
         source_archive_dir,
-        file_list,
+        file_lists: vec![file_list],
         trap_compression: Ok(trap::Compression::Gzip),
     };