Merge pull request #8167 from michaelnebel/csharp/extractor-option-compress

C# Extractor Option for specifying compression.

Author: michaelnebel

csharp/codeql-extractor.yml

@@ -16,3 +16,16 @@ file_types:
     extensions:
       - .cs
 legacy_qltest_extraction: true
+options:
+  trap:
+    title: Options pertaining to TRAP.
+    type: object
+    properties:
+      compression:
+        title: Controls compression for the TRAP files written by the extractor.
+        description: >
+          This option is only intended for use in debugging the extractor. Accepted
+          values are 'brotli' (the default, to write brotli-compressed TRAP), 'gzip', and 'none'
+          (to write uncompressed TRAP).
+        type: string
+        pattern: "^(none|gzip|brotli)$"

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs

@@ -46,7 +46,9 @@ public static Options CreateWithEnvironment(string[] arguments)
             var argsList = new List<string>(arguments);
 
             if (!string.IsNullOrEmpty(extractionOptions))
+            {
                 argsList.AddRange(extractionOptions.Split(' '));
+            }
 
             options.ParseArguments(argsList);
             return options;

csharp/extractor/Semmle.Extraction.Tests/Options.cs

@@ -33,6 +33,7 @@ public void DefaultOptions()
             Assert.False(options.ClrTracer);
             Assert.False(options.PDB);
             Assert.False(options.Fast);
+            Assert.Equal(TrapWriter.CompressionMode.Brotli, options.TrapCompression);
         }
 
         [Fact]
@@ -205,5 +206,25 @@ public void ArchiveArguments()
                 File.Delete(file);
             }
         }
+
+        [Fact]
+        public void CompressionTests()
+        {
+            Environment.SetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_TRAP_COMPRESSION", "gzip");
+            options = CSharp.Options.CreateWithEnvironment(Array.Empty<string>());
+            Assert.Equal(TrapWriter.CompressionMode.Gzip, options.TrapCompression);
+
+            Environment.SetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_TRAP_COMPRESSION", "brotli");
+            options = CSharp.Options.CreateWithEnvironment(Array.Empty<string>());
+            Assert.Equal(TrapWriter.CompressionMode.Brotli, options.TrapCompression);
+
+            Environment.SetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_TRAP_COMPRESSION", "none");
+            options = CSharp.Options.CreateWithEnvironment(Array.Empty<string>());
+            Assert.Equal(TrapWriter.CompressionMode.None, options.TrapCompression);
+
+            Environment.SetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_TRAP_COMPRESSION", null);
+            options = CSharp.Options.CreateWithEnvironment(Array.Empty<string>());
+            Assert.Equal(TrapWriter.CompressionMode.Brotli, options.TrapCompression);
+        }
     }
 }

csharp/extractor/Semmle.Extraction/Options.cs

@@ -1,3 +1,4 @@
+using System;
 using Semmle.Util.Logging;
 using Semmle.Util;
 
@@ -49,10 +50,11 @@ public abstract class CommonOptions : ICommandLineOptions
         /// </summary>
         public bool QlTest { get; private set; } = false;
 
+
         /// <summary>
         /// The compression algorithm used for trap files.
         /// </summary>
-        public TrapWriter.CompressionMode TrapCompression { get; set; } = TrapWriter.CompressionMode.Brotli;
+        public TrapWriter.CompressionMode TrapCompression { get; private set; } = TrapWriter.CompressionMode.Brotli;
 
         public virtual bool HandleOption(string key, string value)
         {
@@ -64,6 +66,13 @@ public virtual bool HandleOption(string key, string value)
                 case "verbosity":
                     Verbosity = (Verbosity)int.Parse(value);
                     return true;
+                case "trap_compression":
+                    if (Enum.TryParse<TrapWriter.CompressionMode>(value, true, out var mode))
+                    {
+                        TrapCompression = mode;
+                        return true;
+                    }
+                    return false;
                 default:
                     return false;
             }
@@ -102,9 +111,6 @@ public virtual bool HandleFlag(string flag, bool value)
                 case "qltest":
                     QlTest = value;
                     return true;
-                case "brotli":
-                    TrapCompression = value ? TrapWriter.CompressionMode.Brotli : TrapWriter.CompressionMode.Gzip;
-                    return true;
                 default:
                     return false;
             }

csharp/extractor/Semmle.Util/CommandLineOptions.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 
 namespace Semmle.Util
@@ -39,8 +40,26 @@ public interface ICommandLineOptions
 
     public static class OptionsExtensions
     {
-        public static void ParseArguments(this ICommandLineOptions options, IReadOnlyList<string> arguments)
+        private static string? GetExtractorOption(string name) =>
+            Environment.GetEnvironmentVariable($"CODEQL_EXTRACTOR_CSHARP_OPTION_{name.ToUpper()}");
+
+        private static List<string> GetExtractorOptions()
+        {
+            var extractorOptions = new List<string>();
+
+            var trapCompression = GetExtractorOption("trap_compression");
+            if (!string.IsNullOrEmpty(trapCompression))
+            {
+                extractorOptions.Add($"--trap_compression:{trapCompression}");
+            }
+
+            return extractorOptions;
+        }
+
+        public static void ParseArguments(this ICommandLineOptions options, IReadOnlyList<string> providedArguments)
         {
+            var arguments = GetExtractorOptions();
+            arguments.AddRange(providedArguments);
             for (var i = 0; i < arguments.Count; ++i)
             {
                 var arg = arguments[i];

csharp/ql/src/change-notes/2022-02-23-extractor-option-compression.md

@@ -0,0 +1,5 @@
+---
+category: minorAnalysis
+---
+* The C# extractor now accepts an extractor option `trap.compression`, which is used to decide the compression format for TRAP files. The legal values are `brotli` (default), `gzip` or `none`.
+The option is added via `codeql database create --language=csharp -Otrap.compression=value ...`.