Merge branch 'main' into js/shared-dataflow-merge-main

Author: asgerf

.github/workflows/buildifier.yml

@@ -24,5 +24,5 @@ jobs:
           extra_args: >
             buildifier --all-files 2>&1 ||
             (
-              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel:buildifier"; exit 1
+              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel/buildifier"; exit 1
             )

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         name: Format bazel files
         files: \.(bazel|bzl)
         language: system
-        entry: bazel run //misc/bazel:buildifier
+        entry: bazel run //misc/bazel/buildifier
         pass_filenames: false
 
 #      DISABLED: can be enabled by copying this config and installing `pre-commit` with `--config` on the copy

cpp/ql/test/library-tests/dataflow/taint-tests/taint.cpp

@@ -450,7 +450,7 @@ void test_qualifiers()
 	b.member = source();
 	sink(b); // $ ir MISSING: ast
 	sink(b.member); // $ ast,ir
-	sink(b.getMember()); // $ ir MISSING: ast
+	sink(b.getMember()); // $  MISSING: ir ast
 
 	c = new MyClass2(0);
 

cpp/ql/test/library-tests/dataflow/taint-tests/vector.cpp

@@ -115,8 +115,8 @@ void test_vector_swap() {
 	v3.swap(v4);
 
 	sink(v1);
-	sink(v2); // $ ir MISSING:ast
-	sink(v3); // $ ir MISSING:ast
+	sink(v2); // $ MISSING:ir ast
+	sink(v3); // $ MISSING:ir ast
 	sink(v4);
 }
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using Microsoft.CodeAnalysis;
+using Semmle.Extraction.Entities;
 using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
@@ -89,13 +90,21 @@ public void PopulatePerformance(PerformanceMetrics p)
             trapFile.compilation_finished(this, (float)p.Total.Cpu.TotalSeconds, (float)p.Total.Elapsed.TotalSeconds);
         }
 
+        public void PopulateAggregatedMessages()
+        {
+            ExtractionMessage.groupedMessageCounts.ForEach(pair =>
+            {
+                Context.TrapWriter.Writer.compilation_info(this, $"Extractor message count for group '{pair.Key}'", pair.Value.ToString());
+            });
+        }
+
         public override void WriteId(EscapingTextWriter trapFile)
         {
             trapFile.Write(hashCode);
             trapFile.Write(";compilation");
         }
 
-        public override Location ReportingLocation => throw new NotImplementedException();
+        public override Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException();
 
         public override bool NeedsPopulation => Context.IsAssemblyScope;
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs

@@ -21,7 +21,7 @@ protected OrdinaryMethod(Context cx, IMethodSymbol init)
         public override Microsoft.CodeAnalysis.Location ReportingLocation =>
             IsCompilerGeneratedDelegate()
                 ? Symbol.ContainingType.GetSymbolLocation()
-                : Symbol.GetSymbolLocation();
+                : BodyDeclaringSymbol.GetSymbolLocation();
 
         public override bool NeedsPopulation => base.NeedsPopulation || IsCompilerGeneratedDelegate();
 

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -250,6 +250,8 @@ private void DoAnalyseCompilation()
 
         public void LogPerformance(Entities.PerformanceMetrics p) => compilationEntity.PopulatePerformance(p);
 
+        public void ExtractAggregatedMessages() => compilationEntity.PopulateAggregatedMessages();
+
 #nullable restore warnings
 
         /// <summary>

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -458,6 +458,7 @@ public static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOpt
 
             sw.Restart();
             analyser.PerformExtraction(options.Threads);
+            analyser.ExtractAggregatedMessages();
             sw.Stop();
             var cpuTime2 = currentProcess.TotalProcessorTime;
             var userTime2 = currentProcess.UserProcessorTime;

csharp/extractor/Semmle.Extraction/BUILD.bazel

@@ -26,6 +26,7 @@ codeql_csharp_library(
         ],
         "//conditions:default": [],
     }),
+    internals_visible_to = ["Semmle.Extraction.CSharp"],
     visibility = ["//csharp:__subpackages__"],
     deps = [
         "//csharp/extractor/Semmle.Util",

csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System.Collections.Concurrent;
+using System.IO;
 using System.Threading;
 using Semmle.Util;
 
@@ -7,6 +8,8 @@ namespace Semmle.Extraction.Entities
     internal class ExtractionMessage : FreshEntity
     {
         private static readonly int limit = EnvironmentVariables.TryGetExtractorNumberOption<int>("MESSAGE_LIMIT") ?? 10000;
+
+        internal static readonly ConcurrentDictionary<string, int> groupedMessageCounts = [];
         private static int messageCount = 0;
 
         private readonly Message msg;
@@ -25,6 +28,10 @@ private ExtractionMessage(Context cx, Message msg, bool bypassLimit) : base(cx)
 
         protected override void Populate(TextWriter trapFile)
         {
+            // For the time being we're counting the number of messages per severity, we could introduce other groupings in the future
+            var key = msg.Severity.ToString();
+            groupedMessageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
+
             if (!bypassLimit)
             {
                 var val = Interlocked.Increment(ref messageCount);