Merge pull request #1174 from calumgrant/cs/extractor-diagnostics

C#: Log compiler and extractor diagnostics

Author: hvitved

csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs

@@ -3,6 +3,7 @@
 using System.Collections.Generic;
 using Semmle.Util.Logging;
 using System;
+using Semmle.Extraction.Entities;
 
 namespace Semmle.Extraction.CIL.Entities
 {
@@ -70,12 +71,7 @@ public override IEnumerable<IExtractionProduct> Contents
                     }
                     catch (InternalError e)
                     {
-                        cx.cx.Extractor.Message(new Message
-                        {
-                            exception = e,
-                            message = "Error processing type definition",
-                            severity = Semmle.Util.Logging.Severity.Error
-                        });
+                        cx.cx.ExtractionError("Error processing type definition", e.Message, GeneratedLocation.Create(cx.cx), e.StackTrace);
                     }
 
                     // Limitation of C#: Cannot yield return inside a try-catch.
@@ -92,12 +88,7 @@ public override IEnumerable<IExtractionProduct> Contents
                     }
                     catch (InternalError e)
                     {
-                        cx.cx.Extractor.Message(new Message
-                        {
-                            exception = e,
-                            message = "Error processing bytecode",
-                            severity = Semmle.Util.Logging.Severity.Error
-                        });
+                        cx.cx.ExtractionError("Error processing bytecode", e.Message, GeneratedLocation.Create(cx.cx), e.StackTrace);
                     }
 
                     if (product != null)
@@ -139,7 +130,7 @@ public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger
                     trapFile = trapWriter.TrapFile;
                     if (nocache || !System.IO.File.Exists(trapFile))
                     {
-                        var cx = new Extraction.Context(extractor, null, trapWriter, null);
+                        var cx = extractor.CreateContext(null, trapWriter, null);
                         ExtractCIL(cx, assemblyPath, extractPdbs);
                         extracted = true;
                     }

csharp/extractor/Semmle.Extraction.CIL/Entities/Instruction.cs

@@ -409,7 +409,7 @@ public override IEnumerable<IExtractionProduct> Contents
                         }
                         else
                         {
-                            throw new InternalError("Unable to create payload type {0} for opcode {1}", PayloadType, OpCode);
+                            throw new InternalError($"Unable to create payload type {PayloadType} for opcode {OpCode}");
                         }
                         break;
                     case Payload.Arg8:
@@ -430,7 +430,7 @@ public override IEnumerable<IExtractionProduct> Contents
                         // Some of these are handled by JumpContents().
                         break;
                     default:
-                        throw new InternalError("Unhandled payload type {0}", PayloadType);
+                        throw new InternalError($"Unhandled payload type {PayloadType}");
                 }
             }
         }
@@ -479,7 +479,7 @@ public IEnumerable<IExtractionProduct> JumpContents(Dictionary<int, IInstruction
                 // TODO: Find a solution to this.
 
                 // For now, just log the error
-                cx.cx.Extractor.Message(new Message { message = "A CIL instruction jumps outside the current method", severity = Util.Logging.Severity.Warning });
+                cx.cx.ExtractionError("A CIL instruction jumps outside the current method", "", Extraction.Entities.GeneratedLocation.Create(cx.cx), "", Util.Logging.Severity.Warning);
             }
         }
     }

csharp/extractor/Semmle.Extraction.CIL/Entities/Method.cs

@@ -497,7 +497,7 @@ public override IEnumerable<IExtractionProduct> Contents
                         constructedTypeSignature = md.DecodeSignature(cx.TypeSignatureDecoder, this);
                         break;
                     default:
-                        throw new InternalError("Unexpected constructed method handle kind {0}", ms.Method.Kind);
+                        throw new InternalError($"Unexpected constructed method handle kind {ms.Method.Kind}");
                 }
 
                 PopulateParameters(constructedTypeSignature.ParameterTypes);

csharp/extractor/Semmle.Extraction.CIL/Id.cs

@@ -197,7 +197,7 @@ public static StringId Id(this PrimitiveTypeCode typeCode)
                 case PrimitiveTypeCode.UIntPtr: return uintptrId;
                 case PrimitiveTypeCode.Void: return voidId;
                 case PrimitiveTypeCode.TypedReference: return typedReferenceId;
-                default: throw new InternalError("Unhandled type code {0}", typeCode);
+                default: throw new InternalError($"Unhandled type code {typeCode}");
             }
         }
     }

csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs

@@ -99,12 +99,7 @@ void SetReferencePaths()
                 }
                 catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
                 {
-                    extractor.Message(new Message
-                    {
-                        exception = ex,
-                        message = string.Format("Exception reading reference file {0}: {1}",
-                        reference.FilePath, ex)
-                    });
+                    extractor.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
                 }
             }
         }
@@ -215,6 +210,38 @@ bool FileIsCached(string src, string dest)
             return options.Cache && FileIsUpToDate(src, dest);
         }
 
+        /// <summary>
+        /// Extracts compilation-wide entities, such as compilations and compiler diagnostics.
+        /// </summary>
+        public void AnalyseCompilation(string cwd, string[] args)
+        {
+            extractionTasks.Add(() => DoAnalyseCompilation(cwd, args));
+        }
+
+        Entities.Compilation compilationEntity;
+        IDisposable compilationTrapFile;
+
+        void DoAnalyseCompilation(string cwd, string[] args)
+        {
+            try
+            {
+                var assemblyPath = extractor.OutputPath;
+                var assembly = compilation.Assembly;
+                var projectLayout = layout.LookupProjectOrDefault(assemblyPath);
+                var trapWriter = projectLayout.CreateTrapWriter(Logger, assemblyPath, true);
+                compilationTrapFile = trapWriter;  // Dispose later
+                var cx = extractor.CreateContext(compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath, true));
+
+                compilationEntity = new Entities.Compilation(cx, cwd, args);
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                Logger.Log(Severity.Error, "  Unhandled exception analyzing {0}: {1}", "compilation", ex);
+            }
+        }
+
+        public void LogPerformance(Entities.PerformanceMetrics p) => compilationEntity.PopulatePerformance(p);
+ 
         /// <summary>
         ///     Extract an assembly to a new trap file.
         ///     If the trap file exists, skip extraction to avoid duplicating
@@ -258,7 +285,7 @@ void DoAnalyseAssembly(PortableExecutableReference r)
 
                         if (assembly != null)
                         {
-                            var cx = new Context(extractor, c, trapWriter, new AssemblyScope(assembly, assemblyPath));
+                            var cx = extractor.CreateContext(c, trapWriter, new AssemblyScope(assembly, assemblyPath, false));
 
                             foreach (var module in assembly.Modules)
                             {
@@ -344,7 +371,7 @@ void DoExtractTree(SyntaxTree tree)
 
                         if (!upToDate)
                         {
-                            Context cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree));
+                            Context cx = extractor.CreateContext(compilation.Clone(), trapWriter, new SourceScope(tree));
                             Populators.CompilationUnit.Extract(cx, tree.GetRoot());
                             cx.PopulateAll();
                             cx.ExtractComments(cx.CommentGenerator);
@@ -356,7 +383,7 @@ void DoExtractTree(SyntaxTree tree)
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
-                extractor.Message(new Message { exception = ex, message = string.Format("Unhandled exception processing {0}: {1}", tree.FilePath, ex), severity = Severity.Error });
+                extractor.Message(new Message("Unhandled exception processing syntax tree", tree.FilePath, null, ex.StackTrace));
             }
         }
 
@@ -373,6 +400,8 @@ public void PerformExtraction(int numberOfThreads)
 
         public void Dispose()
         {
+            compilationTrapFile?.Dispose();
+
             stopWatch.Stop();
             Logger.Log(Severity.Info, "  Peak working set = {0} MB", Process.GetCurrentProcess().PeakWorkingSet64 / (1024 * 1024));
 

csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs

@@ -105,7 +105,7 @@ So we split it into separate lines
                     break;
                 // Strangely, these are reported as SingleLineCommentTrivia.
                 case SyntaxKind.DocumentationCommentExteriorTrivia:
-                    cx.ModelError("Unhandled comment type {0} for {1}", trivia.Kind(), trivia);
+                    cx.ModelError($"Unhandled comment type {trivia.Kind()} for {trivia}");
                     break;
             }
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilation.cs

@@ -0,0 +1,101 @@
+using Microsoft.CodeAnalysis;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+
+namespace Semmle.Extraction.CSharp.Entities
+{
+    class Compilation : FreshEntity
+    {
+        public Compilation(Context cx, string cwd, string[] args) : base(cx)
+        {
+            Extraction.Entities.Assembly.CreateOutputAssembly(cx);
+
+            cx.Emit(Tuples.compilations(this, Extraction.Entities.File.PathAsDatabaseString(cwd)));
+
+            // Arguments
+            int index = 0;
+            foreach(var arg in args)
+            {
+                cx.Emit(Tuples.compilation_args(this, index++, arg));
+            }
+
+            // Files
+            index = 0;
+            foreach(var file in cx.Compilation.SyntaxTrees.Select(tree => Extraction.Entities.File.Create(cx, tree.FilePath)))
+            {
+                cx.Emit(Tuples.compilation_compiling_files(this, index++, file));
+            }
+
+            // References
+            index = 0;
+            foreach(var file in cx.Compilation.References.OfType<PortableExecutableReference>().Select(r => Extraction.Entities.File.Create(cx, r.FilePath)))
+            {
+                cx.Emit(Tuples.compilation_referencing_files(this, index++, file));
+            }
+
+            // Diagnostics
+            index = 0;
+            foreach(var diag in cx.Compilation.GetDiagnostics().Select(d => new Diagnostic(cx, d)))
+            {
+                cx.Emit(Tuples.diagnostic_for(diag, this, 0, index++));
+            }
+        }
+
+        public void PopulatePerformance(PerformanceMetrics p)
+        {
+            int index = 0;
+            foreach(float metric in p.Metrics)
+            {
+                cx.Emit(Tuples.compilation_time(this, -1, index++, metric));
+            }
+            cx.Emit(Tuples.compilation_finished(this, (float)p.Total.Cpu.TotalSeconds, (float)p.Total.Elapsed.TotalSeconds));
+        }
+
+        public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel;
+    }
+
+    class Diagnostic : FreshEntity
+    {
+        public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel;
+
+        public Diagnostic(Context cx, Microsoft.CodeAnalysis.Diagnostic diag) : base(cx)
+        {
+            cx.Emit(Tuples.diagnostics(this, (int)diag.Severity, diag.Id, diag.Descriptor.Title.ToString(),
+                diag.GetMessage(), Extraction.Entities.Location.Create(cx, diag.Location)));
+        }
+    }
+
+    public struct Timings
+    {
+        public TimeSpan Elapsed, Cpu, User;
+    }
+
+    /// <summary>
+    /// The various performance metrics to log.
+    /// </summary>
+    public struct PerformanceMetrics
+    {
+        public Timings Frontend, Extractor, Total;
+        public long PeakWorkingSet;
+
+        /// <summary>
+        /// These are in database order (0 indexed)
+        /// </summary>
+        public IEnumerable<float> Metrics
+        {
+            get
+            {
+                yield return (float)Frontend.Cpu.TotalSeconds;
+                yield return (float)Frontend.Elapsed.TotalSeconds;
+                yield return (float)Extractor.Cpu.TotalSeconds;
+                yield return (float)Extractor.Elapsed.TotalSeconds;
+                yield return (float)Frontend.User.TotalSeconds;
+                yield return (float)Extractor.User.TotalSeconds;
+                yield return PeakWorkingSet / 1024.0f / 1024.0f;
+            }
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Access.cs

@@ -34,7 +34,7 @@ static ExprKind AccessKind(Context cx, ISymbol symbol)
                     return ExprKind.PARAMETER_ACCESS;
 
                 default:
-                    cx.ModelError(symbol, "Unhandled access kind '{0}'", symbol.Kind);
+                    cx.ModelError(symbol, $"Unhandled access kind '{symbol.Kind}'");
                     return ExprKind.UNKNOWN;
             }
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Binary.cs

@@ -53,7 +53,7 @@ static ExprKind GetBinaryTokenKind(Context cx, SyntaxKind kind)
                 case SyntaxKind.QuestionQuestionToken: return ExprKind.NULL_COALESCING;
                 // !! And the rest
                 default:
-                    cx.ModelError("Unhandled operator type {0}", kind);
+                    cx.ModelError($"Unhandled operator type {kind}");
                     return ExprKind.UNKNOWN;
             }
         }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Expressions/Factory.cs

@@ -233,7 +233,7 @@ internal static Expression Create(ExpressionNodeInfo info)
                         return IsPattern.Create(info);
 
                     default:
-                        info.Context.ModelError(info.Node, "Unhandled expression '{0}' of kind '{1}'", info.Node, info.Node.Kind());
+                        info.Context.ModelError(info.Node, $"Unhandled expression '{info.Node}' of kind '{info.Node.Kind()}'");
                         return new Unknown(info);
                 }
             }