Add Profiler based code coverage (#1937)

Do code coverage using the new Profiler approach under UseBreakpoints = $false experimental flag.

src/Main.ps1

@@ -969,6 +969,8 @@ function Invoke-Pester {
                     Path = @($paths)
                     RecursePaths = $PesterPreference.CodeCoverage.RecursePaths.Value
                     TestExtension = $PesterPreference.Run.TestExtension.Value
+                    UseSingleHitBreakpoints = $PesterPreference.CodeCoverage.SingleHitBreakpoints.Value
+                    UseBreakpoints = $PesterPreference.CodeCoverage.UseBreakpoints.Value
                 }
 
                 $plugins += (Get-CoveragePlugin)
@@ -1007,7 +1009,7 @@ function Invoke-Pester {
                     Configuration = $pluginConfiguration
                     GlobalPluginData = $state.PluginData
                     WriteDebugMessages = $PesterPreference.Debug.WriteDebugMessages.Value
-                    Write_PesterDebugMessage = if ($PesterPreference.Debug.WriteDebugMessages) { $script:SafeCommands['Write-PesterDebugMessage'] }
+                    Write_PesterDebugMessage = if ($PesterPreference.Debug.WriteDebugMessages.Value) { $script:SafeCommands['Write-PesterDebugMessage'] }
                 } -ThrowOnFailure
             }
 
@@ -1016,16 +1018,13 @@ function Invoke-Pester {
                 return
             }
 
+
             $r = Invoke-Test -BlockContainer $containers -Plugin $plugins -PluginConfiguration $pluginConfiguration -PluginData $pluginData -SessionState $sessionState -Filter $filter -Configuration $PesterPreference
 
             foreach ($c in $r) {
                 Fold-Container -Container $c  -OnTest { param($t) Add-RSpecTestObjectProperties $t }
             }
 
-            $parameters = @{
-                PSBoundParameters = $PSBoundParameters
-            }
-
             $run = [Pester.Run]::Create()
             $run.Executed = $true
             $run.ExecutedAt = $start
@@ -1068,7 +1067,8 @@ function Invoke-Pester {
                     & $SafeCommands["Write-Host"] -ForegroundColor Magenta "Processing code coverage result."
                 }
                 $breakpoints = @($run.PluginData.Coverage.CommandCoverage)
-                $coverageReport = Get-CoverageReport -CommandCoverage $breakpoints
+                $measure = if (-not $PesterPreference.CodeCoverage.UseBreakpoints.Value) { @($run.PluginData.Coverage.Tracer.Hits) }
+                $coverageReport = Get-CoverageReport -CommandCoverage $breakpoints -Measure $measure
                 $totalMilliseconds = $run.Duration.TotalMilliseconds
 
                 $configuration = $run.PluginConfiguration.Coverage

src/Pester.RSpec.ps1

@@ -351,6 +351,9 @@ function New-PesterConfiguration {
       CoveragePercentTarget: Target percent of code coverage that you want to achieve, default 75%.
       Default value: 75
 
+      UseBreakpoints: EXPERIMENTAL: When false, use Profiler based tracer to do CodeCoverage instead of using breakpoints.
+      Default value: $true
+
       SingleHitBreakpoints: Remove breakpoint when it is hit.
       Default value: $true
 

src/Pester.Runtime.ps1

@@ -1028,7 +1028,7 @@ function Run-Test {
             Configuration = $state.PluginConfiguration
             Data          = $state.PluginData
             WriteDebugMessages = $PesterPreference.Debug.WriteDebugMessages.Value
-            Write_PesterDebugMessage = if ($PesterPreference.Debug.WriteDebugMessages) { $script:SafeCommands['Write-PesterDebugMessage'] }
+            Write_PesterDebugMessage = if ($PesterPreference.Debug.WriteDebugMessages.Value) { $script:SafeCommands['Write-PesterDebugMessage'] }
         } -ThrowOnFailure
     }
     foreach ($rootBlock in $Block) {

src/Pester.psd1

@@ -4,7 +4,7 @@
     RootModule        = 'Pester.psm1'
 
     # Version number of this module.
-    ModuleVersion     = '5.2.1'
+    ModuleVersion     = '5.3.0'
 
     # ID used to uniquely identify this module
     GUID              = 'a699dea5-2c73-4616-a270-1f7abb777e71'
@@ -115,11 +115,11 @@
             ReleaseNotes = 'https://github.com/pester/Pester/releases/tag/5.2.1'
 
             # Prerelease string of this module
-            Prerelease   = ''
+            Prerelease   = 'alpha1'
         }
 
         # Minimum assembly version required
-        RequiredAssemblyVersion = '5.2.0'
+        RequiredAssemblyVersion = '5.3.0'
     }
 
     # HelpInfo URI of this module

src/csharp/Pester/CodeCoverageConfiguration.cs

@@ -28,6 +28,7 @@ public class CodeCoverageConfiguration : ConfigurationSection
         private StringArrayOption _path;
         private BoolOption _excludeTests;
         private BoolOption _recursePaths;
+        private BoolOption _useBps;
         private BoolOption _singleHitBreakpoints;
         private DecimalOption _coveragePercentTarget;
 
@@ -47,6 +48,7 @@ public CodeCoverageConfiguration() : base("CodeCoverage configuration.")
             Path = new StringArrayOption("Directories or files to be used for codecoverage, by default the Path(s) from general settings are used, unless overridden here.", new string[0]);
             ExcludeTests = new BoolOption("Exclude tests from code coverage. This uses the TestFilter from general configuration.", true);
             RecursePaths = new BoolOption("Will recurse through directories in the Path option.", true);
+            UseBreakpoints = new BoolOption("EXPERIMENTAL: When false, use Profiler based tracer to do CodeCoverage instead of using breakpoints.", true);
             CoveragePercentTarget = new DecimalOption("Target percent of code coverage that you want to achieve, default 75%.", 75m);
             SingleHitBreakpoints = new BoolOption("Remove breakpoint when it is hit.", true);
         }
@@ -65,6 +67,7 @@ public CodeCoverageConfiguration(IDictionary configuration) : this()
                 CoveragePercentTarget = configuration.GetValueOrNull<decimal>("CoveragePercentTarget") ?? CoveragePercentTarget;
 
                 SingleHitBreakpoints = configuration.GetValueOrNull<bool>("SingleHitBreakpoints") ?? SingleHitBreakpoints;
+                UseBreakpoints = configuration.GetValueOrNull<bool>("UseBreakpoints") ?? UseBreakpoints;
             }
         }
 
@@ -197,6 +200,23 @@ public DecimalOption CoveragePercentTarget
             }
         }
 
+
+        public BoolOption UseBreakpoints
+        {
+            get { return _useBps; }
+            set
+            {
+                if (_useBps == null)
+                {
+                    _useBps = value;
+                }
+                else
+                {
+                    _useBps = new BoolOption(_useBps, value.Value);
+                }
+            }
+        }
+
         public BoolOption SingleHitBreakpoints
         {
             get { return _singleHitBreakpoints; }

src/csharp/Pester/Tracing/CodeCoveragePoint.cs

@@ -0,0 +1,46 @@
+namespace Pester.Tracing
+{
+    public struct CodeCoveragePoint
+    {
+        public static CodeCoveragePoint Create(string path, int line, int column, int bpColumn, string astText)
+        {
+            return new CodeCoveragePoint(path, line, column, bpColumn, astText);
+        }
+
+        public CodeCoveragePoint(string path, int line, int column, int bpColumn, string astText)
+        {
+            Path = path;
+            Line = line;
+            Column = column;
+            BpColumn = bpColumn;
+            AstText = astText;
+
+            // those are not for users to set,
+            // we use them to make CC output easier to debug
+            // because this will show in list of hits what we think
+            // should or should not hit, for performance just bool 
+            // would be enough
+            Text = default;
+            Hit = false;
+        }
+
+        public int Line;
+        public int Column;
+        public int BpColumn;
+        public string Path;
+        public string AstText;
+
+        // those are not for users to set,
+        // we use them to make CC output easier to debug
+        // because this will show in list of hits what we think
+        // should or should not hit, for performance just bool 
+        // would be enough
+        public string Text;
+        public bool Hit;
+
+        public override string ToString()
+        {
+            return $"{Hit}:'{AstText}':{Line}:{Column}:{Path}";
+        }
+    }
+}

src/csharp/Pester/Tracing/CodeCoverageTracer.cs

@@ -0,0 +1,66 @@
+using System.Collections.Generic;
+using System.Management.Automation;
+using System.Management.Automation.Language;
+
+namespace Pester.Tracing
+{
+    public class CodeCoverageTracer : ITracer
+    {
+        public static CodeCoverageTracer Create(List<CodeCoveragePoint> points)
+        {
+            return new CodeCoverageTracer(points);
+        }
+
+        public CodeCoverageTracer(List<CodeCoveragePoint> points)
+        {
+            foreach (var point in points)
+            {
+                var key = $"{point.Line}:{point.Column}";
+                if (!Hits.ContainsKey(point.Path))
+                {
+                    var lineColumn = new Dictionary<string, CodeCoveragePoint> { [key] = point };
+                    Hits.Add(point.Path, lineColumn);
+                    continue;
+                }
+
+                var hits = Hits[point.Path];
+                if (!hits.ContainsKey(key))
+                {
+                    hits.Add(key, point);
+                    continue;
+                }
+
+                // if the key is there do nothing, we already set it to false
+            }
+        }
+
+        // list of what Pester figures out from the AST that we care about for CC
+        // keyed as path -> line:column -> CodeCoveragePoint
+        public Dictionary<string, Dictionary<string, CodeCoveragePoint>> Hits { get; } = new Dictionary<string, Dictionary<string, CodeCoveragePoint>>();
+
+        public void Trace(IScriptExtent extent, ScriptBlock _, int __)
+        {
+            // ignore unbound scriptblocks
+            if (extent?.File == null)
+                return;
+
+            // Console.WriteLine($"{extent.File}:{extent.StartLineNumber}:{extent.StartColumnNumber}:{extent.Text}");
+            if (!Hits.TryGetValue(extent.File, out var lineColumn))
+                return;
+
+            var key2 = $"{extent.StartLineNumber}:{extent.StartColumnNumber}";
+            if (!lineColumn.ContainsKey(key2))
+                return;
+
+
+            var point = lineColumn[key2];
+            if (point.Hit == true)
+                return;
+
+            point.Hit = true;
+            point.Text = extent.Text;
+
+            lineColumn[key2] = point;
+        }
+    }
+}

src/csharp/Pester/Tracing/ExternalTracerAdapter.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Management.Automation;
+using System.Management.Automation.Language;
+using System.Reflection;
+
+namespace Pester.Tracing
+{
+    class ExternalTracerAdapter : ITracer
+    {
+        private object _tracer;
+        private MethodInfo _traceMethod;
+
+        public ExternalTracerAdapter(object tracer)
+        {
+            _tracer = tracer ?? new NullReferenceException(nameof(tracer));
+            var traceMethod = tracer.GetType().GetMethod("Trace", new Type[] { typeof(IScriptExtent), typeof(ScriptBlock), typeof(int)  });
+            _traceMethod = traceMethod ?? throw new InvalidOperationException("The provided tracer does not have Trace method with this signature: Trace(IScriptExtent extent, ScriptBlock scriptBlock, int level)");
+        }
+
+        public void Trace(IScriptExtent extent, ScriptBlock scriptBlock, int level)
+        {
+            _traceMethod.Invoke(_tracer, new object[] { extent, scriptBlock, level });
+        }
+    }
+}

src/csharp/Pester/Tracing/ITracer.cs

@@ -0,0 +1,10 @@
+using System.Management.Automation;
+using System.Management.Automation.Language;
+
+namespace Pester.Tracing
+{
+    public interface ITracer
+    {
+        void Trace(IScriptExtent extent, ScriptBlock scriptBlock, int level);
+    }
+}