Adds MessageFormat option.

Docs/CHANGELOG.md

@@ -4,6 +4,10 @@ Changes are relative to v1.0.0
 
 ## Changes since v1.1.0
 
+### New Features
+
+- Added `<MessageFormat>` option to configure what information is shown in the message part of the test case detail view. The default is to only show the assertion statistics for the test. It is also possible to choose to show all failure info in the message, or to not have a message at all.
+
 ### Extended Features
 
 - Improve error handling, specifically when a discovery timeout occurs it is now logged as a warning at logging level normal.

Docs/Readme.md

@@ -25,7 +25,8 @@ The following is an example _.runsettings_ file that contains options specific f
  <FilenameFilter>^Catch_</FilenameFilter><!-- Regex filter -->
  <IncludeHidden>true</IncludeHidden>
  <Logging>normal</Logging>
- <StackTraceFormat>Filename</StackTraceFormat>
+ <MessageFormat>StatsOnly</MessageFormat>
+ <StackTraceFormat>ShortInfo</StackTraceFormat>
  <TestCaseTimeout>20000</TestCaseTimeout><!-- Milliseconds -->
  <WorkingDirectory>..\TestData</WorkingDirectory>
  <WorkingDirectoryRoot>Executable</WorkingDirectoryRoot>

Docs/Settings.md

@@ -9,6 +9,7 @@ In order for the **Test Adapter for Catch2** to do its job, it requires certain
 - [`<FilenameFilter>`](#filenamefilter)*
 - [`<IncludeHidden>`](#includehidden)
 - [`<Logging>`](#logging)
+- [`<MessageFormat>`](#messageformat)
 - [`<StackTraceFormat>`](#stacktraceformat)
 - [`<TestCaseTimeout>`](#testcasetimeout)
 - [`<WorkingDirectory>`](#workingdirectory)
@@ -30,7 +31,8 @@ The following _.runsettings_ file examples only contains settings specific to th
  <FilenameFilter>^Catch_</FilenameFilter><!-- Regex filter -->
  <IncludeHidden>true</IncludeHidden>
  <Logging>normal</Logging>
- <StackTraceFormat>Filename</StackTraceFormat>
+ <MessageFormat>StatsOnly</MessageFormat>
+ <StackTraceFormat>ShortInfo</StackTraceFormat>
  <TestCaseTimeout>20000</TestCaseTimeout><!-- Milliseconds -->
  <WorkingDirectory>..\TestData</WorkingDirectory>
  <WorkingDirectoryRoot>Executable</WorkingDirectoryRoot>
@@ -241,6 +243,16 @@ The `<Logging>` option has four settings, `quiet`, `normal`, `verbose`, and `deb
 
 **Note: The `debug` level setting was introduced in v1.1.0. In v1.0.0 the `verbose` level is similar to the `debug` level.**
 
+## MessageFormat
+
+Default: StatsOnly
+
+The `<MessageFormat>` option has three settings, `Full`, `None` and `StatsOnly`. The addition of this setting is basically the result of fixing the stack trace link issue. Now the stack trace links to source are working, a significant part of the information is duplicated. Also, in case of many failures the stack trace information typically requires scrolling to get to it. Originally the full failure info was never intended to be shown in the error message. Only the test assertion statistics were originally envisioned for this area. As such the default is to only show the assertion statistics as message. To get all failure info in the message set this setting to `Full`. For the minimalists you can also set this setting to `None`, in which case no message is generated.
+
+When you opt to not have the full failure info in the message, this info is placed in the standard output in front of any standard output actually generated by the test. This info can then be reached via the output link that then appears in the detail view of the test case.
+
+**Note: This setting was introduced after v1.1.0.**
+
 ## StackTraceFormat
 
 ### post v1.1.0

Libraries/Catch2Interface/Constants.cs

@@ -35,7 +35,6 @@ public static class Constants
  public const string NodeName_OverallResult = "OverallResult";
  public const string NodeName_OverallResults = "OverallResults";
  public const string NodeName_Section = "Section";
- public const string NodeName_StackTraceFormat = "StackTraceFormat";
  public const string NodeName_TestCase = "TestCase";
  public const string NodeName_Warning = "Warning";
 
@@ -47,6 +46,8 @@ public static class Constants
  public const string NodeName_FilenameFilter = "FilenameFilter";
  public const string NodeName_IncludeHidden = "IncludeHidden";
  public const string NodeName_Logging = "Logging";
+ public const string NodeName_MessageFormat = "MessageFormat";
+ public const string NodeName_StackTraceFormat = "StackTraceFormat";
  public const string NodeName_TestCaseTimeout = "TestCaseTimeout";
  public const string NodeName_WorkingDirectory = "WorkingDirectory";
  public const string NodeName_WorkingDirectoryRoot = "WorkingDirectoryRoot";
@@ -62,6 +63,7 @@ public static class Constants
  public const string S_DefaultWorkingDirectory = "";
 
  public const LoggingLevels S_DefaultLoggingLevel = LoggingLevels.Normal;
+ public const MessageFormats S_DefaultMessageFormat = MessageFormats.StatsOnly;
  public const StacktraceFormats S_DefaultStackTraceFormat = StacktraceFormats.ShortInfo;
  public const WorkingDirectoryRoots S_DefaultWorkingDirectoryRoot = WorkingDirectoryRoots.Executable;
 

Libraries/Catch2Interface/Settings.cs

@@ -29,6 +29,18 @@ public enum LoggingLevels
  Debug
  }
 
+/*YAML
+Enum :
+ Description : >
+ This enum represents the valid message format settings.
+*/
+ public enum MessageFormats
+ {
+ None,
+ StatsOnly,
+ Full
+ }
+
 /*YAML
 Enum :
  Description : >
@@ -79,6 +91,10 @@ public class Settings
  static readonly Regex _rgxLogLevel_Quiet = new Regex(@"^(?i:quiet)$", RegexOptions.Singleline);
  static readonly Regex _rgxLogLevel_Verbose = new Regex(@"^(?i:verbose)$", RegexOptions.Singleline);
 
+ static readonly Regex _rgxMessageFormat_Full = new Regex(@"^(?i:full)$", RegexOptions.Singleline);
+ static readonly Regex _rgxMessageFormat_None = new Regex(@"^(?i:none)$", RegexOptions.Singleline);
+ static readonly Regex _rgxMessageFormat_StatsOnly = new Regex(@"^(?i:statsonly)$", RegexOptions.Singleline);
+
  static readonly Regex _rgxStackTraceFormat_None = new Regex(@"^(?i:none)$", RegexOptions.Singleline);
  static readonly Regex _rgxStackTraceFormat_ShortInfo = new Regex(@"^(?i:shortinfo)$", RegexOptions.Singleline);
 
@@ -98,6 +114,7 @@ public class Settings
  public string FilenameFilter { get; set; } = Constants.S_DefaultFilenameFilter;
  public bool IncludeHidden { get; set; } = Constants.S_DefaultIncludeHidden;
  public LoggingLevels LoggingLevel { get; set; } = Constants.S_DefaultLoggingLevel;
+ public MessageFormats MessageFormat { get; set; } = Constants.S_DefaultMessageFormat;
  public StacktraceFormats StacktraceFormat { get; set; } = Constants.S_DefaultStackTraceFormat;
  public int TestCaseTimeout { get; set; } = Constants.S_DefaultTestCaseTimeout;
  public string WorkingDirectory { get; set; } = Constants.S_DefaultWorkingDirectory;
@@ -182,6 +199,14 @@ public static Settings Extract(XmlNode node)
  settings.LoggingLevel = ConvertToLoggingLevel(logging.Value.Trim());
  }
 
+ // MessageFormat
+ var messageformat = node.SelectSingleNode(Constants.NodeName_MessageFormat)?.FirstChild;
+ if (messageformat != null
+ && messageformat.NodeType == XmlNodeType.Text)
+ {
+ settings.MessageFormat = ConvertToMessageFormat(messageformat.Value.Trim());
+ }
+
  // StacktraceFormat
  var stacktraceformat = node.SelectSingleNode(Constants.NodeName_StackTraceFormat)?.FirstChild;
  if (stacktraceformat != null
@@ -244,7 +269,27 @@ private static LoggingLevels ConvertToLoggingLevel(string level)
  return LoggingLevels.Debug;
  }
 
- return LoggingLevels.Normal;
+ return Constants.S_DefaultLoggingLevel;
+ }
+
+ private static MessageFormats ConvertToMessageFormat(string format)
+ {
+ if (_rgxMessageFormat_Full.IsMatch(format))
+ {
+ return MessageFormats.Full;
+ }
+
+ if (_rgxMessageFormat_None.IsMatch(format))
+ {
+ return MessageFormats.None;
+ }
+
+ if (_rgxMessageFormat_StatsOnly.IsMatch(format))
+ {
+ return MessageFormats.StatsOnly;
+ }
+
+ return Constants.S_DefaultMessageFormat;
  }
 
  private static StacktraceFormats ConvertToStacktraceFormat(string format)
@@ -259,7 +304,7 @@ private static StacktraceFormats ConvertToStacktraceFormat(string format)
  return StacktraceFormats.None;
  }
 
- return StacktraceFormats.ShortInfo;
+ return Constants.S_DefaultStackTraceFormat;
  }
 
  private static WorkingDirectoryRoots ConvertToWorkingDirectoryRoot(string root)
@@ -279,7 +324,7 @@ private static WorkingDirectoryRoots ConvertToWorkingDirectoryRoot(string root)
  return WorkingDirectoryRoots.TestRun;
  }
 
- return WorkingDirectoryRoots.Executable;
+ return Constants.S_DefaultWorkingDirectoryRoot;
  }
 
  #endregion // Private Static

Libraries/Catch2Interface/TestResult.cs

@@ -76,6 +76,7 @@ public TestResult( TimeSpan duration
  public string ErrorMessage { get; private set; }
  public string ErrorStackTrace { get; private set; }
 
+ public string AdditionalInfo { get; private set; }
  public string StandardOut { get; private set; }
  public string StandardError { get; private set; }
 
@@ -306,7 +307,7 @@ void ExtractTestResult(XmlNode nodeGroup)
  // Statistics
  ExtractOverallResults(nodeGroup);
 
- GenerateErrorMessage();
+ GenerateMessages();
  }
  else
  {
@@ -321,22 +322,56 @@ private void ExtractOverallResults(XmlNode nodeGroup)
  OverallResults = new Reporter.OverallResults(nodeOvRes);
  }
 
- private void GenerateErrorMessage()
+ private void GenerateMessages()
  {
- if(OverallResults == null)  // Sanity check
+ if(OverallResults == null) // Sanity check
  {
  ErrorMessage = string.Empty;
+ return;
  }
- else if (_msgbuilder.Length == 0)
+
+ if (_msgbuilder.Length == 0)
  {
- ErrorMessage = $"Total Assertions: {OverallResults.TotalAssertions} (Passed: {OverallResults.Successes} | Failed: {OverallResults.Failures}){Environment.NewLine}";
+ AdditionalInfo = string.Empty;
  }
  else
  {
- ErrorMessage = $"Total Assertions: {OverallResults.TotalAssertions} (Passed: {OverallResults.Successes} | Failed: {OverallResults.Failures}){Environment.NewLine}"
- + $"-------------------------------------------------------------------------------{Environment.NewLine}"
- + $"{_msgbuilder.ToString()}"
- + $"-------------------------------------------------------------------------------";
+ AdditionalInfo = $"-------------------------------------------------------------------------------{Environment.NewLine}"
+ + $"{_msgbuilder.ToString()}"
+ + $"-------------------------------------------------------------------------------";
+ }
+
+ switch( _settings.MessageFormat )
+ {
+ case MessageFormats.None:
+ ErrorMessage = string.Empty;
+
+ AdditionalInfo = $"Total Assertions: {OverallResults.TotalAssertions} (Passed: {OverallResults.Successes} | Failed: {OverallResults.Failures}){Environment.NewLine}"
+ + AdditionalInfo;
+
+ StandardOut = $"Additional Info:{Environment.NewLine}{AdditionalInfo}{Environment.NewLine}{StandardOut}";
+
+ break;
+ case MessageFormats.Full:
+ if ( string.IsNullOrEmpty(AdditionalInfo) )
+ {
+ ErrorMessage = $"Total Assertions: {OverallResults.TotalAssertions} (Passed: {OverallResults.Successes} | Failed: {OverallResults.Failures}){Environment.NewLine}";
+ }
+ else
+ {
+ ErrorMessage = $"Total Assertions: {OverallResults.TotalAssertions} (Passed: {OverallResults.Successes} | Failed: {OverallResults.Failures}){Environment.NewLine}"
+ + AdditionalInfo;
+ }
+ break;
+ default: // MessageFormats.StatsOnly:
+ ErrorMessage = ErrorMessage = $"Total Assertions: {OverallResults.TotalAssertions} (Passed: {OverallResults.Successes} | Failed: {OverallResults.Failures}){Environment.NewLine}";
+
+ // Prepend AdditionalInfo to StandardOut output
+ if ( !string.IsNullOrEmpty(AdditionalInfo) )
+ {
+ StandardOut = $"Additional Info:{Environment.NewLine}{AdditionalInfo}{Environment.NewLine}{StandardOut}";
+ }
+ break;
  }
  }
 
@@ -345,9 +380,9 @@ private void SetInvalidTestRunnerOutput()
  Success = false;
  OverallResults = new Reporter.OverallResults();
  ErrorMessage = $"Invalid test runner output.{Environment.NewLine}"
-  + $"-------------------------------------------------------------------------------{Environment.NewLine}"
-  + $"{_xmloutput}"
-  + $"-------------------------------------------------------------------------------";
+ + $"-------------------------------------------------------------------------------{Environment.NewLine}"
+ + $"{_xmloutput}"
+ + $"-------------------------------------------------------------------------------";
  }
 
  private void ProcessXml()

Libraries/Catch2TestAdapter/TestExecutor.cs

@@ -220,15 +220,20 @@ private TestResult RunTest(TestCase test)
  result.ErrorMessage = testresult.ErrorMessage;
  result.ErrorStackTrace = testresult.ErrorStackTrace;
 
- if(testresult.StandardOut != string.Empty )
+ if( !string.IsNullOrEmpty(testresult.StandardOut) )
  {
  result.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, testresult.StandardOut ));
  }
 
- if(testresult.StandardError != string.Empty )
+ if( !string.IsNullOrEmpty(testresult.StandardError) )
  {
  result.Messages.Add(new TestResultMessage(TestResultMessage.StandardErrorCategory, testresult.StandardError ));
  }
+
+ if( !string.IsNullOrEmpty(testresult.AdditionalInfo) )
+ {
+ result.Messages.Add(new TestResultMessage(TestResultMessage.AdditionalInfoCategory, testresult.AdditionalInfo ));
+ }
  }
  }
 

ReferenceTests/MSBuild/Projects/Catch_Testset03/Catch_Testset03.vcxproj

@@ -16,6 +16,7 @@
  <ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests04.cpp" />
  <ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests05.cpp" />
  <ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests06.cpp" />
+ <ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests07.cpp" />
  <ClCompile Include="..\..\..\Src\Catch2\main.cpp" />
  </ItemGroup>
 </Project>

ReferenceTests/MSBuild/Projects/Catch_Testset03/Catch_Testset03.vcxproj.filters

@@ -30,5 +30,8 @@
  <ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests06.cpp">
  <Filter>Files</Filter>
  </ClCompile>
+ <ClCompile Include="..\..\..\Src\Catch2\Catch_Testset03\UT_Tests07.cpp">
+ <Filter>Files</Filter>
+ </ClCompile>
  </ItemGroup>
 </Project>

ReferenceTests/ReferenceTests.runsettings

@@ -24,8 +24,9 @@
  <DiscoverTimeout>500</DiscoverTimeout>
  <FilenameFilter>^Catch_</FilenameFilter>
  <IncludeHidden>true</IncludeHidden>
- <Logging>verbose</Logging>
- <StackTraceFormat>FilePath</StackTraceFormat>
+ <Logging>Verbose</Logging>
+ <MessageFormat>StatsOnly</MessageFormat>
+ <StackTraceFormat>ShortInfo</StackTraceFormat>
  <TestCaseTimeout>20000</TestCaseTimeout><!-- Milliseconds -->
  <WorkingDirectory></WorkingDirectory>
  <WorkingDirectoryRoot>Executable</WorkingDirectoryRoot>