Fix performance tests project and improvements (#4491)

* Throw error if tests didn't run. * Fix output color. * Use .NET 8. Disable CPM. * Use .NET 8 SDK. * Fix Intall MAUI task. * MAUI iOS to 17 * net6.0-ios * Remove includePreviewVersions. Change LangVersion. * Revert using .NET 8. Use .NET 8 and 6 in build pipeline. * includePreviewVersions: true * Run perf project on net6. * Revert. * Fix.
AzureAD · Jan 13, 2024 · 4c60cef · 4c60cef
1 parent f7f2faf
commit 4c60cef
Show file tree

Hide file tree

Showing 12 changed files with 45 additions and 99 deletions.
diff --git a/LibsAndSamples.sln b/LibsAndSamples.sln
@@ -88,15 +88,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pipelines", "pipelines", "{
 		build\pipeline-pullrequest.yaml = build\pipeline-pullrequest.yaml
 		build\pipeline-releasebuild-netonly.yaml = build\pipeline-releasebuild-netonly.yaml
 		build\pipeline-releasebuild.yaml = build\pipeline-releasebuild.yaml
-		build\template-android-appcenter-tests.yaml = build\template-android-appcenter-tests.yaml
 		build\template-build-and-prep-automation.yaml = build\template-build-and-prep-automation.yaml
 		build\template-build-and-run-all-tests.yaml = build\template-build-and-run-all-tests.yaml
 		build\template-cachecompat-automation.yaml = build\template-cachecompat-automation.yaml
-		build\template-ci-and-pr.yaml = build\template-ci-and-pr.yaml
 		build\template-desktop-unit-and-automation.yaml = build\template-desktop-unit-and-automation.yaml
-		build\template-install-dotnet-core.yaml = build\template-install-dotnet-core.yaml
 		build\template-install-keyvault-secrets.yaml = build\template-install-keyvault-secrets.yaml
-		build\template-ios-appcenter-tests.yaml = build\template-ios-appcenter-tests.yaml
 		build\template-nuget-pack.yaml = build\template-nuget-pack.yaml
 		build\template-OneBranch-CI-libsandsamples.yaml = build\template-OneBranch-CI-libsandsamples.yaml
 		build\template-OneBranch-Release-libsandsamples.yaml = build\template-OneBranch-Release-libsandsamples.yaml

diff --git a/build/pipeline-perf-tests-automation.yaml b/build/pipeline-perf-tests-automation.yaml
@@ -16,23 +16,19 @@ pool:
 variables:
   BuildPlatform: 'any cpu'
   BuildConfiguration: 'release'
+  BuildRuntime: 'net6.0'
 
 jobs:
   - job: 'BuildJob'
     displayName: 'Build and run tests'
     steps:
     - task: NuGetToolInstaller@1
-      displayName: 'Use latest NuGet 6.x'
+      displayName: 'Use the latest NuGet 6.x'
       inputs:
         versionSpec: 6.x
 
     - task: UseDotNet@2
-      displayName: 'Use .Net Core SDK 3.x'
-      inputs:
-        version: 3.x
-
-    - task: UseDotNet@2
-      displayName: 'Use .Net Core sdk 6.x'
+      displayName: 'Use the latest .NET 6'
       inputs:
         version: 6.x
 

diff --git a/build/pipeline-releasebuild-netonly.yaml b/build/pipeline-releasebuild-netonly.yaml
@@ -27,7 +27,7 @@ stages:
         versionSpec: 6.x
 
     - task: UseDotNet@2
-      displayName: 'Use .NET Core SDK 6.x'
+      displayName: 'Use the latest .NET 6'
       inputs:
         version: 6.x
 

diff --git a/build/template-build-and-prep-automation.yaml b/build/template-build-and-prep-automation.yaml
@@ -5,15 +5,8 @@ steps:
   inputs:
     versionSpec: 6.x
 
-# Unit tests require .NET 3.x
 - task: UseDotNet@2
-  displayName: 'Use .Net Core SDK 3.x'
-  inputs:
-    version: 3.x
-
-# Use the latest .NET SDK
-- task: UseDotNet@2
-  displayName: 'Use .Net Core sdk 6.x'
+  displayName: 'Use the latest .NET 6'
   inputs:
     version: 6.x
 

diff --git a/build/template-ci-and-pr.yaml b/build/template-ci-and-pr.yaml
diff --git a/build/template-desktop-unit-and-automation.yaml b/build/template-desktop-unit-and-automation.yaml
@@ -1,19 +1,8 @@
 steps:
 - task: UseDotNet@2
-  displayName: 'Use .Net Core SDK 3.x'
-  inputs:
-    version: 3.x
-
-- task: UseDotNet@2
-  displayName: 'Use .Net Core SDK 5.x'
-  inputs:
-    version: 5.x    
-
-- task: UseDotNet@2
-  displayName: 'Use .Net Core SDK 6.x'
+  displayName: 'Use the latest .NET 6'
   inputs:
     version: 6.x
-    includePreviewVersions: true
 
 - task: DownloadPipelineArtifact@2
   displayName: 'Download Drop'

diff --git a/build/template-install-dotnet-core.yaml b/build/template-install-dotnet-core.yaml
diff --git a/build/template-pack-and-sign-packages.yaml b/build/template-pack-and-sign-packages.yaml
@@ -18,10 +18,9 @@ steps:
 
 # Use the latest .NET SDK
 - task: UseDotNet@2
-  displayName: 'Use .Net Core sdk 6.x'
+  displayName: 'Use the latest .NET 6'
   inputs:
     version: 6.x
-    includePreviewVersions: true
 
 - task: PowerShell@2
   displayName: 'Installing Xamarin iOS/macOS components'
@@ -52,7 +51,6 @@ steps:
     targetType: 'inline'
     script: |
       dotnet nuget locals all --clear 
-      dotnet workload install maui --source https://api.nuget.org/v3/index.json
       dotnet workload install android ios macos maui --source https://api.nuget.org/v3/index.json
 
 - task: JavaToolInstaller@0

diff --git a/build/template-prebuild-code-analysis.yaml b/build/template-prebuild-code-analysis.yaml
@@ -8,7 +8,7 @@ steps:
     versionSpec: '16.x'
 
 - task: UseDotNet@2
-  displayName: 'Use .NET SDK 6.x'
+  displayName: 'Use the latest .NET 6'
   inputs:
     version: 6.x
 

diff --git a/build/template-restore-build-libsandsamples.yaml b/build/template-restore-build-libsandsamples.yaml
@@ -33,8 +33,7 @@ steps:
   inputs:
     targetType: 'inline'
     script: |
-      dotnet workload install maui --source https://api.nuget.org/v3/index.json
-      dotnet workload install android --source https://api.nuget.org/v3/index.json
+      dotnet workload install android ios macos maui --source https://api.nuget.org/v3/index.json
 
 - task: JavaToolInstaller@0
   displayName: 'Use Java 11'

diff --git a/tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj b/tests/Microsoft.Identity.Test.Performance/Microsoft.Identity.Test.Performance.csproj
@@ -2,15 +2,16 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
-    <Configurations>Debug;Release;Debug + MobileApps</Configurations>
+    <TargetFramework>net6.0</TargetFramework>
+    <Configurations>Debug;Release</Configurations>
     <!-- Uncomment only when running EtwProfiler diagnoser on Release-->
     <!-- https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/compiler-options/code-generation#debugtype -->
     <!--<DebugType>pdbonly</DebugType>
     <DebugSymbols>true</DebugSymbols>-->
     <LangVersion>latest</LangVersion>
     <IsPackable>false</IsPackable>
     <GenerateProgramFile>false</GenerateProgramFile>
+    <ManagePackageVersionsCentrally>false</ManagePackageVersionsCentrally>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(UseIdentityWeb)' == 'true'">
@@ -28,13 +29,12 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" />
-    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" />
-    <PackageReference Include="System.CodeDom" />
-  </ItemGroup>
-  <ItemGroup Condition="'$(TargetFramework)' == 'net6.0'">
-    <PackageReference Include="Microsoft.NET.Test.Sdk" />
-    <PackageReference Include="MSTest.TestAdapter" />
-    <PackageReference Include="MSTest.TestFramework" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.12" />
+    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.12" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.1.1" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.1.1" />
+    <PackageReference Update="Microsoft.CodeAnalysis.NetAnalyzers" Version="8.0.0" />
+    <PackageReference Update="Microsoft.VisualStudio.Threading.Analyzers" Version="17.8.14" />
   </ItemGroup>
 </Project>
diff --git a/tests/Microsoft.Identity.Test.Performance/Program.cs b/tests/Microsoft.Identity.Test.Performance/Program.cs
@@ -2,11 +2,13 @@
 // Licensed under the MIT License.
 
 using System;
+using System.Linq;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Order;
+using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 
 namespace Microsoft.Identity.Test.Performance
@@ -19,33 +21,40 @@ public static void Main(string[] args)
 
             try
             {
-                BenchmarkSwitcher.FromTypes(new[] {
+                var results = BenchmarkSwitcher.FromTypes(new[] {
                     typeof(AcquireTokenForClientCacheTests),
                     typeof(AcquireTokenForOboCacheTests),
                     typeof(TokenCacheTests),
                     typeof(AcquireTokenNoCacheTests),
-            }).RunAll(
+                }).RunAll(
 #if DEBUG
-                    new DebugInProcessConfig()
-                        .WithOptions(ConfigOptions.DontOverwriteResults) // When running manually locally
+                    new DebugInProcessConfig() // Allows debugging into benchmarks
+                        .WithOptions(ConfigOptions.DontOverwriteResults) // When debugging locally
 #else
                     DefaultConfig.Instance
+                        .AddJob(Job.Default.WithId("Job-PerfTests"))
 #endif
                         .WithOptions(ConfigOptions.DisableLogFile)
                         .WithOptions(ConfigOptions.StopOnFirstError)
-                        //.WithOptions(ConfigOptions.JoinSummary)
+                        //.WithOptions(ConfigOptions.JoinSummary) // Should be commented for Benchmark GitHub Action to work.
                         .WithOrderer(new DefaultOrderer(SummaryOrderPolicy.Method))
                         .HideColumns(Column.UnrollFactor, Column.Type, Column.InvocationCount, Column.Error, Column.StdDev, Column.Median, Column.Job)
                         .AddDiagnoser(MemoryDiagnoser.Default) // https://benchmarkdotnet.org/articles/configs/diagnosers.html
                         //.AddDiagnoser(new EtwProfiler()) // https://adamsitnik.com/ETW-Profiler/
-                .AddJob(
-                    Job.Default
-                        .WithId("Job-PerfTests")), args);
+                 , args);
+
+                // If no tests ran for whatever reason, throw an exception to break the build.
+                Summary summary = results?.FirstOrDefault();
+                BenchmarkReport report = summary?.Reports.FirstOrDefault();
+                if (summary == null || report == null || !report.Success)
+                {
+                    throw new InvalidOperationException("No performance tests ran.");
+                }
             }
             catch (Exception ex)
             {
-                Logger.Log("Error running performance tests.");
-                Logger.Log(ex.ToString());
+                Logger.LogError("Error running performance tests. See logs for details.");
+                Logger.LogError(ex.ToString());
                 throw;
             }
 
@@ -56,6 +65,12 @@ public static void Main(string[] args)
     public static class Logger
     {
         private const string LogPrefix = "[Test.Performance]";
-        public static void Log(string message) => Console.WriteLine($"{LogPrefix} {message}");
+        public static void Log(string message, ConsoleColor color = ConsoleColor.Blue)
+        {
+            Console.ForegroundColor = color;
+            Console.WriteLine($"{LogPrefix} {message}");
+            Console.ForegroundColor = ConsoleColor.White;
+        }
+        public static void LogError(string message) => Log(message, ConsoleColor.Red);
     }
 }