From ca14c114a61bd3693f629c89063655fc6ee2c522 Mon Sep 17 00:00:00 2001 From: Rohit Ranjan <90008725+RohitRanjanMS@users.noreply.github.com> Date: Thu, 28 Dec 2023 17:01:22 -0800 Subject: [PATCH] Adding support for multi-dimensional metrics (#3052) * Adding support for MutiDimensional Metrics --- .../ApplicationInsightsLogger.cs | 18 ++- .../ApplicationInsightsLoggerOptions.cs | 9 +- .../Constants/LoggingConstants.cs | 1 + .../WebJobsTelemetryInitializer.cs | 107 ++++++++++-------- ...WebJobs.Logging.ApplicationInsights.csproj | 14 +-- .../ApplicationInsightsEndToEndTests.cs | 61 +++++++++- .../Loggers/ApplicationInsightsLoggerTests.cs | 1 + 7 files changed, 149 insertions(+), 62 deletions(-) diff --git a/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLogger.cs b/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLogger.cs index 9cffb7565..ed8d0c2ea 100644 --- a/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLogger.cs +++ b/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLogger.cs @@ -112,8 +112,12 @@ private void LogMetric(IEnumerable> values, LogLeve // Always apply scope first to allow state to override. ApplyScopeProperties(telemetry); - // Add known properties like category, logLevel and event - ApplyKnownProperties(telemetry.Properties, logLevel, eventId); + // Do not apply state properties if optimization is enabled + if (!_loggerOptions.EnableMetricsCustomDimensionOptimization) + { + // Add known properties like category, logLevel and event + ApplyKnownProperties(telemetry.Properties, logLevel, eventId); + } foreach (var entry in values) { @@ -357,8 +361,14 @@ private void LogFunctionResultAggregate(IEnumerable } } - IDictionary properties = new Dictionary(2); - ApplyKnownProperties(properties, logLevel, eventId); + IDictionary properties = null; + + // Do not apply state properties if optimization is enabled + if (!_loggerOptions.EnableMetricsCustomDimensionOptimization) + { + properties = new Dictionary(2); + ApplyKnownProperties(properties, logLevel, eventId); + } foreach (KeyValuePair metric in metrics) { diff --git a/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLoggerOptions.cs b/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLoggerOptions.cs index a23bb2337..08c370923 100644 --- a/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLoggerOptions.cs +++ b/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLoggerOptions.cs @@ -147,7 +147,13 @@ public TimeSpan QuickPulseInitializationDelay /// Gets or sets the flag that enables standard metrics collection in ApplicationInsights. /// Disabled by default. /// - public bool EnableAutocollectedMetricsExtractor { get; set; } = false; + public bool EnableAutocollectedMetricsExtractor { get; set; } = false; + + /// + /// Gets or sets the flag that bypass custom dimensions in Metrics telemetry. + /// Disabled by default. + /// + public bool EnableMetricsCustomDimensionOptimization { get; set; } = false; public string Format() { @@ -239,6 +245,7 @@ public string Format() { nameof(TokenCredentialOptions), tokenCredentialOptions }, { nameof(DiagnosticsEventListenerLogLevel), DiagnosticsEventListenerLogLevel?.ToString() }, { nameof(EnableAutocollectedMetricsExtractor), EnableAutocollectedMetricsExtractor }, + { nameof(EnableMetricsCustomDimensionOptimization), EnableMetricsCustomDimensionOptimization }, }; return options.ToString(Formatting.Indented); diff --git a/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Constants/LoggingConstants.cs b/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Constants/LoggingConstants.cs index 4d6cd0908..6d63ed834 100644 --- a/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Constants/LoggingConstants.cs +++ b/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Constants/LoggingConstants.cs @@ -8,5 +8,6 @@ internal static class LoggingConstants public const string ZeroIpAddress = "0.0.0.0"; public const string Unknown = "[Unknown]"; public const string ClientIpKey = "ClientIp"; + public const string HostInstanceIdKey = "HostInstanceId"; } } diff --git a/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Initializers/WebJobsTelemetryInitializer.cs b/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Initializers/WebJobsTelemetryInitializer.cs index 3ba8c2299..56d7163e0 100644 --- a/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Initializers/WebJobsTelemetryInitializer.cs +++ b/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Initializers/WebJobsTelemetryInitializer.cs @@ -52,75 +52,86 @@ public void Initialize(ITelemetry telemetry) { telemetryContext.Location.Ip = LoggingConstants.ZeroIpAddress; } - telemetryContext.Properties[LogConstants.ProcessIdKey] = _currentProcessId; - - // Apply our special scope properties - IReadOnlyDictionary scopeProps = DictionaryLoggerScope.GetMergedStateDictionaryOrNull(); - - // this could be telemetry tracked in scope of function call - then we should apply the logger scope - // or RequestTelemetry tracked by the WebJobs SDK or AppInsight SDK - then we should apply Activity.Tags - if (scopeProps?.Count > 0) + + // Do not apply state properties if optimization is enabled. + if (_options.EnableMetricsCustomDimensionOptimization && telemetry is MetricTelemetry) + { + // Remove the Host instance ID property, since it's not needed. + telemetryContext.Properties.Remove(LoggingConstants.HostInstanceIdKey); + return; + } + else { - if (!telemetryContext.Properties.ContainsKey(LogConstants.InvocationIdKey)) + telemetryContext.Properties[LogConstants.ProcessIdKey] = _currentProcessId; + + // Apply our special scope properties + IReadOnlyDictionary scopeProps = DictionaryLoggerScope.GetMergedStateDictionaryOrNull(); + + // this could be telemetry tracked in scope of function call - then we should apply the logger scope + // or RequestTelemetry tracked by the WebJobs SDK or AppInsight SDK - then we should apply Activity.Tags + if (scopeProps?.Count > 0) { - if (scopeProps?.GetValueOrDefault(ScopeKeys.FunctionInvocationId) is string invocationId) + if (!telemetryContext.Properties.ContainsKey(LogConstants.InvocationIdKey)) { - telemetryContext.Properties[LogConstants.InvocationIdKey] = invocationId; + if (scopeProps?.GetValueOrDefault(ScopeKeys.FunctionInvocationId) is string invocationId) + { + telemetryContext.Properties[LogConstants.InvocationIdKey] = invocationId; + } } - } - - telemetryContext.Operation.Name = scopeProps.GetValueOrDefault(ScopeKeys.FunctionName); + + telemetryContext.Operation.Name = scopeProps.GetValueOrDefault(ScopeKeys.FunctionName); - // Apply Category, LogLevel event details to all telemetry - if (!telemetryContext.Properties.ContainsKey(LogConstants.CategoryNameKey)) - { - if (scopeProps.GetValueOrDefault(LogConstants.CategoryNameKey) is string category) + // Apply Category, LogLevel event details to all telemetry + if (!telemetryContext.Properties.ContainsKey(LogConstants.CategoryNameKey)) { - telemetryContext.Properties[LogConstants.CategoryNameKey] = category; + if (scopeProps.GetValueOrDefault(LogConstants.CategoryNameKey) is string category) + { + telemetryContext.Properties[LogConstants.CategoryNameKey] = category; + } } - } - if (!telemetryContext.Properties.ContainsKey(LogConstants.LogLevelKey)) - { - if (scopeProps.GetValueOrDefault(LogConstants.LogLevelKey) is LogLevel logLevel) + if (!telemetryContext.Properties.ContainsKey(LogConstants.LogLevelKey)) { - telemetryContext.Properties[LogConstants.LogLevelKey] = logLevel.ToStringOptimized(); + if (scopeProps.GetValueOrDefault(LogConstants.LogLevelKey) is LogLevel logLevel) + { + telemetryContext.Properties[LogConstants.LogLevelKey] = logLevel.ToStringOptimized(); + } } - } - if (!telemetryContext.Properties.ContainsKey(LogConstants.EventIdKey)) - { - if (scopeProps.GetValueOrDefault(LogConstants.EventIdKey) is int eventId && eventId != 0) + if (!telemetryContext.Properties.ContainsKey(LogConstants.EventIdKey)) { - telemetryContext.Properties[LogConstants.EventIdKey] = eventId.ToString(CultureInfo.InvariantCulture); + if (scopeProps.GetValueOrDefault(LogConstants.EventIdKey) is int eventId && eventId != 0) + { + telemetryContext.Properties[LogConstants.EventIdKey] = eventId.ToString(CultureInfo.InvariantCulture); + } } - } - if (!telemetryContext.Properties.ContainsKey(LogConstants.EventNameKey)) - { - if (scopeProps.GetValueOrDefault(LogConstants.EventNameKey) is string eventName) + if (!telemetryContext.Properties.ContainsKey(LogConstants.EventNameKey)) { - telemetryContext.Properties[LogConstants.EventNameKey] = eventName; + if (scopeProps.GetValueOrDefault(LogConstants.EventNameKey) is string eventName) + { + telemetryContext.Properties[LogConstants.EventNameKey] = eventName; + } } } - } - // we may track traces/dependencies after function scope ends - we don't want to update those - if (telemetry is RequestTelemetry request) - { - UpdateRequestProperties(request); - - Activity currentActivity = Activity.Current; - if (currentActivity != null) + // we may track traces/dependencies after function scope ends - we don't want to update those + if (telemetry is RequestTelemetry request) { - foreach (var tag in currentActivity.Tags) + UpdateRequestProperties(request); + + Activity currentActivity = Activity.Current; + if (currentActivity != null) { - // Apply well-known tags and custom properties, - // but ignore internal ai tags - if (!TryApplyProperty(request, tag) && - !tag.Key.StartsWith("ai_")) + foreach (var tag in currentActivity.Tags) { - request.Properties[tag.Key] = tag.Value; + // Apply well-known tags and custom properties, + // but ignore internal ai tags + if (!TryApplyProperty(request, tag) && + !tag.Key.StartsWith("ai_")) + { + request.Properties[tag.Key] = tag.Value; + } } } } diff --git a/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/WebJobs.Logging.ApplicationInsights.csproj b/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/WebJobs.Logging.ApplicationInsights.csproj index 1d42f61b2..43c00b7f3 100644 --- a/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/WebJobs.Logging.ApplicationInsights.csproj +++ b/src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/WebJobs.Logging.ApplicationInsights.csproj @@ -28,16 +28,16 @@ - - - - - + + + + + all - - + + diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/ApplicationInsights/ApplicationInsightsEndToEndTests.cs b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/ApplicationInsights/ApplicationInsightsEndToEndTests.cs index 06f51169a..189f8c4bd 100644 --- a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/ApplicationInsights/ApplicationInsightsEndToEndTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/ApplicationInsights/ApplicationInsightsEndToEndTests.cs @@ -104,6 +104,7 @@ private IHost ConfigureHost( o.EnableQueryStringTracing = applicationInsightsOptions.EnableQueryStringTracing; o.EnableLiveMetricsFilters = applicationInsightsOptions.EnableLiveMetricsFilters; o.EnableLiveMetrics = applicationInsightsOptions.EnableLiveMetrics; + o.EnableMetricsCustomDimensionOptimization = applicationInsightsOptions.EnableMetricsCustomDimensionOptimization; } }); }) @@ -855,7 +856,7 @@ public async Task ApplicationInsights_EventHubTrackingByWebJobs_SingleLink(bool traceId, parentSpanId); - Assert.Equal(sampledIn ? SamplingDecision.SampledIn : SamplingDecision.None, functionRequest.ProactiveSamplingDecision); + Assert.Equal(SamplingDecision.None, functionRequest.ProactiveSamplingDecision); // Make sure operation ids match var traces = _channel.Telemetries.OfType().Where(t => t.Context.Operation.Id == functionRequest.Context.Operation.Id); @@ -961,7 +962,7 @@ public async Task ApplicationInsights_EventHubTrackingByWebJobs_MultipleLinks(bo foreach (var trace in traces) { - Assert.Equal(anySampledIn ? SamplingDecision.SampledIn : SamplingDecision.None, trace.ProactiveSamplingDecision); + Assert.Equal(SamplingDecision.None, trace.ProactiveSamplingDecision); } Assert.Equal(anySampledIn ? SamplingDecision.SampledIn : SamplingDecision.None, @@ -1312,6 +1313,62 @@ await TestHelpers.Await(() => } } + [Theory] + [InlineData(true, false)] + [InlineData(false, true)] + public async Task ApplicationInsights_CustomDimensionOptimization(bool customDimensionOptimization, bool customDimension) + { + string testName = nameof(TestApplicationInsightsInformation); + using (IHost host = ConfigureHost(applicationInsightsOptions: new ApplicationInsightsLoggerOptions { EnableMetricsCustomDimensionOptimization = customDimensionOptimization })) + { + await host.StartAsync(); + MethodInfo methodInfo = GetType().GetMethod(testName, BindingFlags.Public | BindingFlags.Static); + await host.GetJobHost().CallAsync(methodInfo, new { input = "function input" }); + await host.StopAsync(); + + var metrics = _channel.Telemetries + .OfType(); + if (!customDimension) + { + foreach (var item in metrics) + { + Assert.Equal(3, item.Properties.Count); + Assert.True(item.Properties.Keys.Contains("prop__MyCustomMetricProperty")); + Assert.True(item.Properties.Keys.Contains("prop__MyCustomScopeKey")); + Assert.True(item.Properties.Keys.Contains("InvocationId")); + } + } + else + { + foreach (var item in metrics) + { + Assert.Equal(7, item.Properties.Count); + } + } + } + } + + [Fact] + public async Task ApplicationInsights_CustomDimensionOptimization_Default() + { + string testName = nameof(TestApplicationInsightsInformation); + using (IHost host = ConfigureHost()) + { + await host.StartAsync(); + MethodInfo methodInfo = GetType().GetMethod(testName, BindingFlags.Public | BindingFlags.Static); + await host.GetJobHost().CallAsync(methodInfo, new { input = "function input" }); + await host.StopAsync(); + + var metrics = _channel.Telemetries + .OfType(); + + foreach (var item in metrics) + { + Assert.Equal(7, item.Properties.Count); + } + } + } + // Test Functions [NoAutomaticTrigger] public static void TestApplicationInsightsInformation(string input, TraceWriter trace, ILogger logger) diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Loggers/ApplicationInsightsLoggerTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Loggers/ApplicationInsightsLoggerTests.cs index bd0aafd9a..cbf8a35f8 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Loggers/ApplicationInsightsLoggerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Loggers/ApplicationInsightsLoggerTests.cs @@ -777,6 +777,7 @@ public void ApplicationInsightsLoggerOptions_Format() Assert.Equal("*******", deserializedOptions.TokenCredentialOptions.ClientId); Assert.Equal(options.EnableLiveMetrics, deserializedOptions.EnableLiveMetrics); Assert.Equal(options.EnableLiveMetricsFilters, deserializedOptions.EnableLiveMetricsFilters); + Assert.Equal(false, deserializedOptions.EnableMetricsCustomDimensionOptimization); } [Fact]