Skip to content

Commit

Permalink
Adding support for multi-dimensional metrics (#3052)
Browse files Browse the repository at this point in the history 
* Adding support for MutiDimensional Metrics
  • Loading branch information
@RohitRanjanMS
RohitRanjanMS authored Dec 29, 2023
1 parent bb79ef3 commit ca14c11
Show file tree
Hide file tree
Showing 7 changed files with 149 additions and 62 deletions.
18 changes: 14 additions & 4 deletions src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLogger.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,12 @@ private void LogMetric(IEnumerable<KeyValuePair<string, object>> values, LogLeve
// Always apply scope first to allow state to override.
ApplyScopeProperties(telemetry);

// Add known properties like category, logLevel and event
ApplyKnownProperties(telemetry.Properties, logLevel, eventId);
// Do not apply state properties if optimization is enabled
if (!_loggerOptions.EnableMetricsCustomDimensionOptimization)
{
// Add known properties like category, logLevel and event
ApplyKnownProperties(telemetry.Properties, logLevel, eventId);
}

foreach (var entry in values)
{
Expand Down Expand Up @@ -357,8 +361,14 @@ private void LogFunctionResultAggregate(IEnumerable<KeyValuePair<string, object>
}
}

IDictionary<string, string> properties = new Dictionary<string, string>(2);
ApplyKnownProperties(properties, logLevel, eventId);
IDictionary<string, string> properties = null;

// Do not apply state properties if optimization is enabled
if (!_loggerOptions.EnableMetricsCustomDimensionOptimization)
{
properties = new Dictionary<string, string>(2);
ApplyKnownProperties(properties, logLevel, eventId);
}

foreach (KeyValuePair<string, double> metric in metrics)
{
Expand Down
9 changes: 8 additions & 1 deletion src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/ApplicationInsightsLoggerOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,13 @@ public TimeSpan QuickPulseInitializationDelay
/// Gets or sets the flag that enables standard metrics collection in ApplicationInsights.
/// Disabled by default.
/// </summary>
public bool EnableAutocollectedMetricsExtractor { get; set; } = false;
public bool EnableAutocollectedMetricsExtractor { get; set; } = false;

/// <summary>
/// Gets or sets the flag that bypass custom dimensions in Metrics telemetry.
/// Disabled by default.
/// </summary>
public bool EnableMetricsCustomDimensionOptimization { get; set; } = false;

public string Format()
{
Expand Down Expand Up @@ -239,6 +245,7 @@ public string Format()
{ nameof(TokenCredentialOptions), tokenCredentialOptions },
{ nameof(DiagnosticsEventListenerLogLevel), DiagnosticsEventListenerLogLevel?.ToString() },
{ nameof(EnableAutocollectedMetricsExtractor), EnableAutocollectedMetricsExtractor },
{ nameof(EnableMetricsCustomDimensionOptimization), EnableMetricsCustomDimensionOptimization },
};

return options.ToString(Formatting.Indented);
Expand Down
1 change: 1 addition & 0 deletions src/Microsoft.Azure.WebJobs.Logging.ApplicationInsights/Constants/LoggingConstants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@ internal static class LoggingConstants
public const string ZeroIpAddress = "0.0.0.0";
public const string Unknown = "[Unknown]";
public const string ClientIpKey = "ClientIp";
public const string HostInstanceIdKey = "HostInstanceId";
}
}
107 changes: 59 additions & 48 deletions ...oft.Azure.WebJobs.Logging.ApplicationInsights/Initializers/WebJobsTelemetryInitializer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,75 +52,86 @@ public void Initialize(ITelemetry telemetry)
{
telemetryContext.Location.Ip = LoggingConstants.ZeroIpAddress;
}
telemetryContext.Properties[LogConstants.ProcessIdKey] = _currentProcessId;

// Apply our special scope properties
IReadOnlyDictionary<string, object> scopeProps = DictionaryLoggerScope.GetMergedStateDictionaryOrNull();

// this could be telemetry tracked in scope of function call - then we should apply the logger scope
// or RequestTelemetry tracked by the WebJobs SDK or AppInsight SDK - then we should apply Activity.Tags
if (scopeProps?.Count > 0)

// Do not apply state properties if optimization is enabled.
if (_options.EnableMetricsCustomDimensionOptimization && telemetry is MetricTelemetry)
{
// Remove the Host instance ID property, since it's not needed.
telemetryContext.Properties.Remove(LoggingConstants.HostInstanceIdKey);
return;
}
else
{
if (!telemetryContext.Properties.ContainsKey(LogConstants.InvocationIdKey))
telemetryContext.Properties[LogConstants.ProcessIdKey] = _currentProcessId;

// Apply our special scope properties
IReadOnlyDictionary<string, object> scopeProps = DictionaryLoggerScope.GetMergedStateDictionaryOrNull();

// this could be telemetry tracked in scope of function call - then we should apply the logger scope
// or RequestTelemetry tracked by the WebJobs SDK or AppInsight SDK - then we should apply Activity.Tags
if (scopeProps?.Count > 0)
{
if (scopeProps?.GetValueOrDefault<string>(ScopeKeys.FunctionInvocationId) is string invocationId)
if (!telemetryContext.Properties.ContainsKey(LogConstants.InvocationIdKey))
{
telemetryContext.Properties[LogConstants.InvocationIdKey] = invocationId;
if (scopeProps?.GetValueOrDefault<string>(ScopeKeys.FunctionInvocationId) is string invocationId)
{
telemetryContext.Properties[LogConstants.InvocationIdKey] = invocationId;
}
}
}

telemetryContext.Operation.Name = scopeProps.GetValueOrDefault<string>(ScopeKeys.FunctionName);

telemetryContext.Operation.Name = scopeProps.GetValueOrDefault<string>(ScopeKeys.FunctionName);

// Apply Category, LogLevel event details to all telemetry
if (!telemetryContext.Properties.ContainsKey(LogConstants.CategoryNameKey))
{
if (scopeProps.GetValueOrDefault<string>(LogConstants.CategoryNameKey) is string category)
// Apply Category, LogLevel event details to all telemetry
if (!telemetryContext.Properties.ContainsKey(LogConstants.CategoryNameKey))
{
telemetryContext.Properties[LogConstants.CategoryNameKey] = category;
if (scopeProps.GetValueOrDefault<string>(LogConstants.CategoryNameKey) is string category)
{
telemetryContext.Properties[LogConstants.CategoryNameKey] = category;
}
}
}

if (!telemetryContext.Properties.ContainsKey(LogConstants.LogLevelKey))
{
if (scopeProps.GetValueOrDefault<LogLevel?>(LogConstants.LogLevelKey) is LogLevel logLevel)
if (!telemetryContext.Properties.ContainsKey(LogConstants.LogLevelKey))
{
telemetryContext.Properties[LogConstants.LogLevelKey] = logLevel.ToStringOptimized();
if (scopeProps.GetValueOrDefault<LogLevel?>(LogConstants.LogLevelKey) is LogLevel logLevel)
{
telemetryContext.Properties[LogConstants.LogLevelKey] = logLevel.ToStringOptimized();
}
}
}

if (!telemetryContext.Properties.ContainsKey(LogConstants.EventIdKey))
{
if (scopeProps.GetValueOrDefault<int>(LogConstants.EventIdKey) is int eventId && eventId != 0)
if (!telemetryContext.Properties.ContainsKey(LogConstants.EventIdKey))
{
telemetryContext.Properties[LogConstants.EventIdKey] = eventId.ToString(CultureInfo.InvariantCulture);
if (scopeProps.GetValueOrDefault<int>(LogConstants.EventIdKey) is int eventId && eventId != 0)
{
telemetryContext.Properties[LogConstants.EventIdKey] = eventId.ToString(CultureInfo.InvariantCulture);
}
}
}

if (!telemetryContext.Properties.ContainsKey(LogConstants.EventNameKey))
{
if (scopeProps.GetValueOrDefault<string>(LogConstants.EventNameKey) is string eventName)
if (!telemetryContext.Properties.ContainsKey(LogConstants.EventNameKey))
{
telemetryContext.Properties[LogConstants.EventNameKey] = eventName;
if (scopeProps.GetValueOrDefault<string>(LogConstants.EventNameKey) is string eventName)
{
telemetryContext.Properties[LogConstants.EventNameKey] = eventName;
}
}
}
}

// we may track traces/dependencies after function scope ends - we don't want to update those
if (telemetry is RequestTelemetry request)
{
UpdateRequestProperties(request);

Activity currentActivity = Activity.Current;
if (currentActivity != null)
// we may track traces/dependencies after function scope ends - we don't want to update those
if (telemetry is RequestTelemetry request)
{
foreach (var tag in currentActivity.Tags)
UpdateRequestProperties(request);

Activity currentActivity = Activity.Current;
if (currentActivity != null)
{
// Apply well-known tags and custom properties,
// but ignore internal ai tags
if (!TryApplyProperty(request, tag) &&
!tag.Key.StartsWith("ai_"))
foreach (var tag in currentActivity.Tags)
{
request.Properties[tag.Key] = tag.Value;
// Apply well-known tags and custom properties,
// but ignore internal ai tags
if (!TryApplyProperty(request, tag) &&
!tag.Key.StartsWith("ai_"))
{
request.Properties[tag.Key] = tag.Value;
}
}
}
}
Expand Down
14 changes: 7 additions & 7 deletions ...soft.Azure.WebJobs.Logging.ApplicationInsights/WebJobs.Logging.ApplicationInsights.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,16 @@

<ItemGroup>
<PackageReference Include="Azure.Identity" Version="1.10.4" />
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
<PackageReference Include="Microsoft.ApplicationInsights.PerfCounterCollector" Version="2.21.0" />
<PackageReference Include="Microsoft.ApplicationInsights.SnapshotCollector" Version="1.4.3" />
<PackageReference Include="Microsoft.ApplicationInsights.WindowsServer" Version="2.21.0" />
<PackageReference Include="Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel" Version="2.21.0" />
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />
<PackageReference Include="Microsoft.ApplicationInsights.PerfCounterCollector" Version="2.22.0" />
<PackageReference Include="Microsoft.ApplicationInsights.SnapshotCollector" Version="1.4.4" />
<PackageReference Include="Microsoft.ApplicationInsights.WindowsServer" Version="2.22.0" />
<PackageReference Include="Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel" Version="2.22.0" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.0-beta004">
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Microsoft.ApplicationInsights" Version="2.21.0" />
<PackageReference Include="Microsoft.ApplicationInsights.DependencyCollector" Version="2.21.0" />
<PackageReference Include="Microsoft.ApplicationInsights" Version="2.22.0" />
<PackageReference Include="Microsoft.ApplicationInsights.DependencyCollector" Version="2.22.0" />
<PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="2.1.1" />
<PackageReference Include="System.Diagnostics.TraceSource" Version="4.3.0" />
Expand Down
61 changes: 59 additions & 2 deletions ....Azure.WebJobs.Host.EndToEndTests/ApplicationInsights/ApplicationInsightsEndToEndTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ private IHost ConfigureHost(
o.EnableQueryStringTracing = applicationInsightsOptions.EnableQueryStringTracing;
o.EnableLiveMetricsFilters = applicationInsightsOptions.EnableLiveMetricsFilters;
o.EnableLiveMetrics = applicationInsightsOptions.EnableLiveMetrics;
o.EnableMetricsCustomDimensionOptimization = applicationInsightsOptions.EnableMetricsCustomDimensionOptimization;
}
});
})
Expand Down Expand Up @@ -855,7 +856,7 @@ public async Task ApplicationInsights_EventHubTrackingByWebJobs_SingleLink(bool
traceId,
parentSpanId);

Assert.Equal(sampledIn ? SamplingDecision.SampledIn : SamplingDecision.None, functionRequest.ProactiveSamplingDecision);
Assert.Equal(SamplingDecision.None, functionRequest.ProactiveSamplingDecision);

// Make sure operation ids match
var traces = _channel.Telemetries.OfType<TraceTelemetry>().Where(t => t.Context.Operation.Id == functionRequest.Context.Operation.Id);
Expand Down Expand Up @@ -961,7 +962,7 @@ public async Task ApplicationInsights_EventHubTrackingByWebJobs_MultipleLinks(bo

foreach (var trace in traces)
{
Assert.Equal(anySampledIn ? SamplingDecision.SampledIn : SamplingDecision.None, trace.ProactiveSamplingDecision);
Assert.Equal(SamplingDecision.None, trace.ProactiveSamplingDecision);
}

Assert.Equal(anySampledIn ? SamplingDecision.SampledIn : SamplingDecision.None,
Expand Down Expand Up @@ -1312,6 +1313,62 @@ await TestHelpers.Await(() =>
}
}

[Theory]
[InlineData(true, false)]
[InlineData(false, true)]
public async Task ApplicationInsights_CustomDimensionOptimization(bool customDimensionOptimization, bool customDimension)
{
string testName = nameof(TestApplicationInsightsInformation);
using (IHost host = ConfigureHost(applicationInsightsOptions: new ApplicationInsightsLoggerOptions { EnableMetricsCustomDimensionOptimization = customDimensionOptimization }))
{
await host.StartAsync();
MethodInfo methodInfo = GetType().GetMethod(testName, BindingFlags.Public | BindingFlags.Static);
await host.GetJobHost().CallAsync(methodInfo, new { input = "function input" });
await host.StopAsync();

var metrics = _channel.Telemetries
.OfType<MetricTelemetry>();
if (!customDimension)
{
foreach (var item in metrics)
{
Assert.Equal(3, item.Properties.Count);
Assert.True(item.Properties.Keys.Contains("prop__MyCustomMetricProperty"));
Assert.True(item.Properties.Keys.Contains("prop__MyCustomScopeKey"));
Assert.True(item.Properties.Keys.Contains("InvocationId"));
}
}
else
{
foreach (var item in metrics)
{
Assert.Equal(7, item.Properties.Count);
}
}
}
}

[Fact]
public async Task ApplicationInsights_CustomDimensionOptimization_Default()
{
string testName = nameof(TestApplicationInsightsInformation);
using (IHost host = ConfigureHost())
{
await host.StartAsync();
MethodInfo methodInfo = GetType().GetMethod(testName, BindingFlags.Public | BindingFlags.Static);
await host.GetJobHost().CallAsync(methodInfo, new { input = "function input" });
await host.StopAsync();

var metrics = _channel.Telemetries
.OfType<MetricTelemetry>();

foreach (var item in metrics)
{
Assert.Equal(7, item.Properties.Count);
}
}
}

// Test Functions
[NoAutomaticTrigger]
public static void TestApplicationInsightsInformation(string input, TraceWriter trace, ILogger logger)
Expand Down
1 change: 1 addition & 0 deletions test/Microsoft.Azure.WebJobs.Host.UnitTests/Loggers/ApplicationInsightsLoggerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -777,6 +777,7 @@ public void ApplicationInsightsLoggerOptions_Format()
Assert.Equal("*******", deserializedOptions.TokenCredentialOptions.ClientId);
Assert.Equal(options.EnableLiveMetrics, deserializedOptions.EnableLiveMetrics);
Assert.Equal(options.EnableLiveMetricsFilters, deserializedOptions.EnableLiveMetricsFilters);
Assert.Equal(false, deserializedOptions.EnableMetricsCustomDimensionOptimization);
}

[Fact]
Expand Down

0 comments on commit ca14c11

Please sign in to comment.