From f756fef82f11e9eaf01b1b99ef3d2e5b0a285890 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 6 Nov 2024 11:24:01 -0800 Subject: [PATCH 1/4] Refactor func metadata to allow multiple sources --- ...nctionMetadataProviderGenerator.Emitter.cs | 37 ++++--- .../IFunctionMetadataProvider.cs | 1 - .../IFunctionsMetadataSource.cs | 18 ++++ .../Hosting/WorkerOptions.cs | 17 ++++ .../DefaultFunctionMetadataProvider.cs | 98 +++++++++++-------- 5 files changed, 111 insertions(+), 60 deletions(-) create mode 100644 src/DotNetWorker.Core/FunctionMetadata/IFunctionsMetadataSource.cs diff --git a/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs b/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs index 0fa52199e..36c897439 100644 --- a/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs +++ b/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs @@ -25,48 +25,47 @@ public static string Emit(GeneratorExecutionContext context, IReadOnlyList - using System; - using System.Collections.Generic; - using System.Collections.Immutable; - using System.Text.Json; - using System.Threading.Tasks; - using Microsoft.Azure.Functions.Worker; - using Microsoft.Azure.Functions.Worker.Core.FunctionMetadata; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; + using IHostBuilder = global::Microsoft.Extensions.Hosting.IHostBuilder; + using WorkerOptions = global::Microsoft.Azure.Functions.Worker.WorkerOptions; + using IFunctionMetadataSource = global::Microsoft.Azure.Functions.Worker.Core.FunctionMetadata.IFunctionMetadataSource; + using IFunctionMetadata = global::Microsoft.Azure.Functions.Worker.Core.FunctionMetadata.IFunctionMetadata; namespace {{FunctionsUtil.GetNamespaceForGeneratedCode(context)}} { /// - /// Custom implementation that returns function metadata definitions for the current worker."/> + /// Custom implementation that returns function metadata definitions for the current worker."/> /// [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Never)] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - public class GeneratedFunctionMetadataProvider : IFunctionMetadataProvider + public class GeneratedFunctionMetadataSource : IFunctionMetadataSource { - /// - public Task> GetFunctionMetadataAsync(string directory) + public GeneratedFunctionMetadataSource() { - var metadataList = new List(); + var metadataList = new global::System.Collections.Generic.List(); {{functionMetadataInfo}} - return global::System.Threading.Tasks.Task.FromResult(metadataList.ToImmutableArray()); + Metadata = metadataList; } + + public global::System.Collections.Generic.IReadOnlyList Metadata { get; } } /// - /// Extension methods to enable registration of the custom implementation generated for the current worker. + /// Extension methods to enable registration of the custom implementation generated for the current worker. /// - public static class WorkerHostBuilderFunctionMetadataProviderExtension + public static class WorkerHostBuilderFunctionMetadataSourceExtension { /// - /// Adds the GeneratedFunctionMetadataProvider to the service collection. + /// Adds the GeneratedFunctionMetadataSource to the service collection. /// During initialization, the worker will return generated function metadata instead of relying on the Azure Functions host for function indexing. /// - public static IHostBuilder ConfigureGeneratedFunctionMetadataProvider(this IHostBuilder builder) + public static IHostBuilder ConfigureGeneratedFunctionMetadataSource(this IHostBuilder builder) { - builder.ConfigureServices(s => + builder.ConfigureServices(s => { - s.AddSingleton(); + s.TryAddEnumerable(global::Microsoft.Extensions.DependencyInjection.ServiceDescriptor.Singleton(new GeneratedFunctionMetadataSource())); + s.Configure(options => options.Internal.DisableDefaultFunctionMetadata = true); }); return builder; } diff --git a/src/DotNetWorker.Core/FunctionMetadata/IFunctionMetadataProvider.cs b/src/DotNetWorker.Core/FunctionMetadata/IFunctionMetadataProvider.cs index 8e6977fc5..76ed41ede 100644 --- a/src/DotNetWorker.Core/FunctionMetadata/IFunctionMetadataProvider.cs +++ b/src/DotNetWorker.Core/FunctionMetadata/IFunctionMetadataProvider.cs @@ -11,7 +11,6 @@ namespace Microsoft.Azure.Functions.Worker.Core.FunctionMetadata /// public interface IFunctionMetadataProvider { - /// /// Gets all function metadata that this provider knows about asynchronously /// diff --git a/src/DotNetWorker.Core/FunctionMetadata/IFunctionsMetadataSource.cs b/src/DotNetWorker.Core/FunctionMetadata/IFunctionsMetadataSource.cs new file mode 100644 index 000000000..51deaf824 --- /dev/null +++ b/src/DotNetWorker.Core/FunctionMetadata/IFunctionsMetadataSource.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace Microsoft.Azure.Functions.Worker.Core.FunctionMetadata +{ + /// + /// A source for providing metadata. + /// + public interface IFunctionMetadataSource + { + /// + /// Gets the function metadata. + /// + IReadOnlyList Metadata { get; } + } +} diff --git a/src/DotNetWorker.Core/Hosting/WorkerOptions.cs b/src/DotNetWorker.Core/Hosting/WorkerOptions.cs index acd500c8b..bd78e5f7b 100644 --- a/src/DotNetWorker.Core/Hosting/WorkerOptions.cs +++ b/src/DotNetWorker.Core/Hosting/WorkerOptions.cs @@ -24,6 +24,12 @@ public class WorkerOptions /// public InputConverterCollection InputConverters { get; } = new InputConverterCollection(); + /// + /// Gets the internal options. + /// + [Obsolete("This property is for internal use only and not expected to be called by user code.")] + public InternalOptions Internal { get; } = new InternalOptions(); + /// /// Gets the optional worker capabilities. /// @@ -81,5 +87,16 @@ private void SetBoolCapability(string name, bool value) Capabilities.Remove(name); } } + + /// + /// Options for internal configurations. Typically not expected to be set by users. + /// + public class InternalOptions + { + /// + /// Gets or sets a value indicating whether to disable providing default function metadata (from functions.metadata). + /// + public bool DisableDefaultFunctionMetadata { get; set; } + } } } diff --git a/src/DotNetWorker.Grpc/FunctionMetadata/DefaultFunctionMetadataProvider.cs b/src/DotNetWorker.Grpc/FunctionMetadata/DefaultFunctionMetadataProvider.cs index 1d1fc46f7..81fca3c3f 100644 --- a/src/DotNetWorker.Grpc/FunctionMetadata/DefaultFunctionMetadataProvider.cs +++ b/src/DotNetWorker.Grpc/FunctionMetadata/DefaultFunctionMetadataProvider.cs @@ -5,66 +5,40 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.IO; +using System.Linq; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Azure.Functions.Worker.Core.FunctionMetadata; using Microsoft.Azure.Functions.Worker.Grpc.Messages; +using Microsoft.Extensions.Options; namespace Microsoft.Azure.Functions.Worker { internal class DefaultFunctionMetadataProvider : IFunctionMetadataProvider { private const string FileName = "functions.metadata"; - private JsonSerializerOptions deserializationOptions; - public DefaultFunctionMetadataProvider() + private readonly IEnumerable _sources; + private readonly WorkerOptions _options; + private readonly JsonSerializerOptions _deserializationOptions = new() { PropertyNameCaseInsensitive = true }; + + public DefaultFunctionMetadataProvider(IEnumerable sources, IOptions options) { - deserializationOptions = new JsonSerializerOptions(); - deserializationOptions.PropertyNameCaseInsensitive = true; + _sources = sources; + _options = options.Value; } public virtual async Task> GetFunctionMetadataAsync(string directory) { - string metadataFile = Path.Combine(directory, FileName); + ImmutableArray.Builder builder = ImmutableArray.CreateBuilder(); + builder.AddRange(await GetDefaultFunctionMetadataAsync(directory)); - if (!File.Exists(metadataFile)) + foreach (var source in _sources) { - throw new FileNotFoundException($"Function metadata file not found. File path used:{metadataFile}"); + builder.AddRange(source.Metadata); } - using (var fs = File.OpenRead(metadataFile)) - { - // deserialize as json element to preserve raw bindings - var jsonMetadataList = await JsonSerializer.DeserializeAsync(fs); - - var functionMetadataResults = new List(jsonMetadataList.GetArrayLength()); - - foreach (var jsonMetadata in jsonMetadataList.EnumerateArray()) - { - var functionMetadata = JsonSerializer.Deserialize(jsonMetadata.GetRawText(), deserializationOptions); - - if (functionMetadata is null) - { - throw new NullReferenceException("Function metadata could not be found."); - } - - // hard-coded values that are checked for when the host validates functions - functionMetadata.IsProxy = false; - functionMetadata.Language = "dotnet-isolated"; - functionMetadata.FunctionId = Guid.NewGuid().ToString(); - - var rawBindings = GetRawBindings(jsonMetadata); - - foreach (var binding in rawBindings.EnumerateArray()) - { - functionMetadata.RawBindings.Add(binding.GetRawText()); - } - - functionMetadataResults.Add(functionMetadata); - } - - return functionMetadataResults.ToImmutableArray(); - } + return builder.ToImmutable(); } internal static JsonElement GetRawBindings(JsonElement jsonMetadata) @@ -79,5 +53,49 @@ internal static JsonElement GetRawBindings(JsonElement jsonMetadata) return bindingsJson; } + + private async Task> GetDefaultFunctionMetadataAsync(string directory) + { +#pragma warning disable CS0618 // Type or member is obsolete + if (_options.Internal.DisableDefaultFunctionMetadata) + { + return Enumerable.Empty(); + } +#pragma warning restore CS0618 // Type or member is obsolete + + string metadataFile = Path.Combine(directory, FileName); + if (!File.Exists(metadataFile)) + { + throw new FileNotFoundException($"Function metadata file not found. File path used:{metadataFile}"); + } + + using var fs = File.OpenRead(metadataFile); + // deserialize as json element to preserve raw bindings + var jsonMetadataList = await JsonSerializer.DeserializeAsync(fs); + return ParseMetadata(jsonMetadataList); + } + + private IEnumerable ParseMetadata(JsonElement json) + { + foreach (var jsonMetadata in json.EnumerateArray()) + { + var functionMetadata = JsonSerializer.Deserialize(jsonMetadata.GetRawText(), _deserializationOptions) + ?? throw new NullReferenceException("Function metadata could not be found."); + + // hard-coded values that are checked for when the host validates functions + functionMetadata.IsProxy = false; + functionMetadata.Language = "dotnet-isolated"; + functionMetadata.FunctionId = Guid.NewGuid().ToString(); + + var rawBindings = GetRawBindings(jsonMetadata); + + foreach (var binding in rawBindings.EnumerateArray()) + { + functionMetadata.RawBindings.Add(binding.GetRawText()); + } + + yield return functionMetadata; + } + } } } From 9bd9cfa2173d99326fe1ae4ccfcaf158f2b72307 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Wed, 6 Nov 2024 14:22:23 -0800 Subject: [PATCH 2/4] Fix source gen issues --- ...nctionMetadataProviderGenerator.Emitter.cs | 140 ++++++++++-------- .../Hosting/WorkerOptions.cs | 1 + 2 files changed, 76 insertions(+), 65 deletions(-) diff --git a/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs b/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs index 36c897439..fd1246e5c 100644 --- a/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs +++ b/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. using System.Collections.Generic; @@ -24,54 +24,63 @@ public static string Emit(GeneratorExecutionContext context, IReadOnlyList - using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Hosting; - using IHostBuilder = global::Microsoft.Extensions.Hosting.IHostBuilder; - using WorkerOptions = global::Microsoft.Azure.Functions.Worker.WorkerOptions; - using IFunctionMetadataSource = global::Microsoft.Azure.Functions.Worker.Core.FunctionMetadata.IFunctionMetadataSource; - using IFunctionMetadata = global::Microsoft.Azure.Functions.Worker.Core.FunctionMetadata.IFunctionMetadata; - - namespace {{FunctionsUtil.GetNamespaceForGeneratedCode(context)}} - { - /// - /// Custom implementation that returns function metadata definitions for the current worker."/> - /// - [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Never)] - [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - public class GeneratedFunctionMetadataSource : IFunctionMetadataSource - { - public GeneratedFunctionMetadataSource() - { - var metadataList = new global::System.Collections.Generic.List(); - {{functionMetadataInfo}} - Metadata = metadataList; - } - - public global::System.Collections.Generic.IReadOnlyList Metadata { get; } - } - - /// - /// Extension methods to enable registration of the custom implementation generated for the current worker. - /// - public static class WorkerHostBuilderFunctionMetadataSourceExtension - { - /// - /// Adds the GeneratedFunctionMetadataSource to the service collection. - /// During initialization, the worker will return generated function metadata instead of relying on the Azure Functions host for function indexing. - /// - public static IHostBuilder ConfigureGeneratedFunctionMetadataSource(this IHostBuilder builder) - { - builder.ConfigureServices(s => - { - s.TryAddEnumerable(global::Microsoft.Extensions.DependencyInjection.ServiceDescriptor.Singleton(new GeneratedFunctionMetadataSource())); - s.Configure(options => options.Internal.DisableDefaultFunctionMetadata = true); - }); - return builder; - } - }{{GetAutoConfigureStartupClass(includeAutoRegistrationCode)}} - } - """; +// + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using IHostBuilder = global::Microsoft.Extensions.Hosting.IHostBuilder; +using WorkerOptions = global::Microsoft.Azure.Functions.Worker.WorkerOptions; +using IFunctionMetadataSource = global::Microsoft.Azure.Functions.Worker.Core.FunctionMetadata.IFunctionMetadataSource; +using IFunctionMetadata = global::Microsoft.Azure.Functions.Worker.Core.FunctionMetadata.IFunctionMetadata; +using DefaultFunctionMetadata = global::Microsoft.Azure.Functions.Worker.Core.FunctionMetadata.DefaultFunctionMetadata; + +namespace {{FunctionsUtil.GetNamespaceForGeneratedCode(context)}} +{ + /// + /// Custom implementation that returns function metadata definitions for the current worker. + /// + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Never)] + public class GeneratedFunctionMetadataSource : IFunctionMetadataSource + { + /// + /// Initializes a new instance of the class. + /// + public GeneratedFunctionMetadataSource() + { + var metadataList = new global::System.Collections.Generic.List(); +{{functionMetadataInfo}} + Metadata = metadataList; + } + + // + public global::System.Collections.Generic.IReadOnlyList Metadata { get; } + } + + /// + /// Extension methods to enable registration of the custom implementation generated for the current worker. + /// + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + public static class WorkerHostBuilderFunctionMetadataSourceExtension + { + /// + /// Adds the GeneratedFunctionMetadataSource to the service collection. + /// During initialization, the worker will return generated function metadata instead of relying on the Azure Functions host for function indexing. + /// + public static IHostBuilder ConfigureGeneratedFunctionMetadataProvider(this IHostBuilder builder) + { + builder.ConfigureServices(s => + { + s.TryAddEnumerable(global::Microsoft.Extensions.DependencyInjection.ServiceDescriptor.Singleton(new GeneratedFunctionMetadataSource())); + s.Configure(options => options.Internal.DisableDefaultFunctionMetadata = true); + }); + return builder; + } + } +{{GetAutoConfigureStartupClass(includeAutoRegistrationCode)}} +} + +"""; } private static string GetAutoConfigureStartupClass(bool includeAutoRegistrationCode) @@ -80,22 +89,23 @@ private static string GetAutoConfigureStartupClass(bool includeAutoRegistrationC { string result = $$""" - /// - /// Auto startup class to register the custom implementation generated for the current worker. - /// - [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Never)] - public class FunctionMetadataProviderAutoStartup : global::Microsoft.Azure.Functions.Worker.IAutoConfigureStartup - { - /// - /// Configures the to use the custom implementation generated for the current worker. - /// - /// The instance to use for service registration. - public void Configure(IHostBuilder hostBuilder) - { - hostBuilder.ConfigureGeneratedFunctionMetadataProvider(); - } - } - """; + /// + /// Auto startup class to register the custom implementation generated for the current worker. + /// + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Never)] + public class FunctionMetadataProviderAutoStartup : global::Microsoft.Azure.Functions.Worker.IAutoConfigureStartup + { + /// + /// Configures the to use the custom implementation generated for the current worker. + /// + /// The instance to use for service registration. + public void Configure(IHostBuilder hostBuilder) + { + hostBuilder.ConfigureGeneratedFunctionMetadataProvider(); + } + } +"""; return result; } diff --git a/src/DotNetWorker.Core/Hosting/WorkerOptions.cs b/src/DotNetWorker.Core/Hosting/WorkerOptions.cs index bd78e5f7b..7a7486aeb 100644 --- a/src/DotNetWorker.Core/Hosting/WorkerOptions.cs +++ b/src/DotNetWorker.Core/Hosting/WorkerOptions.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; using System.Collections.Generic; using System.Text.Json; using Azure.Core.Serialization; From e1a58d671279cdb850af8f531163ede34f2cbf15 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 7 Nov 2024 09:24:08 -0800 Subject: [PATCH 3/4] Add IFunctionMetadataProvider interface back to generated provider --- ...nctionMetadataProviderGenerator.Emitter.cs | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs b/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs index fd1246e5c..8fe1105b2 100644 --- a/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs +++ b/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs @@ -23,6 +23,7 @@ public static string Emit(GeneratorExecutionContext context, IReadOnlyList @@ -41,37 +42,43 @@ namespace {{FunctionsUtil.GetNamespaceForGeneratedCode(context)}} /// [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Never)] - public class GeneratedFunctionMetadataSource : IFunctionMetadataSource + public class GeneratedFunctionMetadataProvider : IFunctionMetadataProvider, IFunctionMetadataSource { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public GeneratedFunctionMetadataSource() + public GeneratedFunctionMetadataProvider() { var metadataList = new global::System.Collections.Generic.List(); {{functionMetadataInfo}} Metadata = metadataList; } - // + /// public global::System.Collections.Generic.IReadOnlyList Metadata { get; } + + /// + public global::System.Threading.Tasks.Task> GetFunctionMetadataAsync(string directory) + { + return global::System.Threading.Tasks.Task.FromResult(global::System.Collections.Immutable.ImmutableArray.CreateRange(Metadata)); + } } /// /// Extension methods to enable registration of the custom implementation generated for the current worker. /// [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - public static class WorkerHostBuilderFunctionMetadataSourceExtension + public static class WorkerHostBuilderFunctionMetadataProviderExtension { /// - /// Adds the GeneratedFunctionMetadataSource to the service collection. + /// Adds the GeneratedFunctionMetadataProvider to the service collection. /// During initialization, the worker will return generated function metadata instead of relying on the Azure Functions host for function indexing. /// public static IHostBuilder ConfigureGeneratedFunctionMetadataProvider(this IHostBuilder builder) { builder.ConfigureServices(s => { - s.TryAddEnumerable(global::Microsoft.Extensions.DependencyInjection.ServiceDescriptor.Singleton(new GeneratedFunctionMetadataSource())); + s.TryAddEnumerable(global::Microsoft.Extensions.DependencyInjection.ServiceDescriptor.Singleton(new GeneratedFunctionMetadataProvider())); s.Configure(options => options.Internal.DisableDefaultFunctionMetadata = true); }); return builder; From b2ec904080d727a693e2592926bb60f150355579 Mon Sep 17 00:00:00 2001 From: Jacob Viau Date: Thu, 7 Nov 2024 09:28:13 -0800 Subject: [PATCH 4/4] Add ObsoleteAttribute to generated method --- .../FunctionMetadataProviderGenerator.Emitter.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs b/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs index 8fe1105b2..8aeb02258 100644 --- a/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs +++ b/sdk/Sdk.Generators/FunctionMetadataProviderGenerator/FunctionMetadataProviderGenerator.Emitter.cs @@ -58,6 +58,7 @@ public GeneratedFunctionMetadataProvider() public global::System.Collections.Generic.IReadOnlyList Metadata { get; } /// + [global::System.Runtime.ObsoleteAttribute("IFunctionMetadataProvider.GetFunctionMetadataAsync is no longer used for function indexing with this class. Use IFunctionMetadataSource.Metadata instead.")] public global::System.Threading.Tasks.Task> GetFunctionMetadataAsync(string directory) { return global::System.Threading.Tasks.Task.FromResult(global::System.Collections.Immutable.ImmutableArray.CreateRange(Metadata));