diff --git a/CustomDictionary.xml b/CustomDictionary.xml index 5fada7ba9..7674556d3 100644 --- a/CustomDictionary.xml +++ b/CustomDictionary.xml @@ -73,6 +73,7 @@ Prefetch ScopeId poco + ResolutionPolicyType diff --git a/README.md b/README.md index f31265069..0f1fcd892 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ Azure WebJobs SDK === + +|Branch|Status| +|---|---| +|dev|[![Build status](https://ci.appveyor.com/api/projects/status/3qmk6ukn942q220j/branch/dev?svg=true)](https://ci.appveyor.com/project/appsvc/azure-webjobs-sdk-rqm4t/branch/dev)| +|master|[![Build status](https://ci.appveyor.com/api/projects/status/3qmk6ukn942q220j/branch/master?svg=true)](https://ci.appveyor.com/project/appsvc/azure-webjobs-sdk-rqm4t/branch/master)| + The **Azure WebJobs SDK** is a framework that simplifies the task of writing background processing code that runs in Azure. The Azure WebJobs SDK includes a declarative **binding** and **trigger** system that works with Azure Storage Blobs, Queues and Tables as well as Service Bus. The binding system makes it incredibly easy to write code that reads or writes Azure Storage objects. The trigger system automatically invokes a function in your code whenever any new data is received in a queue or blob. In addition to the built in triggers/bindings, the WebJobs SDK is **fully extensible**, allowing new types of triggers/bindings to be created and plugged into the framework in a first class way. See [Azure WebJobs SDK Extensions](https://github.com/Azure/azure-webjobs-sdk-extensions) for details. Many useful extensions have already been created and can be used in your applications today. Extensions include a File trigger/binder, a Timer/Cron trigger, a WebHook HTTP trigger, as well as a SendGrid email binding. diff --git a/WebJobs.proj b/WebJobs.proj index 63c380cb4..0fab8adc4 100644 --- a/WebJobs.proj +++ b/WebJobs.proj @@ -70,7 +70,7 @@ - + @@ -141,6 +141,21 @@ + + + + + + + + + + + + + @@ -233,7 +248,7 @@ { Log.LogMessage("Downloading latest version of NuGet.exe..."); WebClient webClient = new WebClient(); - webClient.DownloadFile("https://nuget.org/nuget.exe", OutputFileName); + webClient.DownloadFile("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", OutputFileName); } return true; diff --git a/WebJobs.sln b/WebJobs.sln index e79d93a72..88b9f1f6c 100644 --- a/WebJobs.sln +++ b/WebJobs.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 14 -VisualStudioVersion = 14.0.24720.0 +VisualStudioVersion = 14.0.25420.1 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{639967B0-0544-4C52-94AC-9A3D25E33256}" EndProject @@ -52,6 +52,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebJobs.Logging", "src\Micr EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebJobs.Logging.FunctionalTests", "test\Microsoft.Azure.WebJobs.Logging.FunctionalTests\WebJobs.Logging.FunctionalTests.csproj", "{C8EAAE01-E8CF-4131-9D4B-F0FDF00DA4BE}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sample", "Sample", "{72A798F0-699B-4C8E-8D43-C1749661471E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SampleHost", "sample\SampleHost\SampleHost.csproj", "{93429246-CCE9-4EB0-B94D-68522862BA79}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -122,6 +126,10 @@ Global {C8EAAE01-E8CF-4131-9D4B-F0FDF00DA4BE}.Debug|Any CPU.Build.0 = Debug|Any CPU {C8EAAE01-E8CF-4131-9D4B-F0FDF00DA4BE}.Release|Any CPU.ActiveCfg = Release|Any CPU {C8EAAE01-E8CF-4131-9D4B-F0FDF00DA4BE}.Release|Any CPU.Build.0 = Release|Any CPU + {93429246-CCE9-4EB0-B94D-68522862BA79}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {93429246-CCE9-4EB0-B94D-68522862BA79}.Debug|Any CPU.Build.0 = Debug|Any CPU + {93429246-CCE9-4EB0-B94D-68522862BA79}.Release|Any CPU.ActiveCfg = Release|Any CPU + {93429246-CCE9-4EB0-B94D-68522862BA79}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -135,5 +143,6 @@ Global {28BC5EE0-9227-4124-AA8F-1C0CAA218D12} = {639967B0-0544-4C52-94AC-9A3D25E33256} {C6B834AB-7B6A-47AE-A7C3-C102B0C861FF} = {639967B0-0544-4C52-94AC-9A3D25E33256} {C8EAAE01-E8CF-4131-9D4B-F0FDF00DA4BE} = {639967B0-0544-4C52-94AC-9A3D25E33256} + {93429246-CCE9-4EB0-B94D-68522862BA79} = {72A798F0-699B-4C8E-8D43-C1749661471E} EndGlobalSection EndGlobal diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..b09520183 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,9 @@ +build_script: + - msbuild WebJobs.proj /t:BuildTestBinaries /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /p:OutputPath=%APPVEYOR_BUILD_FOLDER%\bin + +test_script: + - vstest.console /logger:Appveyor /TestAdapterPath:bin bin/Microsoft.Azure.WebJobs.Host.UnitTests.dll bin/Microsoft.Azure.WebJobs.Host.FunctionalTests.dll bin/Microsoft.Azure.WebJobs.ServiceBus.UnitTests.dll bin/Dashboard.UnitTests.dll bin/Microsoft.Azure.WebJobs.Host.EndToEndTests.dll + +# if you need to rdp into build machine to investigate +# on_finish: +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) diff --git a/perf/Microsoft.Azure.WebJobs.Perf/WebJobs.Perf.csproj b/perf/Microsoft.Azure.WebJobs.Perf/WebJobs.Perf.csproj index 5796a39cb..cbe1724d7 100644 --- a/perf/Microsoft.Azure.WebJobs.Perf/WebJobs.Perf.csproj +++ b/perf/Microsoft.Azure.WebJobs.Perf/WebJobs.Perf.csproj @@ -24,7 +24,7 @@ prompt 4 true - 5 + default pdbonly @@ -34,7 +34,7 @@ prompt 4 true - 5 + default @@ -45,15 +45,15 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True @@ -72,7 +72,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True diff --git a/perf/Microsoft.Azure.WebJobs.Perf/app.config b/perf/Microsoft.Azure.WebJobs.Perf/app.config index 53d004e55..8e91b3a8c 100644 --- a/perf/Microsoft.Azure.WebJobs.Perf/app.config +++ b/perf/Microsoft.Azure.WebJobs.Perf/app.config @@ -17,7 +17,7 @@ - + \ No newline at end of file diff --git a/perf/Microsoft.Azure.WebJobs.Perf/packages.config b/perf/Microsoft.Azure.WebJobs.Perf/packages.config index dbcdd7985..da5913211 100644 --- a/perf/Microsoft.Azure.WebJobs.Perf/packages.config +++ b/perf/Microsoft.Azure.WebJobs.Perf/packages.config @@ -1,12 +1,16 @@  - - - + + + - + + + + + diff --git a/sample/SampleHost/App.config b/sample/SampleHost/App.config new file mode 100644 index 000000000..f634f3861 --- /dev/null +++ b/sample/SampleHost/App.config @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sample/SampleHost/Functions.cs b/sample/SampleHost/Functions.cs new file mode 100644 index 000000000..e90876ca7 --- /dev/null +++ b/sample/SampleHost/Functions.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.WebJobs; +using Newtonsoft.Json.Linq; + +namespace SampleHost +{ + public static class Functions + { + public static void BlobTrigger( + [BlobTrigger("test")] string blob) + { + Console.WriteLine("Processed blob: " + blob); + } + + public static void BlobPoisonBlobHandler( + [QueueTrigger("webjobs-blobtrigger-poison")] JObject blobInfo) + { + string container = (string)blobInfo["ContainerName"]; + string blobName = (string)blobInfo["BlobName"]; + + Console.WriteLine($"Poison blob: {container}/{blobName}"); + } + + public static void QueueTrigger( + [QueueTrigger("test")] string message) + { + Console.WriteLine("Processed message: " + message); + } + } +} diff --git a/sample/SampleHost/Program.cs b/sample/SampleHost/Program.cs new file mode 100644 index 000000000..248613df9 --- /dev/null +++ b/sample/SampleHost/Program.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.WebJobs; + +namespace SampleHost +{ + class Program + { + static void Main(string[] args) + { + var config = new JobHostConfiguration(); + config.Queues.VisibilityTimeout = TimeSpan.FromSeconds(15); + config.Queues.MaxDequeueCount = 3; + + if (config.IsDevelopment) + { + config.UseDevelopmentSettings(); + } + + var host = new JobHost(config); + host.RunAndBlock(); + } + } +} diff --git a/sample/SampleHost/Properties/AssemblyInfo.cs b/sample/SampleHost/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..9de00bcd5 --- /dev/null +++ b/sample/SampleHost/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("SampleHost")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("SampleHost")] +[assembly: AssemblyCopyright("Copyright © 2016")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("93429246-cce9-4eb0-b94d-68522862ba79")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/sample/SampleHost/SampleHost.csproj b/sample/SampleHost/SampleHost.csproj new file mode 100644 index 000000000..3a36addfd --- /dev/null +++ b/sample/SampleHost/SampleHost.csproj @@ -0,0 +1,76 @@ + + + + + Debug + AnyCPU + {93429246-CCE9-4EB0-B94D-68522862BA79} + Exe + Properties + SampleHost + SampleHost + v4.5.2 + 512 + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll + True + + + + + + + + + + + + + + + + + + + + + + {0e095cb2-3030-49ff-966c-785f1a55f0c1} + WebJobs.Host + + + {e3f2b2c8-6b8d-4d6a-a3ae-98366c9f3b49} + WebJobs + + + + + \ No newline at end of file diff --git a/sample/SampleHost/packages.config b/sample/SampleHost/packages.config new file mode 100644 index 000000000..9d64bf364 --- /dev/null +++ b/sample/SampleHost/packages.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/src/Dashboard/ApiControllers/FunctionsController.cs b/src/Dashboard/ApiControllers/FunctionsController.cs index 8550d0b9e..c86b9050c 100644 --- a/src/Dashboard/ApiControllers/FunctionsController.cs +++ b/src/Dashboard/ApiControllers/FunctionsController.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Reflection; using System.Threading.Tasks; using System.Web; using System.Web.Caching; @@ -249,7 +250,7 @@ public async Task GetRecentInvocationsTimeline( { StartBucket = entity.TimeBucket, Start = entity.Time, - TotalPass = entity.TotalPass, + TotalPass = entity.TotalPass, TotalFail = entity.TotalFail, TotalRun = entity.TotalRun }); @@ -341,6 +342,7 @@ private InvocationLogViewModel CreateInvocationEntry(RecentInvocationEntry entry metadataSnapshot.EndTime = entry.EndTime; metadataSnapshot.Succeeded = entry.Succeeded; metadataSnapshot.Heartbeat = entry.Heartbeat; + metadataSnapshot.FunctionInstanceHeartbeatExpiry = entry.FunctionInstanceHeartbeatExpiry; return new InvocationLogViewModel(metadataSnapshot, HostInstanceHasHeartbeat(metadataSnapshot)); } @@ -465,7 +467,10 @@ private static ParameterModel[] CreateParameterModels(CloudStorageAccount accoun // If host is specified, then only return definitions for that host. If null, return all hosts. [Route("api/functions/definitions")] - public IHttpActionResult GetFunctionDefinitions([FromUri]PagingInfo pagingInfo, string host = null) + public IHttpActionResult GetFunctionDefinitions( + [FromUri]PagingInfo pagingInfo, + string host = null, + bool skipStats = false) { if (pagingInfo == null) { @@ -518,7 +523,9 @@ public IHttpActionResult GetFunctionDefinitions([FromUri]PagingInfo pagingInfo, model.IsOldHost = OnlyBeta1HostExists(alreadyFoundNoNewerEntries: true); } - if (model.Entries != null) + // This is very slow. Allow a flag to skip it, and then client can query the stats independently + // via the /timeline API. + if ((model.Entries != null) && !skipStats) { foreach (FunctionStatisticsViewModel statisticsModel in model.Entries) { @@ -579,6 +586,19 @@ public IHttpActionResult Abort(string instanceQueueName) return Ok(); } + // Diagnostics endpoint, getting the version of the service that's running. + [Route("api/version")] + public IHttpActionResult GetVersionInfo() + { + var assembly = this.GetType().Assembly; + AssemblyFileVersionAttribute fileVersionAttr = assembly.GetCustomAttribute(); + + return Ok(new + { + Version = fileVersionAttr.Version + }); + } + private bool? HostHasHeartbeat(FunctionIndexEntry function) { if (!function.HeartbeatExpirationInSeconds.HasValue) @@ -626,6 +646,12 @@ public IHttpActionResult Abort(string instanceQueueName) private bool? HostInstanceHasHeartbeat(FunctionInstanceSnapshot snapshot) { + if (snapshot.FunctionInstanceHeartbeatExpiry.HasValue) + { + var now = DateTime.UtcNow; + return snapshot.FunctionInstanceHeartbeatExpiry.Value > now; + } + HeartbeatDescriptor heartbeat = snapshot.Heartbeat; if (heartbeat == null) diff --git a/src/Dashboard/Dashboard.csproj b/src/Dashboard/Dashboard.csproj index 31f51a1bc..bf17ab9a3 100644 --- a/src/Dashboard/Dashboard.csproj +++ b/src/Dashboard/Dashboard.csproj @@ -37,7 +37,7 @@ prompt 4 false - 5 + default pdbonly @@ -47,7 +47,7 @@ prompt 4 false - 5 + default @@ -68,15 +68,15 @@ - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True @@ -100,7 +100,7 @@ True - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True @@ -513,9 +513,9 @@ This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + - + TMessage" converter, then that takes precedence. - // Else, bind over an array of "byte --> TMessage" converters - - argumentBuilder = DynamicInvokeBuildOutArgument(elementType, converterManager, buildFromAttribute, cloner); - - if (argumentBuilder != null) - { - } - else if (elementType.IsArray) - { - if (elementType == typeof(TMessage[])) - { - argumentBuilder = (attrResolved, context) => - { - IAsyncCollector raw = buildFromAttribute(attrResolved); - var invokeString = cloner.GetInvokeString(attrResolved); - return new OutArrayValueProvider(raw, invokeString); - }; - } - else - { - // out TMessage[] - var e2 = elementType.GetElementType(); - argumentBuilder = DynamicBuildOutArrayArgument(e2, converterManager, buildFromAttribute, cloner); - } - } - else - { - // Single enqueue - // out TMessage - if (elementType == typeof(TMessage)) - { - argumentBuilder = (attrResolved, context) => - { - IAsyncCollector raw = buildFromAttribute(attrResolved); - var invokeString = cloner.GetInvokeString(attrResolved); - return new OutValueProvider(raw, invokeString); - }; - } - } - - // For out-param, give some rich errors. - if (argumentBuilder == null) - { - if (typeof(IEnumerable).IsAssignableFrom(elementType)) - { - throw new InvalidOperationException( - "Enumerable types are not supported. Use ICollector or IAsyncCollector instead."); - } - else if (typeof(object) == elementType) - { - throw new InvalidOperationException("Object element types are not supported."); - } - } - - return argumentBuilder; - } - - private static FuncArgumentBuilder DynamicBuildOutArrayArgument( - Type typeMessageSrc, - IConverterManager cm, - Func> buildFromAttribute, - AttributeCloner cloner) - where TAttribute : Attribute - { - var method = typeof(BindingFactoryHelpers).GetMethod("BuildOutArrayArgument", BindingFlags.NonPublic | BindingFlags.Static); - method = method.MakeGenericMethod(typeof(TAttribute), typeof(TMessage), typeMessageSrc); - var argumentBuilder = MethodInvoke>(method, cm, buildFromAttribute, cloner); - return argumentBuilder; - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode", Justification = "Dynamically invoked")] - private static FuncArgumentBuilder BuildOutArrayArgument( - IConverterManager cm, - Func> buildFromAttribute, - AttributeCloner cloner) - where TAttribute : Attribute - { - // Other - var convert = cm.GetConverter(); - FuncArgumentBuilder argumentBuilder = (attrResolved, context) => - { - IAsyncCollector raw = buildFromAttribute(attrResolved); - IAsyncCollector obj = new TypedAsyncCollectorAdapter( - raw, convert, attrResolved, context); - string invokeString = cloner.GetInvokeString(attrResolved); - return new OutArrayValueProvider(obj, invokeString); - }; - return argumentBuilder; - } - - // Helper to dynamically invoke BuildICollectorArgument with the proper generics - // Can we bind to 'out TUser'? Requires converter manager to supply a TUser--> TMessage converter. - // Return null if we can't bind it. - private static FuncArgumentBuilder DynamicInvokeBuildOutArgument( - Type typeMessageSrc, - IConverterManager cm, - Func> buildFromAttribute, - AttributeCloner cloner) - where TAttribute : Attribute - { - var method = typeof(BindingFactoryHelpers).GetMethod("BuildOutArgument", BindingFlags.NonPublic | BindingFlags.Static); - method = method.MakeGenericMethod(typeof(TAttribute), typeof(TMessage), typeMessageSrc); - var argumentBuilder = MethodInvoke>(method, cm, buildFromAttribute, cloner); - return argumentBuilder; - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode", Justification = "Dynamically invoked")] - private static FuncArgumentBuilder BuildOutArgument( - IConverterManager cm, - Func> buildFromAttribute, - AttributeCloner cloner) - where TAttribute : Attribute - { - // Other - var convert = cm.GetConverter(); - if (convert == null) - { - return null; - } - FuncArgumentBuilder argumentBuilder = (attrResolved, context) => - { - IAsyncCollector raw = buildFromAttribute(attrResolved); - IAsyncCollector obj = new TypedAsyncCollectorAdapter( - raw, convert, attrResolved, context); - string invokeString = cloner.GetInvokeString(attrResolved); - return new OutValueProvider(obj, invokeString); - }; - return argumentBuilder; - } - - // Helper to dynamically invoke BuildICollectorArgument with the proper generics - private static FuncArgumentBuilder DynamicInvokeBuildICollectorArgument( - Type typeMessageSrc, - IConverterManager cm, - Func> buildFromAttribute, - AttributeCloner cloner) - where TAttribute : Attribute - { - var method = typeof(BindingFactoryHelpers).GetMethod("BuildICollectorArgument", BindingFlags.NonPublic | BindingFlags.Static); - method = method.MakeGenericMethod(typeof(TAttribute), typeof(TMessage), typeMessageSrc); - var argumentBuilder = MethodInvoke>(method, cm, buildFromAttribute, cloner); - return argumentBuilder; - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode", Justification = "Dynamic invoke")] - private static FuncArgumentBuilder BuildICollectorArgument( - IConverterManager cm, - Func> buildFromAttribute, - AttributeCloner cloner) - where TAttribute : Attribute - { - // Other - var convert = cm.GetConverter(); - if (convert == null) - { - ThrowMissingConversionError(typeof(TMessageSrc)); - } - FuncArgumentBuilder argumentBuilder = (attrResolved, context) => - { - IAsyncCollector raw = buildFromAttribute(attrResolved); - IAsyncCollector obj = new TypedAsyncCollectorAdapter( - raw, convert, attrResolved, context); - ICollector obj2 = new SyncAsyncCollectorAdapter(obj); - string invokeString = cloner.GetInvokeString(attrResolved); - return new AsyncCollectorValueProvider, TMessage>(obj2, raw, invokeString); - }; - return argumentBuilder; - } - - // Helper to dynamically invoke BuildIAsyncCollectorArgument with the proper generics - private static FuncArgumentBuilder DynamicInvokeBuildIAsyncCollectorArgument( - Type typeMessageSrc, - IConverterManager cm, - Func> buildFromAttribute, - AttributeCloner cloner) - where TAttribute : Attribute - { - var method = typeof(BindingFactoryHelpers).GetMethod("BuildIAsyncCollectorArgument", BindingFlags.NonPublic | BindingFlags.Static); - method = method.MakeGenericMethod(typeof(TAttribute), typeof(TMessage), typeMessageSrc); - var argumentBuilder = MethodInvoke>(method, cm, buildFromAttribute, cloner); - return argumentBuilder; - } - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode", Justification = "Dynamically invoked")] - private static FuncArgumentBuilder BuildIAsyncCollectorArgument( - IConverterManager cm, - Func> buildFromAttribute, - AttributeCloner cloner) - where TAttribute : Attribute - { - var convert = cm.GetConverter(); - if (convert == null) - { - ThrowMissingConversionError(typeof(TMessageSrc)); - } - FuncArgumentBuilder argumentBuilder = (attrResolved, context) => - { - IAsyncCollector raw = buildFromAttribute(attrResolved); - IAsyncCollector obj = new TypedAsyncCollectorAdapter( - raw, convert, attrResolved, context); - var invokeString = cloner.GetInvokeString(attrResolved); - return new AsyncCollectorValueProvider, TMessage>(obj, raw, invokeString); - }; - return argumentBuilder; - } - - // typeUser - type in the user's parameter. - private static void ThrowMissingConversionError(Type typeUser) - { - if (typeUser.IsPrimitive) - { - throw new NotSupportedException("Primitive types are not supported."); - } - - if (typeof(IEnumerable).IsAssignableFrom(typeUser)) - { - throw new InvalidOperationException("Nested collections are not supported."); - } - throw new InvalidOperationException("Can't convert from type '" + typeUser.FullName); - } - - // Helper to invoke and unwrap teh target exception. - private static TReturn MethodInvoke(MethodInfo method, params object[] args) + + // Helper to invoke and unwrap the target exception. + public static TReturn MethodInvoke(MethodInfo method, params object[] args) { try { diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/AsyncCollectorBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/AsyncCollectorBindingProvider.cs new file mode 100644 index 000000000..bfa94b6f4 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/AsyncCollectorBindingProvider.cs @@ -0,0 +1,299 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Protocols; + +namespace Microsoft.Azure.WebJobs.Host.Bindings +{ + // General rule for binding parameters to an AsyncCollector. + // Supports the various flavors like IAsyncCollector, ICollector, out T, out T[]. + internal class AsyncCollectorBindingProvider : FluentBindingProvider, IBindingProvider + where TAttribute : Attribute + { + private readonly INameResolver _nameResolver; + private readonly IConverterManager _converterManager; + private readonly PatternMatcher _patternMatcher; + + public AsyncCollectorBindingProvider( + INameResolver nameResolver, + IConverterManager converterManager, + PatternMatcher patternMatcher) + { + this._nameResolver = nameResolver; + this._converterManager = converterManager; + this._patternMatcher = patternMatcher; + } + + // Describe different flavors of IAsyncCollector bindings. + private enum Mode + { + IAsyncCollector, + ICollector, + OutSingle, + OutArray + } + + public Task TryCreateAsync(BindingProviderContext context) + { + if (context == null) + { + throw new ArgumentNullException("context"); + } + + var parameter = context.Parameter; + + var mode = GetMode(parameter); + if (mode == null) + { + return Task.FromResult(null); + } + + var type = typeof(ExactBinding<>).MakeGenericType(typeof(TAttribute), typeof(TType), mode.ElementType); + var method = type.GetMethod("TryBuild", BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public); + var binding = BindingFactoryHelpers.MethodInvoke(method, this, mode.Mode, context); + + return Task.FromResult(binding); + } + + // Parse the signature to determine which mode this is. + // Can also check with converter manager to disambiguate some cases. + private CollectorBindingPattern GetMode(ParameterInfo parameter) + { + Type parameterType = parameter.ParameterType; + if (parameterType.IsGenericType) + { + var genericType = parameterType.GetGenericTypeDefinition(); + var elementType = parameterType.GetGenericArguments()[0]; + + if (genericType == typeof(IAsyncCollector<>)) + { + return new CollectorBindingPattern(Mode.IAsyncCollector, elementType); + } + else if (genericType == typeof(ICollector<>)) + { + return new CollectorBindingPattern(Mode.ICollector, elementType); + } + + // A different interface. Let another rule try it. + return null; + } + + if (parameter.IsOut) + { + // How should "out byte[]" bind? + // If there's an explicit "byte[] --> TMessage" converter, then that takes precedence. + // Else, bind over an array of "byte --> TMessage" converters + Type elementType = parameter.ParameterType.GetElementType(); + bool hasConverter = this._converterManager.HasConverter(elementType, typeof(TType)); + if (hasConverter) + { + // out T, where T might be an array + return new CollectorBindingPattern(Mode.OutSingle, elementType); + } + + if (elementType.IsArray) + { + // out T[] + var messageType = elementType.GetElementType(); + return new CollectorBindingPattern(Mode.OutArray, messageType); + } + + var validator = ConverterManager.GetTypeValidator(); + if (validator.IsMatch(elementType)) + { + // out T, t is not an array + return new CollectorBindingPattern(Mode.OutSingle, elementType); + } + + // For out-param ,we don't expect another rule to claim it. So give some rich errors on mismatch. + if (typeof(IEnumerable).IsAssignableFrom(elementType)) + { + throw new InvalidOperationException( + "Enumerable types are not supported. Use ICollector or IAsyncCollector instead."); + } + else if (typeof(object) == elementType) + { + throw new InvalidOperationException("Object element types are not supported."); + } + } + + // No match. Let another rule claim it + return null; + } + + // Represent the different possible flavors for binding to an async collector + private class CollectorBindingPattern + { + public CollectorBindingPattern(Mode mode, Type elementType) + { + this.Mode = mode; + this.ElementType = elementType; + } + public Mode Mode { get; set; } + public Type ElementType { get; set; } + } + + // TType - specified in the rule. + // TMessage - element type of the IAsyncCollector<> we matched to. + private class ExactBinding : BindingBase + { + private readonly Func _buildFromAttribute; + + private readonly FuncConverter _converter; + private readonly Mode _mode; + + public ExactBinding( + AttributeCloner cloner, + ParameterDescriptor param, + Mode mode, + Func buildFromAttribute, + FuncConverter converter) : base(cloner, param) + { + this._buildFromAttribute = buildFromAttribute; + this._mode = mode; + this._converter = converter; + } + + public static ExactBinding TryBuild( + AsyncCollectorBindingProvider parent, + Mode mode, + BindingProviderContext context) + { + var patternMatcher = parent._patternMatcher; + + var parameter = context.Parameter; + TAttribute attributeSource = parameter.GetCustomAttribute(inherit: false); + + Func> hookWrapper = null; + if (parent.PostResolveHook != null) + { + hookWrapper = (attrResolved) => parent.PostResolveHook(attrResolved, parameter, parent._nameResolver); + } + + Func buildFromAttribute; + FuncConverter converter = null; + + // Prefer the shortest route to creating the user type. + // If TType matches the user type directly, then we should be able to directly invoke the builder in a single step. + // TAttribute --> TUserType + var checker = ConverterManager.GetTypeValidator(); + if (checker.IsMatch(typeof(TMessage))) + { + buildFromAttribute = patternMatcher.TryGetConverterFunc( + typeof(TAttribute), typeof(IAsyncCollector)); + } + else + { + var converterManager = parent._converterManager; + + // Try with a converter + // Find a builder for : TAttribute --> TType + // and then couple with a converter: TType --> TParameterType + converter = converterManager.GetConverter(); + if (converter == null) + { + // Preserves legacy behavior. This means we can only have 1 async collector. + // However, the collector's builder object can switch. + throw NewMissingConversionError(typeof(TMessage)); + } + + buildFromAttribute = patternMatcher.TryGetConverterFunc( + typeof(TAttribute), typeof(IAsyncCollector)); + } + + if (buildFromAttribute == null) + { + return null; + } + + ParameterDescriptor param; + if (parent.BuildParameterDescriptor != null) + { + param = parent.BuildParameterDescriptor(attributeSource, parameter, parent._nameResolver); + } + else + { + param = new ParameterDescriptor + { + Name = parameter.Name, + DisplayHints = new ParameterDisplayHints + { + Description = "output" + } + }; + } + + var cloner = new AttributeCloner(attributeSource, context.BindingDataContract, parent._nameResolver, hookWrapper); + return new ExactBinding(cloner, param, mode, buildFromAttribute, converter); + } + + // typeUser - type in the user's parameter. + private static Exception NewMissingConversionError(Type typeUser) + { + if (typeUser.IsPrimitive) + { + return new NotSupportedException("Primitive types are not supported."); + } + + if (typeof(IEnumerable).IsAssignableFrom(typeUser)) + { + return new InvalidOperationException("Nested collections are not supported."); + } + return new InvalidOperationException("Can't convert from type '" + typeUser.FullName); + } + + protected override Task BuildAsync( + TAttribute attrResolved, + ValueBindingContext context) + { + string invokeString = Cloner.GetInvokeString(attrResolved); + + object obj = _buildFromAttribute(attrResolved); + + IAsyncCollector collector; + if (_converter != null) + { + // Apply a converter + var innerCollector = (IAsyncCollector)obj; + + collector = new TypedAsyncCollectorAdapter( + innerCollector, _converter, attrResolved, context); + } + else + { + collector = (IAsyncCollector)obj; + } + + var vp = CoerceValueProvider(_mode, invokeString, collector); + return Task.FromResult(vp); + } + + // Get a ValueProvider that's in the right mode. + private static IValueProvider CoerceValueProvider(Mode mode, string invokeString, IAsyncCollector collector) + { + switch (mode) + { + case Mode.IAsyncCollector: + return new AsyncCollectorValueProvider, TMessage>(collector, collector, invokeString); + + case Mode.ICollector: + ICollector syncCollector = new SyncAsyncCollectorAdapter(collector); + return new AsyncCollectorValueProvider, TMessage>(syncCollector, collector, invokeString); + + case Mode.OutArray: + return new OutArrayValueProvider(collector, invokeString); + + case Mode.OutSingle: + return new OutValueProvider(collector, invokeString); + + default: + throw new NotImplementedException($"mode ${mode} not implemented"); + } + } + } + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/AsyncCollectorWithConverterManagerBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/AsyncCollectorWithConverterManagerBindingProvider.cs deleted file mode 100644 index 47464487a..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/AsyncCollectorWithConverterManagerBindingProvider.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Protocols; - -namespace Microsoft.Azure.WebJobs.Host.Bindings -{ - // Bind Attribute --> IAsyncCollector - // USe the converter manager to coerce from the user parameter type to TMessage. - internal class AsyncCollectorWithConverterManagerBindingProvider : - IBindingProvider - where TAttribute : Attribute - { - private readonly INameResolver _nameResolver; - private readonly IConverterManager _converterManager; - private readonly Func> _buildFromAttribute; - private readonly Func _buildParamDescriptor; - private readonly Func> _postResolveHook; - - public AsyncCollectorWithConverterManagerBindingProvider( - INameResolver nameResolver, - IConverterManager converterManager, - Func> buildFromAttribute, - Func buildParamDescriptor = null, - Func> postResolveHook = null) - { - this._nameResolver = nameResolver; - this._buildFromAttribute = buildFromAttribute; - this._converterManager = converterManager; - this._buildParamDescriptor = buildParamDescriptor; - this._postResolveHook = postResolveHook; - } - - // Called once per method definition. Very static. - public Task TryCreateAsync(BindingProviderContext context) - { - if (context == null) - { - throw new ArgumentNullException("context"); - } - - ParameterInfo parameter = context.Parameter; - TAttribute attribute = parameter.GetCustomAttribute(inherit: false); - - if (attribute == null) - { - return Task.FromResult(null); - } - - IBinding binding = BindingFactoryHelpers.BindCollector( - parameter, - _nameResolver, - _converterManager, - context.BindingDataContract, - _buildFromAttribute, - _buildParamDescriptor, - _postResolveHook); - - return Task.FromResult(binding); - } - } -} \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/BindToInputBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/BindToInputBindingProvider.cs new file mode 100644 index 000000000..5201a78a2 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/BindToInputBindingProvider.cs @@ -0,0 +1,161 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Protocols; + +namespace Microsoft.Azure.WebJobs.Host.Bindings +{ + // General rule for binding to input parameters. + // Can invoke Converter manager. + // Can leverage OpenTypes for pattern matchers. + internal class BindToInputBindingProvider : FluentBindingProvider, IBindingProvider + where TAttribute : Attribute + { + private readonly INameResolver _nameResolver; + private readonly IConverterManager _converterManager; + private readonly PatternMatcher _patternMatcher; + + public BindToInputBindingProvider( + INameResolver nameResolver, + IConverterManager converterManager, + PatternMatcher patternMatcher) + { + this._nameResolver = nameResolver; + this._converterManager = converterManager; + this._patternMatcher = patternMatcher; + } + + public Task TryCreateAsync(BindingProviderContext context) + { + if (context == null) + { + throw new ArgumentNullException("context"); + } + + var parameter = context.Parameter; + var typeUser = parameter.ParameterType; + + if (typeUser.IsByRef) + { + return Task.FromResult(null); + } + + var type = typeof(ExactBinding<>).MakeGenericType(typeof(TAttribute), typeof(TType), typeUser); + var method = type.GetMethod("TryBuild", BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public); + var binding = BindingFactoryHelpers.MethodInvoke(method, this, context); + + return Task.FromResult(binding); + } + + private class ExactBinding : BindingBase + { + private readonly Func _buildFromAttribute; + + private readonly FuncConverter _converter; + + public ExactBinding( + AttributeCloner cloner, + ParameterDescriptor param, + Func buildFromAttribute, + FuncConverter converter) : base(cloner, param) + { + this._buildFromAttribute = buildFromAttribute; + this._converter = converter; + } + + public static ExactBinding TryBuild( + BindToInputBindingProvider parent, + BindingProviderContext context) + { + var cm = parent._converterManager; + var patternMatcher = parent._patternMatcher; + + var parameter = context.Parameter; + TAttribute attributeSource = parameter.GetCustomAttribute(inherit: false); + + Func> hookWrapper = null; + if (parent.PostResolveHook != null) + { + hookWrapper = (attrResolved) => parent.PostResolveHook(attrResolved, parameter, parent._nameResolver); + } + + var cloner = new AttributeCloner(attributeSource, context.BindingDataContract, parent._nameResolver, hookWrapper); + + Func buildFromAttribute; + FuncConverter converter = null; + + // Prefer the shortest route to creating the user type. + // If TType matches the user type directly, then we should be able to directly invoke the builder in a single step. + // TAttribute --> TUserType + var checker = ConverterManager.GetTypeValidator(); + if (checker.IsMatch(typeof(TUserType))) + { + buildFromAttribute = patternMatcher.TryGetConverterFunc(typeof(TAttribute), typeof(TUserType)); + } + else + { + // Try with a converter + // Find a builder for : TAttribute --> TType + // and then couple with a converter: TType --> TParameterType + converter = cm.GetConverter(); + if (converter == null) + { + return null; + } + + buildFromAttribute = patternMatcher.TryGetConverterFunc(typeof(TAttribute), typeof(TType)); + } + + if (buildFromAttribute == null) + { + return null; + } + + ParameterDescriptor param; + if (parent.BuildParameterDescriptor != null) + { + param = parent.BuildParameterDescriptor(attributeSource, parameter, parent._nameResolver); + } + else + { + param = new ParameterDescriptor + { + Name = parameter.Name, + DisplayHints = new ParameterDisplayHints + { + Description = "input" + } + }; + } + + return new ExactBinding(cloner, param, buildFromAttribute, converter); + } + + protected override Task BuildAsync( + TAttribute attrResolved, + ValueBindingContext context) + { + string invokeString = Cloner.GetInvokeString(attrResolved); + + object obj = _buildFromAttribute(attrResolved); + TUserType finalObj; + if (_converter == null) + { + finalObj = (TUserType)obj; + } + else + { + var intermediateObj = (TType)obj; + finalObj = _converter(intermediateObj, attrResolved, context); + } + + IValueProvider vp = new ConstantValueProvider(finalObj, typeof(TUserType), invokeString); + + return Task.FromResult(vp); + } + } + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/ExactTypeBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/ExactTypeBindingProvider.cs deleted file mode 100644 index 2754c99a5..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/ExactTypeBindingProvider.cs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Protocols; - -namespace Microsoft.Azure.WebJobs.Host.Bindings -{ - // BindingProvider to bind to an exact type. Useful for binding to a client object. - internal class ExactTypeBindingProvider : IBindingProvider - where TAttribute : Attribute - { - private readonly INameResolver _nameResolver; - private readonly Func> _buildFromAttr; - private readonly Func _buildParameterDescriptor; - private readonly Func> _postResolveHook; - - public ExactTypeBindingProvider( - INameResolver nameResolver, - Func> buildFromAttr, - Func buildParameterDescriptor = null, - Func> postResolveHook = null) - { - this._postResolveHook = postResolveHook; - this._nameResolver = nameResolver; - this._buildParameterDescriptor = buildParameterDescriptor; - this._buildFromAttr = buildFromAttr; - } - - public Task TryCreateAsync(BindingProviderContext context) - { - if (context == null) - { - throw new ArgumentNullException("context"); - } - ParameterInfo parameter = context.Parameter; - - TAttribute attributeSource = parameter.GetCustomAttribute(inherit: false); - if (attributeSource == null) - { - return Task.FromResult(null); - } - - if (parameter.ParameterType != typeof(TUserType)) - { - return Task.FromResult(null); - } - - Func> hookWrapper = null; - if (_postResolveHook != null) - { - hookWrapper = (attrResolved) => _postResolveHook(attrResolved, parameter, _nameResolver); - } - - var cloner = new AttributeCloner(attributeSource, context.BindingDataContract, _nameResolver, hookWrapper); - ParameterDescriptor param; - if (_buildParameterDescriptor != null) - { - param = _buildParameterDescriptor(attributeSource, parameter, _nameResolver); - } - else - { - param = new ParameterDescriptor - { - Name = parameter.Name, - DisplayHints = new ParameterDisplayHints - { - Description = "output" - } - }; - } - - var binding = new ExactBinding(cloner, param, _buildFromAttr); - return Task.FromResult(binding); - } - - private class ExactBinding : BindingBase - { - private readonly Func> _buildFromAttr; - - public ExactBinding( - AttributeCloner cloner, - ParameterDescriptor param, - Func> buildFromAttr) - : base(cloner, param) - { - _buildFromAttr = buildFromAttr; - } - - protected override async Task BuildAsync(TAttribute attrResolved, ValueBindingContext context) - { - string invokeString = Cloner.GetInvokeString(attrResolved); - var obj = await _buildFromAttr(attrResolved); - - IValueProvider vp = new ConstantValueProvider(obj, typeof(TUserType), invokeString); - return vp; - } - } - } -} \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/FluentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/FluentBindingProvider.cs new file mode 100644 index 000000000..7a213aec5 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/FluentBindingProvider.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Protocols; + +namespace Microsoft.Azure.WebJobs.Host +{ + // Base class to aide in private backwards compatability hooks for some bindings. + // Help in implementing a Fluent API design where these extra properties are set + // via method cascading rather than all at once upfront. + internal class FluentBindingProvider + { + protected internal Func BuildParameterDescriptor { get; set; } + protected internal Func> PostResolveHook { get; set; } + } +} \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/FluidBindingProviderExtensions.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/FluidBindingProviderExtensions.cs new file mode 100644 index 000000000..2f607a2a0 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/FluidBindingProviderExtensions.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Bindings; +using Microsoft.Azure.WebJobs.Host.Protocols; + +namespace Microsoft.Azure.WebJobs.Host +{ + // Internal Extension methods for setting backwards compatibility hooks on certain bindings. + // This keeps the hooks out of the public surface. + internal static class FluidBindingProviderExtensions + { + public static IBindingProvider SetPostResolveHook( + this IBindingProvider binder, + Func buildParameterDescriptor = null, + Func> postResolveHook = null) + { + var fluidBinder = (FluentBindingProvider)binder; + + fluidBinder.PostResolveHook = postResolveHook; + fluidBinder.BuildParameterDescriptor = buildParameterDescriptor; + return binder; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/GenericAsyncCollectorBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/GenericAsyncCollectorBindingProvider.cs deleted file mode 100644 index a39e7a4c9..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/GenericAsyncCollectorBindingProvider.cs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Reflection; -using System.Threading.Tasks; - -namespace Microsoft.Azure.WebJobs.Host.Bindings -{ - // Bind Attribute --> IAsyncCollector, where TMessage is determined by the user parameter type. - // This skips the converter manager and instead dynamically allocates a generic IAsyncCollector - // where TMessage matches the user parameter type. - internal class GenericAsyncCollectorBindingProvider : - IBindingProvider - where TAttribute : Attribute - { - private readonly INameResolver _nameResolver; - private readonly Func _builder; - private readonly Func _filter; - - public GenericAsyncCollectorBindingProvider( - INameResolver nameResolver, - Func builder, - Func filter) - { - this._nameResolver = nameResolver; - this._builder = builder; - this._filter = filter ?? DefaultFilter; - } - - private static bool DefaultFilter(TAttribute attribute, Type messageType) - { - return true; - } - - // Called once per method definition. Very static. - public Task TryCreateAsync(BindingProviderContext context) - { - if (context == null) - { - throw new ArgumentNullException("context"); - } - - ParameterInfo parameter = context.Parameter; - TAttribute attribute = parameter.GetCustomAttribute(inherit: false); - - if (attribute == null) - { - return Task.FromResult(null); - } - - // Now we can instantiate against the user's type. - // throws if can't infer the type. - Type typeMessage = BindingFactoryHelpers.GetAsyncCollectorCoreType(parameter.ParameterType); - - if (typeMessage == null) - { - // incompatible type. Skip. - return Task.FromResult(null); - } - // Apply filter - var cloner = new AttributeCloner(attribute, context.BindingDataContract, _nameResolver); - var attrNameResolved = cloner.GetNameResolvedAttribute(); - bool canUse = _filter(attrNameResolved, typeMessage); - - if (!canUse) - { - return Task.FromResult(null); - } - - var wrapper = WrapperBase.New( - typeMessage, _builder, _nameResolver, parameter, context); - - IBinding binding = wrapper.CreateBinding(); - return Task.FromResult(binding); - } - - // Wrappers to help with binding to a dynamically typed IAsyncCollector. - // TMessage is not known until runtime, so we need to dynamically create it. - // These inherit the generic args of the outer class. - private abstract class WrapperBase - { - protected Func Builder { get; set; } - protected INameResolver NameResolver { get; private set; } - protected ParameterInfo Parameter { get; private set; } - protected Type AsyncCollectorType { get; private set; } - - protected BindingProviderContext Context { get; private set; } - - public abstract IBinding CreateBinding(); - - internal static WrapperBase New( - Type typeMessage, - Func builder, - INameResolver nameResolver, - ParameterInfo parameter, - BindingProviderContext context) - { - // These inherit the generic args of the outer class. - var t = typeof(Wrapper<>).MakeGenericType(typeof(TAttribute), typeMessage); - var obj = Activator.CreateInstance(t); - var obj2 = (WrapperBase)obj; - - obj2.Builder = builder; - obj2.NameResolver = nameResolver; - obj2.Parameter = parameter; - obj2.Context = context; - return obj2; - } - } - - private class Wrapper : WrapperBase - { - // This is the builder function that gets passed to the core IAsyncCollector binders. - public IAsyncCollector BuildFromAttribute(TAttribute attribute) - { - var obj = this.Builder(attribute, typeof(TMessage)); - var collector = (IAsyncCollector)obj; - return collector; - } - - public override IBinding CreateBinding() - { - IBinding binding = BindingFactoryHelpers.BindCollector( - Parameter, - NameResolver, - new IdentityConverterManager(), - this.Context.BindingDataContract, - this.BuildFromAttribute, - null); - - return binding; - } - } - - // "Empty" converter manager that only allows identity conversions. - // The GenericAsyncCollector is already instantiated against the user type, so no conversions should be needed. - private class IdentityConverterManager : IConverterManager - { - public void AddConverter(FuncConverter converter) where TAttribute1 : Attribute - { - throw new NotImplementedException(); - } - - public FuncConverter GetConverter() where TAttribute1 : Attribute - { - if (typeof(TSource) != typeof(TDestination)) - { - return null; - } - return (src, attr, ctx) => - { - object obj = (object)src; - return (TDestination)obj; - }; - } - } - } // end class -} \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/GenericItemBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/GenericItemBindingProvider.cs deleted file mode 100644 index c4d322156..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/GenericItemBindingProvider.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Reflection; -using System.Threading.Tasks; - -namespace Microsoft.Azure.WebJobs.Host.Bindings -{ - // Binding provider that can bind to arbitrary user parameter. - // It invokes the builder function and passes the resolved attribute and user parameter type. - // The builder returns an object that is compatible with the user parameter. - internal class GenericItemBindingProvider : IBindingProvider - where TAttribute : Attribute - { - private readonly Func> _builder; - private readonly INameResolver _nameResolver; - - public GenericItemBindingProvider( - Func> builder, - INameResolver nameResolver) - { - _builder = builder; - _nameResolver = nameResolver; - } - - public Task TryCreateAsync(BindingProviderContext context) - { - if (context == null) - { - throw new ArgumentNullException("context"); - } - - ParameterInfo parameter = context.Parameter; - TAttribute attributeSource = parameter.GetCustomAttribute(inherit: false); - if (attributeSource == null) - { - return Task.FromResult(null); - } - - var cloner = new AttributeCloner(attributeSource, context.BindingDataContract, _nameResolver); - - IBinding binding = new Binding(cloner, parameter, _builder); - return Task.FromResult(binding); - } - - private class Binding : BindingBase - { - private readonly ParameterInfo _parameter; - private readonly Func> _builder; - - public Binding( - AttributeCloner cloner, - ParameterInfo parameterInfo, - Func> builder) - : base(cloner, parameterInfo) - { - _parameter = parameterInfo; - _builder = builder; - } - - protected override async Task BuildAsync(TAttribute attrResolved, ValueBindingContext context) - { - string invokeString = Cloner.GetInvokeString(attrResolved); - object obj = await _builder(attrResolved, _parameter.ParameterType); - - IValueProvider vp = new ConstantValueProvider(obj, _parameter.ParameterType, invokeString); - return vp; - } - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/ItemBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/ItemBindingProvider.cs index 6fac4a329..ac93f8350 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/ItemBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingProviders/ItemBindingProvider.cs @@ -54,7 +54,7 @@ private class Binding : BindingBase public Binding( AttributeCloner cloner, Func> builder, - ParameterInfo parameter) + ParameterInfo parameter) : base(cloner, parameter) { this._builder = builder; @@ -88,9 +88,9 @@ public Type Type } } - public object GetValue() + public Task GetValueAsync() { - return _inner.GetValue(); + return _inner.GetValueAsync(); } public Task SetValueAsync(object value, CancellationToken cancellationToken) diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingTemplateExtensions.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingTemplateExtensions.cs index 75d286a42..81516b4ef 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingTemplateExtensions.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/BindingTemplateExtensions.cs @@ -71,7 +71,7 @@ private static void ValidateContractCompatibility(IEnumerable parameterN { foreach (string parameterName in parameterNames) { - if (BindingTemplate.IsSystemBindingParameter(parameterName)) + if (BindingParameterResolver.IsSystemParameter(parameterName)) { continue; } diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/Cancellation/CancellationTokenValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/Cancellation/CancellationTokenValueProvider.cs index 6363d6d2d..a8df7fac1 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/Cancellation/CancellationTokenValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/Cancellation/CancellationTokenValueProvider.cs @@ -3,6 +3,7 @@ using System; using System.Threading; +using System.Threading.Tasks; namespace Microsoft.Azure.WebJobs.Host.Bindings.Cancellation { @@ -20,9 +21,9 @@ public Type Type get { return typeof(CancellationToken); } } - public object GetValue() + public Task GetValueAsync() { - return _token; + return Task.FromResult(_token); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/ConstantValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/ConstantValueProvider.cs index 7e558710f..55f3531d5 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/ConstantValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/ConstantValueProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading.Tasks; namespace Microsoft.Azure.WebJobs.Host.Bindings { @@ -20,9 +21,9 @@ public ConstantValueProvider(object value, Type type, string invokeString) public Type Type { get; set; } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/ConverterManager.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/ConverterManager.cs index 7587a2ee1..87cac3b46 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/ConverterManager.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/ConverterManager.cs @@ -4,24 +4,61 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Reflection; using System.Text; using Microsoft.Azure.WebJobs.Host.Bindings; using Newtonsoft.Json.Linq; namespace Microsoft.Azure.WebJobs -{ +{ // Concrete implementation of IConverterManager internal class ConverterManager : IConverterManager { // Map from to a converter function. - private Dictionary _funcsWithAttr = new Dictionary(); - + // (Type) --> FuncConverter + private readonly Dictionary _funcsWithAttr = new Dictionary(); + + private readonly List _openConverters = new List(); + + public static readonly IConverterManager Identity = new IdentityConverterManager(); + public ConverterManager() { - this.AddConverter(DefaultByteArray2String); + this.AddConverter(DefaultByteArrayToString); + } + + private void AddOpenConverter( + OpenType source, + OpenType dest, + Func> converterBuilder) + where TAttribute : Attribute + { + var entry = new Entry + { + Source = source, + Dest = dest, + Attribute = typeof(TAttribute), + Builder = converterBuilder + }; + this._openConverters.Add(entry); } - private static string DefaultByteArray2String(byte[] bytes) + private Func> TryGetOpenConverter(Type typeSource, Type typeDest, Type typeAttribute) + { + foreach (var entry in _openConverters) + { + if (entry.Attribute == typeAttribute) + { + if (entry.Source.IsMatch(typeSource) && entry.Dest.IsMatch(typeDest)) + { + return entry.Builder; + } + } + } + return null; + } + + private static string DefaultByteArrayToString(byte[] bytes) { string str = Encoding.UTF8.GetString(bytes); return str; @@ -32,11 +69,102 @@ private static string GetKey() return typeof(TSrc).FullName + "|" + typeof(TDest).FullName + "|" + typeof(TAttribute).FullName; } + internal static OpenType GetTypeValidator() + { + return GetTypeValidator(typeof(T)); + } + + internal static OpenType GetTypeValidator(Type type) + { + var openType = GetOpenType(type); + if (openType != null) + { + return openType; + } + + return new ExactMatch(type); + } + + // Gets an OpenType from the given argument. + // Return null if it's a concrete type (ie, not an open type). + private static OpenType GetOpenType() + { + var t = typeof(T); + return GetOpenType(t); + } + + private static OpenType GetOpenType(Type t) + { + if (t == typeof(OpenType) || t == typeof(object)) + { + return new AnythingOpenType(); + } + if (typeof(OpenType).IsAssignableFrom(t)) + { + return (OpenType)Activator.CreateInstance(t); + } + + if (t.IsArray) + { + var elementType = t.GetElementType(); + var innerType = GetTypeValidator(elementType); + return new ArrayOpenType(innerType); + } + + // Rewriter rule for generics so customers can say: IEnumerable + if (t.IsGenericType) + { + var outerType = t.GetGenericTypeDefinition(); + Type[] args = t.GetGenericArguments(); + if (args.Length == 1) + { + var arg1 = GetOpenType(args[0]); + if (arg1 != null) + { + return new SingleGenericArgOpenType(outerType, arg1); + } + // This is a concrete generic type, like IEnumerable. No open types needed. + } + else + { + // Just to sanity check, make sure there's no OpenType buried in the argument. + foreach (var arg in args) + { + if (GetOpenType(arg) != null) + { + throw new NotSupportedException("Embedded Open Types are only supported for types with a single generic argument."); + } + } + } + } + + return null; + } + + public void AddConverter( + Func> converterBuilder) + where TAttribute : Attribute + { + var openTypeSource = GetOpenType(); + var openTypeDest = GetOpenType(); + + if (openTypeSource == null) + { + openTypeSource = new ExactMatch(typeof(TSrc)); + } + if (openTypeDest == null) + { + openTypeDest = new ExactMatch(typeof(TDest)); + } + + AddOpenConverter(openTypeSource, openTypeDest, converterBuilder); + } + public void AddConverter(FuncConverter converter) where TAttribute : Attribute - { + { string key = GetKey(); - _funcsWithAttr[key] = converter; + _funcsWithAttr[key] = converter; } // Return null if not found. @@ -46,15 +174,16 @@ private FuncConverter TryGetConverter(); - if (_funcsWithAttr.TryGetValue(key1, out obj)) - { + string keySpecific = GetKey(); + if (_funcsWithAttr.TryGetValue(keySpecific, out obj)) + { var func = (FuncConverter)obj; return func; } - // No specific case, lookup in the general purpose case. - string key2 = GetKey(); - if (_funcsWithAttr.TryGetValue(key2, out obj)) + + // No specific case, lookup in the general purpose case. + string keyGeneral = GetKey(); + if (_funcsWithAttr.TryGetValue(keyGeneral, out obj)) { var func1 = (FuncConverter)obj; FuncConverter func2 = (src, attr, context) => func1(src, null, context); @@ -77,6 +206,26 @@ public FuncConverter GetConverter would catch everything. + // Users can still register an explicit T-->object converter if they want to + // support it. + if (typeDest != typeof(Object)) + { + return (src, attr, context) => + { + object obj = (object)src; + return (TDest)obj; + }; + } + } + // Object --> TDest // Catch all for any conversion to TDest var objConversion = TryGetConverter(); @@ -89,14 +238,13 @@ public FuncConverter GetConverter + var builder = TryGetOpenConverter(typeSource, typeDest, typeof(TAttribute)); + if (builder != null) { - object obj = (object)src; - return (TDest)obj; - }; + var converter = builder(typeSource, typeDest); + return (src, attr, context) => (TDest)converter(src); + } } // string --> TDest @@ -107,7 +255,7 @@ public FuncConverter GetConverter TDest - if (typeof(TSrc) == typeof(string)) + if (typeSource == typeof(string)) { return (src, attr, context) => { @@ -120,7 +268,7 @@ public FuncConverter GetConverter String --> TDest - if (typeof(TSrc) == typeof(byte[])) + if (typeSource == typeof(byte[])) { var bytes2string = TryGetConverter(); @@ -135,9 +283,9 @@ public FuncConverter GetConverter GetConverter> Builder { get; set; } + } + + // "Empty" converter manager that only allows identity conversions. + // This is useful for constrained rules that don't want to operate against exact types and skip + // arbitrary user conversions. + private class IdentityConverterManager : IConverterManager + { + public void AddConverter(FuncConverter converter) where TAttribute1 : Attribute + { + throw new NotImplementedException(); + } + + public void AddConverter(Func> converterBuilder) where TAttribute1 : Attribute + { + throw new NotImplementedException(); + } + + public FuncConverter GetConverter() where TAttribute1 : Attribute + { + if (typeof(TSource) != typeof(TDestination)) + { + return null; + } + return (src, attr, ctx) => + { + object obj = (object)src; + return (TDestination)obj; + }; + } + } + + // Match a generic type with 1 generic arg. + // like IEnumerable, IQueryable, etc. + private class SingleGenericArgOpenType : OpenType + { + private readonly OpenType _inner; + private readonly Type _outerType; + + public SingleGenericArgOpenType(Type outerType, OpenType inner) + { + _inner = inner; + _outerType = outerType; + } + + public override bool IsMatch(Type type) + { + if (type == null) + { + throw new ArgumentNullException("type"); + } + if (type.IsGenericType && + type.GetGenericTypeDefinition() == _outerType) + { + var args = type.GetGenericArguments(); + + return _inner.IsMatch(args[0]); + } + + return false; + } + } + + // Bind to any type + private class AnythingOpenType : OpenType + { + public override bool IsMatch(Type type) + { + return true; + } + } + + private class ExactMatch : OpenType + { + private readonly Type _type; + public ExactMatch(Type type) + { + _type = type; + } + public override bool IsMatch(Type type) + { + return type == _type; + } + } + + // Matches any T[] + private class ArrayOpenType : OpenType + { + private readonly OpenType _inner; + public ArrayOpenType(OpenType inner) + { + _inner = inner; + } + public override bool IsMatch(Type type) + { + if (type == null) + { + throw new ArgumentNullException(nameof(type)); + } + if (type.IsArray) + { + var elementType = type.GetElementType(); + return _inner.IsMatch(elementType); + } + return false; + } + } } // end class ConverterManager } \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/DefaultResolutionPolicy.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/DefaultResolutionPolicy.cs new file mode 100644 index 000000000..d17896e46 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/DefaultResolutionPolicy.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using Microsoft.Azure.WebJobs.Host.Bindings; +using Microsoft.Azure.WebJobs.Host.Bindings.Path; + +namespace Microsoft.Azure.WebJobs +{ + /// + /// Resolution policy for { } in binding templates. + /// The default policy is just a direct substitution for the binding data. + /// Derived policies can enforce formatting / escaping when they do injection. + /// + internal class DefaultResolutionPolicy : IResolutionPolicy + { + public string TemplateBind(PropertyInfo propInfo, Attribute attribute, BindingTemplate template, IReadOnlyDictionary bindingData) + { + return template.Bind(bindingData); + } + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/FunctionBinding.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/FunctionBinding.cs index ecdc927f3..fc9d5ecf0 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/FunctionBinding.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/FunctionBinding.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Protocols; @@ -45,7 +46,7 @@ internal static BindingContext NewBindingContext( bindingData[kv.Key] = kv.Value; } } - + BindingContext bindingContext = new BindingContext(context, bindingData); return bindingContext; } diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/IResolutionPolicy.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/IResolutionPolicy.cs new file mode 100644 index 000000000..4758a64fb --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/IResolutionPolicy.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using Microsoft.Azure.WebJobs.Host.Bindings.Path; + +namespace Microsoft.Azure.WebJobs.Host.Bindings +{ + /// + /// Resolution policy for "{ }" in binding templates. + /// The default policy is a direct substitution for the binding data. + /// Derived policies can enforce formatting or escaping when they do injection. + /// + public interface IResolutionPolicy + { + /// + /// Resolves the provided . + /// + /// The property being resolved. + /// The Attribute being resolved. + /// The BindingTemplate for the current property. + /// The data for the current function invocation. + /// The resolved property value. + string TemplateBind(PropertyInfo propInfo, Attribute resolvedAttribute, BindingTemplate bindingTemplate, IReadOnlyDictionary bindingData); + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/IValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/IValueProvider.cs index 259cea170..090d4c422 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/IValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/IValueProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading.Tasks; namespace Microsoft.Azure.WebJobs.Host.Bindings { @@ -18,8 +19,8 @@ public interface IValueProvider /// /// Gets the value. /// - /// The value. - object GetValue(); + /// A task that returns the value. + Task GetValueAsync(); /// /// Returns a string representation of the value. diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/ODataFilterResolutionPolicy.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/ODataFilterResolutionPolicy.cs new file mode 100644 index 000000000..1452df0bd --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/ODataFilterResolutionPolicy.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using Microsoft.Azure.WebJobs.Host.Bindings.Path; +using Microsoft.Azure.WebJobs.Host.Tables; + +namespace Microsoft.Azure.WebJobs.Host.Bindings +{ + internal class ODataFilterResolutionPolicy : IResolutionPolicy + { + public string TemplateBind(PropertyInfo propInfo, Attribute attribute, BindingTemplate template, IReadOnlyDictionary bindingData) + { + return TableFilterFormatter.Format(template, bindingData); + } + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/ObjectValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/ObjectValueProvider.cs index f502d37a6..2c64d526d 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/ObjectValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/ObjectValueProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading.Tasks; namespace Microsoft.Azure.WebJobs.Host.Bindings { @@ -26,9 +27,9 @@ public Type Type get { return _valueType; } } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/OpenType.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/OpenType.cs new file mode 100644 index 000000000..560a6c415 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/OpenType.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; + +namespace Microsoft.Azure.WebJobs.Host.Bindings +{ + /// + /// Placeholder to use with converter manager for describing generic types. + /// Derived classes can override IsMatch to provide a constraint. + /// OpenType matches any type. + /// MyDerivedType matches any type where IsMatch(type) is true. + /// Also applies to generics such as: + /// GenericClass<OpenType> + /// + [Obsolete("Not ready for public consumption.")] + public abstract class OpenType + { + /// + /// Return true if and only if given type matches. + /// + /// Type to check + /// + public abstract bool IsMatch(Type type); + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingParameterResolver.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingParameterResolver.cs new file mode 100644 index 000000000..d31063041 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingParameterResolver.cs @@ -0,0 +1,123 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.ObjectModel; +using System.Globalization; +using System.Linq; + +namespace Microsoft.Azure.WebJobs.Host.Bindings.Path +{ + internal abstract class BindingParameterResolver + { + private static Collection _resolvers; + + static BindingParameterResolver() + { + // create the static set of built in system resolvers + _resolvers = new Collection(); + _resolvers.Add(new RandGuidResolver()); + _resolvers.Add(new DateTimeResolver()); + } + + public abstract string Name { get; } + + public static bool IsSystemParameter(string value) + { + if (string.IsNullOrEmpty(value)) + { + throw new ArgumentNullException("value"); + } + + BindingParameterResolver resolver = null; + return TryGetResolver(value, out resolver); + } + + public static bool TryGetResolver(string value, out BindingParameterResolver resolver) + { + if (string.IsNullOrEmpty(value)) + { + throw new ArgumentNullException("value"); + } + + resolver = _resolvers.FirstOrDefault(p => value.StartsWith(p.Name, StringComparison.OrdinalIgnoreCase)); + return resolver != null; + } + + public abstract string Resolve(string value); + + protected string GetFormatOrNull(string value) + { + if (string.IsNullOrEmpty(value)) + { + throw new ArgumentNullException("value"); + } + + if (!value.StartsWith(Name, StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, "The value specified is not a '{0}' binding parameter.", Name), "value"); + } + + if (value.Length > Name.Length && value[Name.Length] == ':') + { + // we have a value of format : + // parse out everything after the first colon + int idx = Name.Length; + return value.Substring(idx + 1); + } + + return null; + } + + private class RandGuidResolver : BindingParameterResolver + { + public override string Name + { + get + { + return "rand-guid"; + } + } + + public override string Resolve(string value) + { + string format = GetFormatOrNull(value); + + if (!string.IsNullOrEmpty(format)) + { + return Guid.NewGuid().ToString(format, CultureInfo.InvariantCulture); + } + else + { + return Guid.NewGuid().ToString(); + } + } + } + + private class DateTimeResolver : BindingParameterResolver + { + public override string Name + { + get + { + return "datetime"; + } + } + + public override string Resolve(string value) + { + string format = GetFormatOrNull(value); + + if (!string.IsNullOrEmpty(format)) + { + return DateTime.UtcNow.ToString(format, CultureInfo.InvariantCulture); + } + else + { + // default to ISO 8601 + return DateTime.UtcNow.ToString("yyyy-MM-ddTHH-mm-ssK", CultureInfo.InvariantCulture); + } + } + } + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingTemplate.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingTemplate.cs index 8a24415b9..f0f3c26b1 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingTemplate.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingTemplate.cs @@ -103,15 +103,18 @@ public string Bind(IReadOnlyDictionary parameters) { if (token.IsParameter) { - // first try to resolve from the binding data parameters - // next try to resolve "built-in" parameters (e.g. rand-guid) string value; + BindingParameterResolver resolver = null; if (parameters != null && parameters.TryGetValue(token.Value, out value)) { + // parameter is resolved from binding data builder.Append(value); } - else if (TryResolveSystem(token.Value, out value)) + else if (BindingParameterResolver.TryGetResolver(token.Value, out resolver)) { + // parameter maps to one of the built-in system binding + // parameters (e.g. rand-guid, datetime, etc.) + value = resolver.Resolve(token.Value); builder.Append(value); } else @@ -136,32 +139,5 @@ public override string ToString() { return _pattern; } - - /// - /// Try to resolve as a built-in "system" parameter. These are built in values - /// that don't come from the binding data bag. - /// - private static bool TryResolveSystem(string value, out string resolvedValue) - { - resolvedValue = null; - - if (string.Compare(value, "rand-guid", StringComparison.OrdinalIgnoreCase) == 0) - { - resolvedValue = Guid.NewGuid().ToString(); - return true; - } - - return false; - } - - internal static bool IsSystemBindingParameter(string parameterName) - { - if (string.Compare(parameterName, "rand-guid", StringComparison.OrdinalIgnoreCase) == 0) - { - return true; - } - - return false; - } } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingTemplateParser.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingTemplateParser.cs index 94761608d..fe4243cfb 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingTemplateParser.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/Path/BindingTemplateParser.cs @@ -154,7 +154,7 @@ public static IEnumerable GetTokens(string input) private static bool IsValidIdentifier(string identifier) { // built-in sysetem identifiers are valid - if (BindingTemplate.IsSystemBindingParameter(identifier)) + if (BindingParameterResolver.IsSystemParameter(identifier)) { return true; } diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/PatternMatcher.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/PatternMatcher.cs new file mode 100644 index 000000000..4c49e86b0 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/PatternMatcher.cs @@ -0,0 +1,383 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Azure.WebJobs.Host.Bindings +{ + // Find a Convert() method on a class that matches the type parameters. + internal abstract class PatternMatcher + { + // Get the type of the converter object. Used with FindAndCreateConverter. + protected abstract Type TypeConverter { get; } + + public static PatternMatcher New(Type typeBuilder, params object[] constructorArgs) + { + return new CreateViaType(typeBuilder, constructorArgs); + } + + public static PatternMatcher New(object instance) + { + return new CreateViaInstance(instance); + } + + /// + /// Get a converter function for the given types. + /// Types can be open generics, so this needs to find the appropriate IConverter interface and pattern match. + /// Throws if can't match - caller is responsible for doing type validation before invoking. + /// + /// source type + /// destination type + /// converter function that takes in source type and returns destination type + public abstract Func TryGetConverterFunc(Type typeSource, Type typeDest); + + // Find an IConverter on the typeConverter interface where Tin,Tout are + // compatible with TypeSource,typeDest. + // Where TIn, TOut may be generic. This will infer the generics and resolved the + // generic converter interface. + // Instantiate a func that will invoke the converter. + private Func FindAndCreateConverter( + Type typeSource, + Type typeDest) + { + Type typeConverter = this.TypeConverter; + + // Search for IConverter<> interfaces on the type converter object. + var interfaces = typeConverter.GetInterfaces(); + foreach (var iface in interfaces) + { + // verify it's an IConverter + + if (!iface.IsGenericType) + { + continue; + } + + bool isTask = false; + + if (iface.GetGenericTypeDefinition() != typeof(IConverter<,>)) + { + if (iface.GetGenericTypeDefinition() != typeof(IAsyncConverter<,>)) + { + continue; + } + isTask = true; + } + + Type typeInput = iface.GetGenericArguments()[0]; + Type typeOutput = iface.GetGenericArguments()[1]; + + // Does it match? + // (typeInput,typeOutput) is on the converter's static interface and may be generic. + // (typeSource,typeDest) is the runtime type and is concerete. + Dictionary genericArgs = new Dictionary(); + + if (!CheckArg(typeOutput, typeDest, genericArgs)) + { + continue; + } + + if (!CheckArg(typeInput, typeSource, genericArgs)) + { + continue; + } + + // Found a match. Now instantiate the converter func. + object instance = this.GetInstance(genericArgs); + + // Create an invoker object + typeInput = ResolveGenerics(typeInput, genericArgs); + typeOutput = ResolveGenerics(typeOutput, genericArgs); + return CreateConverterFunc(isTask, typeInput, typeOutput, instance); + } + + throw new InvalidOperationException($"No Convert method on type {typeConverter.Name} to convert from " + + $"{typeSource.Name} to {typeDest.Name}"); + } + + // Find IConverter on the object instance. + // type parameters should be resolved concrete types. + private static Func CreateConverterFunc( + bool isTask, // IConverter vs. IAsyncConverter + Type typeInput, + Type typeOutput, + object instance) + { + Type invokeType = isTask ? typeof(AsyncInvoker<,>) : typeof(Invoker<,>); + var typeInvoker = invokeType.MakeGenericType(typeInput, typeOutput); + var instanceInvoker = (InvokerBase)Activator.CreateInstance(typeInvoker); + var func = instanceInvoker.Work(instance); + return func; + } + + // Instantiate a type converter given the generic args. + protected abstract object GetInstance(Dictionary genericArgs); + + // Given a type, resolve any generic parameters and return the resolved type. This can be recursive. + // string - non-generic type, does not contain generic parameters. + // T - type parameter + // IEnumerable`1 - generic type definition + // IEnumerable - type signature with 1 generic parameter + // IConvereter - type signature with 2 generic parameters, the 2nd is an open generic. + internal static Type ResolveGenerics(Type type, Dictionary genericArgs) + { + // A generic type definition is an actual class / interface declaration, + // not to be confused with a signature referenced by the definition. + // Here, Foo'1 is the generic type definition, with one generic parameter, named 'T' + // IEnumerable is a type signature that containes a generic parameter. + // T is the generic parameter. + // class Foo : IEnumerable + // + // MakeGenericType can only be called on a GenericTypeDefinition. + if (type.IsGenericTypeDefinition) + { + var typeArgs = type.GetGenericArguments(); + int len = typeArgs.Length; + var actualTypeArgs = new Type[len]; + for (int i = 0; i < len; i++) + { + actualTypeArgs[i] = genericArgs[typeArgs[i].Name]; + } + + var resolvedType = type.MakeGenericType(actualTypeArgs); + + return resolvedType; + } + else + { + // Simple case: T + if (type.IsGenericParameter) + { + var actual = genericArgs[type.Name]; + return actual; + } + else if (type.ContainsGenericParameters) + { + if (type.IsArray) + { + // T[] + var elementType = type.GetElementType(); + var resolved = ResolveGenerics(elementType, genericArgs); + return resolved.MakeArrayType(); + } + // eg, IEnumerable, IConverter + // Must decompose to the generic definition, resolve each arg, and build back up. + + // potentially recursive case: ie, IConverter + var def = type.GetGenericTypeDefinition(); + var args = type.GetGenericArguments(); + + var resolvedArgs = new Type[args.Length]; + for (int i = 0; i < args.Length; i++) + { + resolvedArgs[i] = ResolveGenerics(args[i], genericArgs); + } + + var finalType = def.MakeGenericType(resolvedArgs); + return finalType; + } + else + { + // Easy non-generic case. ie: string, int + return type; + } + } + } + + // Name can only map to a single type. If try to map to difference types, then it's a failed match. + private static bool AddGenericArg(Dictionary genericArgs, string name, Type type) + { + Type typeExisting; + if (genericArgs.TryGetValue(name, out typeExisting)) + { + return typeExisting == type; + } + genericArgs[name] = type; + return true; + } + + // Check if specificType is a valid instance of openType. + // Return true if the types are compatible. + // If openType has generic args, then add a [Name,Type] entry to the genericArgs dictionary. + private static bool CheckArg(Type openType, Type specificType, Dictionary genericArgs) + { + if (openType == specificType) + { + return true; + } + + if (openType.IsAssignableFrom(specificType)) + { + // Allow derived types. + return true; + } + + // Is it a generic match? + // T, string + if (openType.IsGenericParameter) + { + string name = openType.Name; + return AddGenericArg(genericArgs, name, specificType); + } + + // T[] + if (openType.IsArray && specificType.IsArray) + { + var elementType = openType.GetElementType(); + var specificElementType = specificType.GetElementType(); + if (elementType.IsGenericParameter) + { + if (!AddGenericArg(genericArgs, elementType.Name, specificElementType)) + { + return false; + } + return true; + } + } + + // IFoo, IFoo + if (specificType.IsGenericType && openType.IsGenericType) + { + if (specificType.GetGenericTypeDefinition() != openType.GetGenericTypeDefinition()) + { + return false; + } + + var typeArgs = openType.GetGenericArguments(); + var specificTypeArgs = specificType.GetGenericArguments(); + + int len = typeArgs.Length; + + for (int i = 0; i < len; i++) + { + if (!AddGenericArg(genericArgs, typeArgs[i].Name, specificTypeArgs[i])) + { + return false; + } + } + return true; + } + + return false; + } + + // Helper for getting a converter func that invokes the converter interface on an object. + private abstract class InvokerBase + { + public abstract Func Work(object instance); + } + + // Get a converter function that invokes IConverter on an object. + private class Invoker : InvokerBase + { + public override Func Work(object instance) + { + IConverter converter = (IConverter)instance; + + Func func = (input) => + { + TSrc src = (TSrc)input; + var result = converter.Convert(src); + return result; + }; + return func; + } + } + + // Get a converter function that invokes IAsyncConverter on an object. + private class AsyncInvoker : InvokerBase + { + public override Func Work(object instance) + { + IAsyncConverter converter = (IAsyncConverter)instance; + + Func func = (input) => + { + TSrc src = (TSrc)input; + Task resultTask = Task.Run(() => converter.ConvertAsync(src, CancellationToken.None)); + + TDest result = resultTask.GetAwaiter().GetResult(); + return result; + }; + return func; + } + } + + // Wrapper for matching against a static type. + private class CreateViaType : PatternMatcher + { + private readonly Type _typeConverter; + private readonly object[] _constructorArgs; + + public CreateViaType(Type builderType, object[] constructorArgs) + { + _typeConverter = builderType; + _constructorArgs = constructorArgs; + } + + protected override Type TypeConverter + { + get + { + return this._typeConverter; + } + } + + public override Func TryGetConverterFunc(Type typeSource, Type typeDest) + { + var func = this.FindAndCreateConverter(typeSource, typeDest); + return func; + } + + protected override object GetInstance(Dictionary genericArgs) + { + Type finalType = ResolveGenerics(_typeConverter, genericArgs); + + try + { + // common for constructor to throw validation errors. + var instance = Activator.CreateInstance(finalType, _constructorArgs); + return instance; + } + catch (TargetInvocationException e) + { + throw e.InnerException; + } + } + } + + // Wrapper for matching against an instance. + private class CreateViaInstance : PatternMatcher + { + private readonly object _instance; + + public CreateViaInstance(object instance) + { + _instance = instance; + } + + protected override Type TypeConverter + { + get + { + return _instance.GetType(); + } + } + + public override Func TryGetConverterFunc(Type typeSource, Type typeDest) + { + var func = this.FindAndCreateConverter(typeSource, typeDest); + return func; + } + + protected override object GetInstance(Dictionary genericArgs) + { + return _instance; + } + } + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/Runtime/Binder.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/Runtime/Binder.cs index 7c073a67d..dd8f27670 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/Runtime/Binder.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/Runtime/Binder.cs @@ -133,7 +133,8 @@ IWatcher IWatchable.Watcher _disposable.Add(disposableProvider); } - return (TValue)provider.GetValue(); + object result = await provider.GetValueAsync(); + return (TValue)result; } /// diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/Runtime/RuntimeValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/Runtime/RuntimeValueProvider.cs index f55d8387c..98831ab5d 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/Runtime/RuntimeValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/Runtime/RuntimeValueProvider.cs @@ -24,9 +24,9 @@ public Type Type get { return _parameterType; } } - public object GetValue() + public Task GetValueAsync() { - return _binder; + return Task.FromResult(_binder); } public async Task SetValueAsync(object value, CancellationToken cancellationToken) diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/StorageAccount/CloudStorageAccountValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/StorageAccount/CloudStorageAccountValueProvider.cs index e05a36332..888e921c8 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/StorageAccount/CloudStorageAccountValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/StorageAccount/CloudStorageAccountValueProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage; namespace Microsoft.Azure.WebJobs.Host.Bindings.StorageAccount @@ -20,9 +21,9 @@ public Type Type get { return typeof(CloudStorageAccount); } } - public object GetValue() + public Task GetValueAsync() { - return _account; + return Task.FromResult(_account); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Bindings/TraceWriter/TraceWriterBinding.cs b/src/Microsoft.Azure.WebJobs.Host/Bindings/TraceWriter/TraceWriterBinding.cs index d0d8f1b0c..49eb7e943 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Bindings/TraceWriter/TraceWriterBinding.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Bindings/TraceWriter/TraceWriterBinding.cs @@ -83,9 +83,9 @@ public Type Type get { return _type; } } - public object GetValue() + public Task GetValueAsync() { - return _tracer; + return Task.FromResult(_tracer); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobContainerArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobContainerArgumentBindingProvider.cs index 5b540cf25..80e28f118 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobContainerArgumentBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobContainerArgumentBindingProvider.cs @@ -56,9 +56,9 @@ public Type Type } } - public object GetValue() + public Task GetValueAsync() { - return _container; + return Task.FromResult(_container); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobDirectoryArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobDirectoryArgumentBindingProvider.cs index 72f8cff56..3016852ed 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobDirectoryArgumentBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobDirectoryArgumentBindingProvider.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; -using System.IO; using System.Reflection; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; @@ -60,9 +59,9 @@ public Type Type } } - public object GetValue() + public Task GetValueAsync() { - return _directory; + return Task.FromResult(_directory); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobEnumerableArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobEnumerableArgumentBindingProvider.cs index 575974f37..aad235db7 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobEnumerableArgumentBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobEnumerableArgumentBindingProvider.cs @@ -8,7 +8,6 @@ using System.Reflection; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Blobs.Bindings; using Microsoft.Azure.WebJobs.Host.Blobs.Listeners; using Microsoft.Azure.WebJobs.Host.Storage.Blob; using Microsoft.WindowsAzure.Storage.Blob; @@ -68,7 +67,7 @@ public async Task BindAsync(IStorageBlobContainer container, Val } private static async Task ConvertBlobs(Type targetType, IEnumerable blobItems) - { + { Type listType = typeof(List<>).MakeGenericType(targetType); IList list = (IList)Activator.CreateInstance(listType); foreach (var blobItem in blobItems) @@ -126,9 +125,9 @@ public Type Type } } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobStreamArgumentBinderProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobStreamArgumentBinderProvider.cs index 7d7f664e1..bb5f8d08c 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobStreamArgumentBinderProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/CloudBlobStreamArgumentBinderProvider.cs @@ -96,15 +96,15 @@ public IWatcher Watcher get { return _stream; } } - public object GetValue() + public Task GetValueAsync() { - return _stream; + return Task.FromResult(_stream); } public async Task SetValueAsync(object value, CancellationToken cancellationToken) { - Debug.Assert(value == null || value == GetValue(), - "The value argument should be either the same instance as returned by GetValue() or null"); + Debug.Assert(value == null || value == await GetValueAsync(), + "The value argument should be either the same instance as returned by GetValueAsync() or null"); // Not ByRef, so can ignore value argument. diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutByteArrayArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutByteArrayArgumentBindingProvider.cs index 1df511cc0..45f056a4e 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutByteArrayArgumentBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutByteArrayArgumentBindingProvider.cs @@ -95,9 +95,9 @@ public IWatcher Watcher get { return _stream; } } - public object GetValue() + public Task GetValueAsync() { - return null; + return Task.FromResult(null); } /// diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutObjectArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutObjectArgumentBindingProvider.cs index 1dde11377..4ab9e79bf 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutObjectArgumentBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutObjectArgumentBindingProvider.cs @@ -108,9 +108,9 @@ public IWatcher Watcher get { return _stream; } } - public object GetValue() + public Task GetValueAsync() { - return default(TValue); + return Task.FromResult(default(TValue)); } public async Task SetValueAsync(object value, CancellationToken cancellationToken) diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutStringArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutStringArgumentBindingProvider.cs index d517bc2f8..35f859982 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutStringArgumentBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/OutStringArgumentBindingProvider.cs @@ -96,9 +96,9 @@ public IWatcher Watcher get { return _stream; } } - public object GetValue() + public Task GetValueAsync() { - return null; + return Task.FromResult(null); } /// diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/TextWriterArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/TextWriterArgumentBindingProvider.cs index f28809693..22221ad56 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/TextWriterArgumentBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Bindings/TextWriterArgumentBindingProvider.cs @@ -103,15 +103,15 @@ public IWatcher Watcher get { return _stream; } } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public async Task SetValueAsync(object value, CancellationToken cancellationToken) { - Debug.Assert(value == null || value == GetValue(), - "The value argument should be either the same instance as returned by GetValue() or null"); + Debug.Assert(value == null || value == await GetValueAsync(), + "The value argument should be either the same instance as returned by GetValueAsync() or null"); // Not ByRef, so can ignore value argument. cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobValueProvider.cs index 5236acf66..a9bc944f1 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobValueProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; using Microsoft.Azure.WebJobs.Host.Storage.Blob; @@ -40,9 +41,9 @@ public static BlobValueProvider CreateWithNull(IStorageBlob blob) where T : c return new BlobValueProvider(blob, value: null, valueType: typeof(T)); } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobWatchableDisposableValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobWatchableDisposableValueProvider.cs index d7ddf75d5..66a57e766 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobWatchableDisposableValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobWatchableDisposableValueProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; using Microsoft.Azure.WebJobs.Host.Storage.Blob; @@ -42,9 +43,9 @@ public IWatcher Watcher get { return _watcher; } } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobWatchableValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobWatchableValueProvider.cs index e661374ac..258133cca 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobWatchableValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/BlobWatchableValueProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; using Microsoft.Azure.WebJobs.Host.Storage.Blob; @@ -42,9 +43,9 @@ public static BlobWatchableValueProvider Create(IStorageBlob blob, T value, I return new BlobWatchableValueProvider(blob, value: value, valueType: typeof(T), watcher: watcher); } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobListenerFactory.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobListenerFactory.cs index 6f774a3e7..932b7ac3d 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobListenerFactory.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobListenerFactory.cs @@ -20,6 +20,7 @@ internal class BlobListenerFactory : IListenerFactory private const string SingletonBlobListenerScopeId = "WebJobs.Internal.Blobs"; private readonly IHostIdProvider _hostIdProvider; private readonly IQueueConfiguration _queueConfiguration; + private readonly JobHostBlobsConfiguration _blobsConfiguration; private readonly IWebJobsExceptionHandler _exceptionHandler; private readonly IContextSetter _blobWrittenWatcherSetter; private readonly IContextSetter _messageEnqueuedWatcherSetter; @@ -35,6 +36,7 @@ internal class BlobListenerFactory : IListenerFactory public BlobListenerFactory(IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, + JobHostBlobsConfiguration blobsConfiguration, IWebJobsExceptionHandler exceptionHandler, IContextSetter blobWrittenWatcherSetter, IContextSetter messageEnqueuedWatcherSetter, @@ -58,6 +60,11 @@ public BlobListenerFactory(IHostIdProvider hostIdProvider, throw new ArgumentNullException("queueConfiguration"); } + if (blobsConfiguration == null) + { + throw new ArgumentNullException("blobsConfiguration"); + } + if (exceptionHandler == null) { throw new ArgumentNullException("exceptionHandler"); @@ -115,6 +122,7 @@ public BlobListenerFactory(IHostIdProvider hostIdProvider, _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; + _blobsConfiguration = blobsConfiguration; _exceptionHandler = exceptionHandler; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; @@ -133,12 +141,18 @@ public async Task CreateAsync(CancellationToken cancellationToken) { // Note that these clients are intentionally for the storage account rather than for the dashboard account. // We use the storage, not dashboard, account for the blob receipt container and blob trigger queues. - IStorageQueueClient queueClient = _hostAccount.CreateQueueClient(); - IStorageBlobClient blobClient = _hostAccount.CreateBlobClient(); + IStorageQueueClient primaryQueueClient = _hostAccount.CreateQueueClient(); + IStorageBlobClient primaryBlobClient = _hostAccount.CreateBlobClient(); + + // Important: We're using the storage account of the function target here, which is the account that the + // function the listener is for is targeting. This is the account that will be used + // to read the trigger blob. + IStorageBlobClient targetBlobClient = _dataAccount.CreateBlobClient(); + IStorageQueueClient targetQueueClient = _dataAccount.CreateQueueClient(); string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken); string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId); - IStorageQueue hostBlobTriggerQueue = queueClient.GetQueueReference(hostBlobTriggerQueueName); + IStorageQueue hostBlobTriggerQueue = primaryQueueClient.GetQueueReference(hostBlobTriggerQueueName); SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreateInstance( new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter)); @@ -147,23 +161,30 @@ public async Task CreateAsync(CancellationToken cancellationToken) new SharedBlobListenerFactory(hostId, _hostAccount, _exceptionHandler, _blobWrittenWatcherSetter)); // Register the blob container we wish to monitor with the shared blob listener. - await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, blobClient, + await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, primaryBlobClient, hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken); // Create a "bridge" listener that will monitor the blob // notification queue and dispatch to the target job function. SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance( - new SharedBlobQueueListenerFactory(sharedQueueWatcher, queueClient, hostBlobTriggerQueue, + new SharedBlobQueueListenerFactory(_hostAccount, sharedQueueWatcher, hostBlobTriggerQueue, _queueConfiguration, _exceptionHandler, _trace, sharedBlobListener.BlobWritterWatcher)); var queueListener = new BlobListener(sharedBlobQueueListener); - // Important: We're using the "data account" here, which is the account that the - // function the listener is for is targeting. This is the account that will be used - // to read the trigger blob. - IStorageBlobClient userBlobClient = _dataAccount.CreateBlobClient(); + // determine which client to use for the poison queue + // by default this should target the same storage account + // as the blob container we're monitoring + var poisonQueueClient = targetQueueClient; + if (_dataAccount.Type != StorageAccountType.GeneralPurpose || + _blobsConfiguration.CentralizedPoisonQueue) + { + // use the primary storage account if the centralize flag is true, + // or if the target storage account doesn't support queues + poisonQueueClient = primaryQueueClient; + } - // Register our function with the shared queue listener - RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, userBlobClient); + // Register our function with the shared blob queue listener + RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, targetBlobClient, poisonQueueClient); // check a flag in the shared context to see if we've created the singleton // shared blob listener in this host instance @@ -206,12 +227,14 @@ private async Task RegisterWithSharedBlobListenerAsync( private void RegisterWithSharedBlobQueueListenerAsync( SharedBlobQueueListener sharedBlobQueueListener, - IStorageBlobClient blobClient) + IStorageBlobClient blobClient, + IStorageQueueClient queueClient) { BlobQueueRegistration registration = new BlobQueueRegistration { Executor = _executor, - BlobClient = blobClient + BlobClient = blobClient, + QueueClient = queueClient }; sharedBlobQueueListener.Register(_functionId, registration); diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobLogListener.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobLogListener.cs index 2d37374ec..d13e868a8 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobLogListener.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobLogListener.cs @@ -64,11 +64,10 @@ public async Task> GetRecentBlobWritesAsync(Cancellati _scannedBlobNames.Clear(); } - var parsedBlobPaths = from entry in await _parser.ParseLogAsync(blob, cancellationToken) - where entry.IsBlobWrite - select entry.ToBlobPath(); + IEnumerable entries = await _parser.ParseLogAsync(blob, cancellationToken); + IEnumerable filteredBlobs = GetPathsForValidBlobWrites(entries); - foreach (BlobPath path in parsedBlobPaths.Where(p => p != null)) + foreach (BlobPath path in filteredBlobs) { IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName); blobs.Add(container.GetBlockBlobReference(path.BlobName)); @@ -78,6 +77,15 @@ where entry.IsBlobWrite return blobs; } + internal static IEnumerable GetPathsForValidBlobWrites(IEnumerable entries) + { + IEnumerable parsedBlobPaths = from entry in entries + where entry.IsBlobWrite + select entry.ToBlobPath(); + + return parsedBlobPaths.Where(p => p != null); + } + // Return a search prefix for the given start,end time. // $logs/YYYY/MM/DD/HH00 private static string GetSearchPrefix(string service, DateTime startTime, DateTime endTime) diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobQueueRegistration.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobQueueRegistration.cs index 2143e8801..391151d22 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobQueueRegistration.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobQueueRegistration.cs @@ -3,6 +3,7 @@ using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Azure.WebJobs.Host.Storage.Blob; +using Microsoft.Azure.WebJobs.Host.Storage.Queue; namespace Microsoft.Azure.WebJobs.Host.Blobs.Listeners { @@ -23,5 +24,10 @@ internal class BlobQueueRegistration /// to). /// public IStorageBlobClient BlobClient { get; set; } + + /// + /// The storage client to use for the poison queue. + /// + public IStorageQueueClient QueueClient { get; set; } } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobQueueTriggerExecutor.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobQueueTriggerExecutor.cs index cfa7d135a..8cfab6a1d 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobQueueTriggerExecutor.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobQueueTriggerExecutor.cs @@ -35,6 +35,11 @@ public BlobQueueTriggerExecutor(IBlobETagReader eTagReader, _registrations = new ConcurrentDictionary(); } + public bool TryGetRegistration(string functionId, out BlobQueueRegistration registration) + { + return _registrations.TryGetValue(functionId, out registration); + } + public void Register(string functionId, BlobQueueRegistration registration) { _registrations.AddOrUpdate(functionId, registration, (i1, i2) => registration); diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobTriggerQueueWriter.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobTriggerQueueWriter.cs index 4eed7a376..3d533950d 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobTriggerQueueWriter.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/BlobTriggerQueueWriter.cs @@ -7,7 +7,6 @@ using Microsoft.Azure.WebJobs.Host.Protocols; using Microsoft.Azure.WebJobs.Host.Queues; using Microsoft.Azure.WebJobs.Host.Storage.Queue; -using Microsoft.WindowsAzure.Storage.Queue; using Newtonsoft.Json; namespace Microsoft.Azure.WebJobs.Host.Blobs.Listeners diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/SharedBlobQueueListenerFactory.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/SharedBlobQueueListenerFactory.cs index 1ef71af5d..1c7bdc512 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/SharedBlobQueueListenerFactory.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/SharedBlobQueueListenerFactory.cs @@ -2,41 +2,46 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Listeners; using Microsoft.Azure.WebJobs.Host.Queues; using Microsoft.Azure.WebJobs.Host.Queues.Listeners; +using Microsoft.Azure.WebJobs.Host.Storage; using Microsoft.Azure.WebJobs.Host.Storage.Queue; using Microsoft.Azure.WebJobs.Host.Timers; +using Microsoft.WindowsAzure.Storage.Queue; +using Newtonsoft.Json; namespace Microsoft.Azure.WebJobs.Host.Blobs.Listeners { internal class SharedBlobQueueListenerFactory : IFactory { private readonly SharedQueueWatcher _sharedQueueWatcher; - private readonly IStorageQueueClient _queueClient; private readonly IStorageQueue _hostBlobTriggerQueue; private readonly IQueueConfiguration _queueConfiguration; private readonly IWebJobsExceptionHandler _exceptionHandler; private readonly TraceWriter _trace; private readonly IBlobWrittenWatcher _blobWrittenWatcher; + private readonly IStorageAccount _hostAccount; public SharedBlobQueueListenerFactory( + IStorageAccount hostAccount, SharedQueueWatcher sharedQueueWatcher, - IStorageQueueClient queueClient, IStorageQueue hostBlobTriggerQueue, IQueueConfiguration queueConfiguration, IWebJobsExceptionHandler exceptionHandler, TraceWriter trace, IBlobWrittenWatcher blobWrittenWatcher) { - if (sharedQueueWatcher == null) + if (hostAccount == null) { - throw new ArgumentNullException("sharedQueueWatcher"); + throw new ArgumentNullException("hostAccount"); } - if (queueClient == null) + if (sharedQueueWatcher == null) { - throw new ArgumentNullException("queueClient"); + throw new ArgumentNullException("sharedQueueWatcher"); } if (hostBlobTriggerQueue == null) @@ -64,8 +69,8 @@ public SharedBlobQueueListenerFactory( throw new ArgumentNullException("blobWrittenWatcher"); } + _hostAccount = hostAccount; _sharedQueueWatcher = sharedQueueWatcher; - _queueClient = queueClient; _hostBlobTriggerQueue = hostBlobTriggerQueue; _queueConfiguration = queueConfiguration; _exceptionHandler = exceptionHandler; @@ -76,15 +81,66 @@ public SharedBlobQueueListenerFactory( [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope")] public SharedBlobQueueListener Create() { - IStorageQueue blobTriggerPoisonQueue = - _queueClient.GetQueueReference(HostQueueNames.BlobTriggerPoisonQueue); - BlobQueueTriggerExecutor triggerExecutor = - new BlobQueueTriggerExecutor(_blobWrittenWatcher); - IDelayStrategy delayStrategy = new RandomizedExponentialBackoffStrategy(QueuePollingIntervals.Minimum, - _queueConfiguration.MaxPollingInterval); - IListener listener = new QueueListener(_hostBlobTriggerQueue, blobTriggerPoisonQueue, triggerExecutor, - delayStrategy, _exceptionHandler, _trace, _sharedQueueWatcher, _queueConfiguration); + BlobQueueTriggerExecutor triggerExecutor = new BlobQueueTriggerExecutor(_blobWrittenWatcher); + + // The poison queue to use for a given poison blob lives in the same + // storage account as the triggering blob by default. In multi-storage account scenarios + // that means that we'll be writing to different poison queues, determined by + // the triggering blob. + // However we use a poison queue in the host storage account as a fallback default + // in case a particular blob lives in a restricted "blob only" storage account (i.e. no queues). + IStorageQueue defaultPoisonQueue = _hostAccount.CreateQueueClient().GetQueueReference(HostQueueNames.BlobTriggerPoisonQueue); + + // this special queue bypasses the QueueProcessorFactory - we don't want people to + // override this + QueueProcessorFactoryContext context = new QueueProcessorFactoryContext(_hostBlobTriggerQueue.SdkObject, _trace, _queueConfiguration, defaultPoisonQueue.SdkObject); + SharedBlobQueueProcessor queueProcessor = new SharedBlobQueueProcessor(context, triggerExecutor); + + IListener listener = new QueueListener(_hostBlobTriggerQueue, defaultPoisonQueue, triggerExecutor, + _exceptionHandler, _trace, _sharedQueueWatcher, _queueConfiguration, queueProcessor); + return new SharedBlobQueueListener(listener, triggerExecutor); } + + /// + /// Custom queue processor for the shared blob queue. + /// + private class SharedBlobQueueProcessor : QueueProcessor + { + private BlobQueueTriggerExecutor _executor; + + public SharedBlobQueueProcessor(QueueProcessorFactoryContext context, BlobQueueTriggerExecutor executor) : base(context) + { + _executor = executor; + } + + protected override Task CopyMessageToPoisonQueueAsync(CloudQueueMessage message, CloudQueue poisonQueue, CancellationToken cancellationToken) + { + // determine the poison queue based on the storage account + // of the triggering blob, or default + poisonQueue = GetPoisonQueue(message) ?? poisonQueue; + + return base.CopyMessageToPoisonQueueAsync(message, poisonQueue, cancellationToken); + } + + private CloudQueue GetPoisonQueue(CloudQueueMessage message) + { + if (message == null) + { + throw new ArgumentNullException("message"); + } + + var blobTriggerMessage = JsonConvert.DeserializeObject(message.AsString); + + BlobQueueRegistration registration = null; + if (_executor.TryGetRegistration(blobTriggerMessage.FunctionId, out registration)) + { + IStorageQueue poisonQueue = registration.QueueClient.GetQueueReference(HostQueueNames.BlobTriggerPoisonQueue); + return poisonQueue.SdkObject; + } + + return null; + } + } } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageAnalyticsLogEntry.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageAnalyticsLogEntry.cs index 3a4ca5ec9..478cfac59 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageAnalyticsLogEntry.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Listeners/StorageAnalyticsLogEntry.cs @@ -58,7 +58,7 @@ public bool IsBlobWrite ServiceType.HasValue && ServiceType == StorageServiceType.Blob && OperationType.HasValue && - ((OperationType == StorageServiceOperationType.ClearPage) || + ((OperationType == StorageServiceOperationType.ClearPage) || (OperationType == StorageServiceOperationType.CopyBlob) || (OperationType == StorageServiceOperationType.CopyBlobDestination) || (OperationType == StorageServiceOperationType.PutBlob) || @@ -87,7 +87,7 @@ public static StorageAnalyticsLogEntry TryParse(string[] fields) var entry = new StorageAnalyticsLogEntry(); DateTime requestStartTime; - if (!DateTime.TryParseExact(fields[(int)StorageAnalyticsLogColumnId.RequestStartTime], "o", + if (!DateTime.TryParseExact(fields[(int)StorageAnalyticsLogColumnId.RequestStartTime], "o", CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out requestStartTime)) { return null; @@ -119,8 +119,6 @@ public static StorageAnalyticsLogEntry TryParse(string[] fields) /// /// A valid instance of , or null if log entry is not associated with a blob. /// - /// If fails to determine blob path components, i.e. account, container name, - /// and blob name. public BlobPath ToBlobPath() { if (!ServiceType.HasValue || ServiceType != StorageServiceType.Blob) @@ -139,23 +137,19 @@ public BlobPath ToBlobPath() else { // assuming key is "/account/container/blob" - int startPos = RequestedObjectKey.Length > 1 ? RequestedObjectKey.IndexOf('/', 1) : -1; + int startPos = RequestedObjectKey.Length > 1 ? RequestedObjectKey.IndexOf('/', 1) : -1; path = startPos != -1 ? RequestedObjectKey.Substring(startPos + 1) : String.Empty; } if (String.IsNullOrEmpty(path)) { - throw new FormatException("Failed to parse RequestedObjectKey property of the log entry. " + - "It should be in one of the supported formats: " + - @"""https://account.blob.core.windows.net/container/blob"", or" + - @"""/account/container/blob"""); + return null; } BlobPath blobPath; if (!BlobPath.TryParse(path, false, out blobPath)) { - throw new FormatException("Failed to parse RequestedObjectKey property of the log entry. " + - "Blob identifiers must be in the format container/blob."); + return null; } return blobPath; diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/StreamArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/StreamArgumentBindingProvider.cs index 46d882b89..cb0af64cc 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/StreamArgumentBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/StreamArgumentBindingProvider.cs @@ -50,9 +50,9 @@ public IBlobArgumentBinding TryCreate(ParameterInfo parameter, FileAccess? attri if (!actualAccess.HasValue) { throw new InvalidOperationException(String.Format(CultureInfo.CurrentCulture, - "FileAccess must be specified when binding the parameter '{0}' to a blob Stream. " + - "Add a FileAccess argument to the BlobAttribute constructor " + - @"(for example, [Blob(""..."", FileAccess.Read)]).", + "FileAccess must be specified when binding the parameter '{0}' to a blob Stream. " + + "Add a FileAccess argument to the BlobAttribute constructor " + + @"(for example, [Blob(""..."", FileAccess.Read)]).", parameter.Name)); } @@ -147,15 +147,15 @@ public IWatcher Watcher get { return _stream; } } - public object GetValue() + public Task GetValueAsync() { - return _stream; + return Task.FromResult(_stream); } public async Task SetValueAsync(object value, CancellationToken cancellationToken) { - Debug.Assert(value == null || value == GetValue(), - "The value argument should be either the same instance as returned by GetValue() or null"); + Debug.Assert(value == null || value == await GetValueAsync(), + "The value argument should be either the same instance as returned by GetValueAsync() or null"); // Not ByRef, so can ignore value argument. diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Triggers/BlobTriggerAttributeBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Triggers/BlobTriggerAttributeBindingProvider.cs index 8e71ac0b2..c557fe35b 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Triggers/BlobTriggerAttributeBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Triggers/BlobTriggerAttributeBindingProvider.cs @@ -28,6 +28,7 @@ internal class BlobTriggerAttributeBindingProvider : ITriggerBindingProvider private readonly IBlobArgumentBindingProvider _provider; private readonly IHostIdProvider _hostIdProvider; private readonly IQueueConfiguration _queueConfiguration; + private readonly JobHostBlobsConfiguration _blobsConfiguration; private readonly IWebJobsExceptionHandler _exceptionHandler; private readonly IContextSetter _blobWrittenWatcherSetter; private readonly IContextSetter _messageEnqueuedWatcherSetter; @@ -40,6 +41,7 @@ public BlobTriggerAttributeBindingProvider(INameResolver nameResolver, IExtensionTypeLocator extensionTypeLocator, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, + JobHostBlobsConfiguration blobsConfiguration, IWebJobsExceptionHandler exceptionHandler, IContextSetter blobWrittenWatcherSetter, IContextSetter messageEnqueuedWatcherSetter, @@ -67,6 +69,11 @@ public BlobTriggerAttributeBindingProvider(INameResolver nameResolver, throw new ArgumentNullException("queueConfiguration"); } + if (blobsConfiguration == null) + { + throw new ArgumentNullException("blobsConfiguration"); + } + if (exceptionHandler == null) { throw new ArgumentNullException("exceptionHandler"); @@ -102,6 +109,7 @@ public BlobTriggerAttributeBindingProvider(INameResolver nameResolver, _provider = CreateProvider(extensionTypeLocator.GetCloudBlobStreamBinderTypes()); _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; + _blobsConfiguration = blobsConfiguration; _exceptionHandler = exceptionHandler; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; @@ -163,7 +171,7 @@ public async Task TryCreateAsync(TriggerBindingProviderContext dataAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly); ITriggerBinding binding = new BlobTriggerBinding(parameter, argumentBinding, hostAccount, dataAccount, path, - _hostIdProvider, _queueConfiguration, _exceptionHandler, _blobWrittenWatcherSetter, + _hostIdProvider, _queueConfiguration, _blobsConfiguration, _exceptionHandler, _blobWrittenWatcherSetter, _messageEnqueuedWatcherSetter, _sharedContextProvider, _singletonManager, _trace); return binding; diff --git a/src/Microsoft.Azure.WebJobs.Host/Blobs/Triggers/BlobTriggerBinding.cs b/src/Microsoft.Azure.WebJobs.Host/Blobs/Triggers/BlobTriggerBinding.cs index 0636c3ec4..5bb1018a0 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Blobs/Triggers/BlobTriggerBinding.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Blobs/Triggers/BlobTriggerBinding.cs @@ -32,6 +32,7 @@ internal class BlobTriggerBinding : ITriggerBinding private readonly IBlobPathSource _path; private readonly IHostIdProvider _hostIdProvider; private readonly IQueueConfiguration _queueConfiguration; + private readonly JobHostBlobsConfiguration _blobsConfiguration; private readonly IWebJobsExceptionHandler _exceptionHandler; private readonly IContextSetter _blobWrittenWatcherSetter; private readonly IContextSetter _messageEnqueuedWatcherSetter; @@ -48,6 +49,7 @@ public BlobTriggerBinding(ParameterInfo parameter, IBlobPathSource path, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, + JobHostBlobsConfiguration blobsConfiguration, IWebJobsExceptionHandler exceptionHandler, IContextSetter blobWrittenWatcherSetter, IContextSetter messageEnqueuedWatcherSetter, @@ -90,6 +92,11 @@ public BlobTriggerBinding(ParameterInfo parameter, throw new ArgumentNullException("queueConfiguration"); } + if (blobsConfiguration == null) + { + throw new ArgumentNullException("blobsConfiguration"); + } + if (exceptionHandler == null) { throw new ArgumentNullException("exceptionHandler"); @@ -133,6 +140,7 @@ public BlobTriggerBinding(ParameterInfo parameter, _path = path; _hostIdProvider = hostIdProvider; _queueConfiguration = queueConfiguration; + _blobsConfiguration = blobsConfiguration; _exceptionHandler = exceptionHandler; _blobWrittenWatcherSetter = blobWrittenWatcherSetter; _messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter; @@ -231,7 +239,7 @@ public Task CreateListenerAsync(ListenerFactoryContext context) IStorageBlobContainer container = _blobClient.GetContainerReference(_path.ContainerNamePattern); - var factory = new BlobListenerFactory(_hostIdProvider, _queueConfiguration, + var factory = new BlobListenerFactory(_hostIdProvider, _queueConfiguration, _blobsConfiguration, _exceptionHandler, _blobWrittenWatcherSetter, _messageEnqueuedWatcherSetter, _sharedContextProvider, _trace, context.Descriptor.Id, _hostAccount, _dataAccount, container, _path, context.Executor, _singletonManager); diff --git a/src/Microsoft.Azure.WebJobs.Host/Config/ExtensionConfigContext.cs b/src/Microsoft.Azure.WebJobs.Host/Config/ExtensionConfigContext.cs index 8a468ad33..04d56ec20 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Config/ExtensionConfigContext.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Config/ExtensionConfigContext.cs @@ -17,11 +17,6 @@ public class ExtensionConfigContext /// /// Gets or sets the . /// - public TraceWriter Trace { get; set; } - - /// - /// The being configured. - /// - public JobHost Host { get; set; } + public TraceWriter Trace { get; set; } } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Config/ExtensionConfigContextExtensions.cs b/src/Microsoft.Azure.WebJobs.Host/Config/ExtensionConfigContextExtensions.cs index 643e09891..0c49bc643 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Config/ExtensionConfigContextExtensions.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Config/ExtensionConfigContextExtensions.cs @@ -9,6 +9,7 @@ namespace Microsoft.Azure.WebJobs.Host.Config /// /// Helper Extension methods for extension configuration. /// + [Obsolete("Not ready for public consumption.")] public static class ExtensionConfigContextExtensions { /// diff --git a/src/Microsoft.Azure.WebJobs.Host/Converters/IAsyncConverter.cs b/src/Microsoft.Azure.WebJobs.Host/Converters/IAsyncConverter.cs index e49e3bb48..a1ad7a9f1 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Converters/IAsyncConverter.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Converters/IAsyncConverter.cs @@ -4,14 +4,14 @@ using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Azure.WebJobs.Host.Converters +namespace Microsoft.Azure.WebJobs { /// /// Interface defining methods for performing asynchronous conversion operations. /// /// The type to convert from. /// The type to convert to. - internal interface IAsyncConverter + public interface IAsyncConverter { /// /// Convert the specified input value to the output type. diff --git a/src/Microsoft.Azure.WebJobs.Host/Converters/IConverter.cs b/src/Microsoft.Azure.WebJobs.Host/Converters/IConverter.cs index 2b895aaf6..af0cc4de4 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Converters/IConverter.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Converters/IConverter.cs @@ -1,14 +1,14 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -namespace Microsoft.Azure.WebJobs.Host.Converters +namespace Microsoft.Azure.WebJobs { /// /// Defines methods for performing value conversions /// /// The input value type. /// The output value type. - internal interface IConverter + public interface IConverter { /// /// Convert the specified input value. @@ -16,5 +16,5 @@ internal interface IConverter /// The value to convert /// The converted value. TOutput Convert(TInput input); - } + } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Executors/DefaultStorageAccountProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Executors/DefaultStorageAccountProvider.cs index 1b30a2f1a..a6ac9e9f4 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Executors/DefaultStorageAccountProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Executors/DefaultStorageAccountProvider.cs @@ -150,7 +150,7 @@ private IStorageAccount StorageAccount } } - public async Task GetAccountAsync(string connectionStringName, CancellationToken cancellationToken) + public async Task TryGetAccountAsync(string connectionStringName, CancellationToken cancellationToken) { IStorageAccount account = null; var isPrimary = true; diff --git a/src/Microsoft.Azure.WebJobs.Host/Executors/DefaultStorageCredentialsValidator.cs b/src/Microsoft.Azure.WebJobs.Host/Executors/DefaultStorageCredentialsValidator.cs index 5fd25e6c9..31b359f39 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Executors/DefaultStorageCredentialsValidator.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Executors/DefaultStorageCredentialsValidator.cs @@ -39,7 +39,7 @@ public async Task ValidateCredentialsAsync(IStorageAccount account, Cancellation } // Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes - private async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken) + private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken) { // Verify the credentials are correct. // Have to actually ping a storage operation. diff --git a/src/Microsoft.Azure.WebJobs.Host/Executors/FunctionExecutor.cs b/src/Microsoft.Azure.WebJobs.Host/Executors/FunctionExecutor.cs index fef377c6c..b0d84baed 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Executors/FunctionExecutor.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Executors/FunctionExecutor.cs @@ -303,8 +303,8 @@ private async Task ExecuteWithLoggingAsync(IFunctionInstance instance, F if (exceptionInfo != null) { // release any held singleton lock immediately - SingletonLock singleton = null; - if (TryGetSingletonLock(parameters, out singleton) && singleton.IsHeld) + SingletonLock singleton = await GetSingletonLockAsync(parameters); + if (singleton != null && singleton.IsHeld) { await singleton.ReleaseAsync(cancellationToken); } @@ -515,9 +515,11 @@ internal static async Task ExecuteWithWatchersAsync(IFunctionInstance instance, { IFunctionInvoker invoker = instance.Invoker; IReadOnlyList parameterNames = invoker.ParameterNames; - IDelayedException delayedBindingException; - object[] invokeParameters = PrepareParameters(parameterNames, parameters, out delayedBindingException); + Tuple preparedParameters = await PrepareParametersAsync(parameterNames, parameters); + + object[] invokeParameters = preparedParameters.Item1; + IDelayedException delayedBindingException = preparedParameters.Item2; if (delayedBindingException != null) { @@ -527,8 +529,8 @@ internal static async Task ExecuteWithWatchersAsync(IFunctionInstance instance, } // if the function is a Singleton, aquire the lock - SingletonLock singleton = null; - if (TryGetSingletonLock(parameters, out singleton)) + SingletonLock singleton = await GetSingletonLockAsync(parameters); + if (singleton != null) { await singleton.AcquireAsync(functionCancellationTokenSource.Token); } @@ -661,21 +663,20 @@ private static async Task TryHandleTimeoutAsync(Task invokeTask, Cancellat return false; } - private static bool TryGetSingletonLock(IReadOnlyDictionary parameters, out SingletonLock singleton) + private static async Task GetSingletonLockAsync(IReadOnlyDictionary parameters) { IValueProvider singletonValueProvider = null; - singleton = null; + SingletonLock singleton = null; if (parameters.TryGetValue(SingletonValueProvider.SingletonParameterName, out singletonValueProvider)) { - singleton = (SingletonLock)singletonValueProvider.GetValue(); - return true; + singleton = (SingletonLock)(await singletonValueProvider.GetValueAsync()); } - return false; + return singleton; } - private static object[] PrepareParameters(IReadOnlyList parameterNames, - IReadOnlyDictionary parameters, out IDelayedException delayedBindingException) + private static async Task> PrepareParametersAsync(IReadOnlyList parameterNames, + IReadOnlyDictionary parameters) { object[] reflectionParameters = new object[parameterNames.Count]; List bindingExceptions = new List(); @@ -692,23 +693,20 @@ private static object[] PrepareParameters(IReadOnlyList parameterNames, bindingExceptions.Add(exceptionProvider.Exception); } - reflectionParameters[index] = parameters[name].GetValue(); + reflectionParameters[index] = await parameters[name].GetValueAsync(); } - if (bindingExceptions.Count == 0) - { - delayedBindingException = null; - } - else if (bindingExceptions.Count == 1) + IDelayedException delayedBindingException = null; + if (bindingExceptions.Count == 1) { delayedBindingException = new DelayedException(bindingExceptions[0]); } - else + else if (bindingExceptions.Count > 1) { delayedBindingException = new DelayedException(new AggregateException(bindingExceptions)); } - return reflectionParameters; + return new Tuple(reflectionParameters, delayedBindingException); } private FunctionStartedMessage CreateStartedMessageWithoutArguments(IFunctionInstance instance) diff --git a/src/Microsoft.Azure.WebJobs.Host/Executors/IStorageAccountProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Executors/IStorageAccountProvider.cs index 332ba733d..4538681e3 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Executors/IStorageAccountProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Executors/IStorageAccountProvider.cs @@ -9,6 +9,6 @@ namespace Microsoft.Azure.WebJobs.Host.Executors { internal interface IStorageAccountProvider { - Task GetAccountAsync(string connectionStringName, CancellationToken cancellationToken); + Task TryGetAccountAsync(string connectionStringName, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Executors/JobHostContextFactory.cs b/src/Microsoft.Azure.WebJobs.Host/Executors/JobHostContextFactory.cs index 95f41edaa..0a4fba5a6 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Executors/JobHostContextFactory.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Executors/JobHostContextFactory.cs @@ -51,7 +51,7 @@ public async Task CreateAndLogHostStartedAsync(JobHost host, Can var fastLogger = _config.GetService>(); IWebJobsExceptionHandler exceptionHandler = _config.GetService(); - return await CreateAndLogHostStartedAsync(host, _storageAccountProvider, _config.Queues, _config.TypeLocator, _config.JobActivator, + return await CreateAndLogHostStartedAsync(host, _storageAccountProvider, _config.Queues, _config.Blobs, _config.TypeLocator, _config.JobActivator, _config.NameResolver, _consoleProvider, _config, shutdownToken, cancellationToken, exceptionHandler, hostIdProvider, fastLogger: fastLogger); } @@ -59,6 +59,7 @@ public static async Task CreateAndLogHostStartedAsync( JobHost host, IStorageAccountProvider storageAccountProvider, IQueueConfiguration queueConfiguration, + JobHostBlobsConfiguration blobsConfiguration, ITypeLocator typeLocator, IJobActivator activator, INameResolver nameResolver, @@ -82,6 +83,10 @@ public static async Task CreateAndLogHostStartedAsync( hostIdProvider = new DynamicHostIdProvider(storageAccountProvider, () => functionIndexProvider); } + AzureStorageDeploymentValidator.Validate(); + + var converterManager = config.ConverterManager; + IExtensionTypeLocator extensionTypeLocator = new ExtensionTypeLocator(typeLocator); ContextAccessor messageEnqueuedWatcherAccessor = new ContextAccessor(); @@ -103,8 +108,7 @@ public static async Task CreateAndLogHostStartedAsync( ExtensionConfigContext context = new ExtensionConfigContext { Config = config, - Trace = trace, - Host = host + Trace = trace }; InvokeExtensionConfigProviders(context); @@ -115,12 +119,12 @@ public static async Task CreateAndLogHostStartedAsync( IExtensionRegistry extensions = config.GetExtensions(); ITriggerBindingProvider triggerBindingProvider = DefaultTriggerBindingProvider.Create(nameResolver, - storageAccountProvider, extensionTypeLocator, hostIdProvider, queueConfiguration, exceptionHandler, + storageAccountProvider, extensionTypeLocator, hostIdProvider, queueConfiguration, blobsConfiguration, exceptionHandler, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, sharedContextProvider, extensions, singletonManager, trace); if (bindingProvider == null) { - bindingProvider = DefaultBindingProvider.Create(nameResolver, storageAccountProvider, extensionTypeLocator, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, extensions); + bindingProvider = DefaultBindingProvider.Create(nameResolver, converterManager, storageAccountProvider, extensionTypeLocator, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, extensions); } bool hasFastTableHook = config.GetService>() != null; @@ -177,7 +181,17 @@ public static async Task CreateAndLogHostStartedAsync( config.HostId = hostId; } - if (dashboardAccount != null) + if (dashboardAccount == null) + { + hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, functionExecutor); + + IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory); + IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener); + listener = shutdownListener; + + hostOutputMessage = new DataOnlyHostOutputMessage(); + } + else { string sharedQueueName = HostQueueNames.GetHostQueueName(hostId); IStorageQueueClient dashboardQueueClient = dashboardAccount.CreateQueueClient(); @@ -229,16 +243,6 @@ public static async Task CreateAndLogHostStartedAsync( // Publish this to Azure logging account so that a web dashboard can see it. await LogHostStartedAsync(functions, hostOutputMessage, hostInstanceLogger, combinedCancellationToken); } - else - { - hostCallExecutor = new ShutdownFunctionExecutor(shutdownToken, functionExecutor); - - IListener factoryListener = new ListenerFactoryListener(functionsListenerFactory); - IListener shutdownListener = new ShutdownListener(shutdownToken, factoryListener); - listener = shutdownListener; - - hostOutputMessage = new DataOnlyHostOutputMessage(); - } functionExecutor.HostOutputMessage = hostOutputMessage; diff --git a/src/Microsoft.Azure.WebJobs.Host/Executors/StorageAccountParser.cs b/src/Microsoft.Azure.WebJobs.Host/Executors/StorageAccountParser.cs index 955435c74..fa3b39da4 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Executors/StorageAccountParser.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Executors/StorageAccountParser.cs @@ -74,7 +74,7 @@ public static string FormatParseAccountErrorMessage(StorageAccountParseResult er { case StorageAccountParseResult.MissingOrEmptyConnectionStringError: return String.Format(CultureInfo.CurrentCulture, - "Microsoft Azure WebJobs SDK {0} connection string is missing or empty. " + + "Microsoft Azure WebJobs SDK '{0}' connection string is missing or empty. " + "The Microsoft Azure Storage account connection string can be set in the following ways:" + Environment.NewLine + "1. Set the connection string named '{1}' in the connectionStrings section of the .config file in the following format " + ", or" + Environment.NewLine + @@ -87,7 +87,7 @@ public static string FormatParseAccountErrorMessage(StorageAccountParseResult er return String.Format(CultureInfo.CurrentCulture, "Failed to validate Microsoft Azure WebJobs SDK {0} connection string. " + "The Microsoft Azure Storage account connection string is not formatted " + - "correctly. Please visit http://msdn.microsoft.com/en-us/library/windowsazure/ee758697.aspx for " + + "correctly. Please visit https://go.microsoft.com/fwlink/?linkid=841340 for " + "details about configuring Microsoft Azure Storage connection strings.", connectionStringName); } diff --git a/src/Microsoft.Azure.WebJobs.Host/Executors/StorageAccountProviderExtensions.cs b/src/Microsoft.Azure.WebJobs.Host/Executors/StorageAccountProviderExtensions.cs index 3c5394da3..40317442c 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Executors/StorageAccountProviderExtensions.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Executors/StorageAccountProviderExtensions.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. - using System; +using System.Globalization; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -19,20 +19,33 @@ public static Task GetDashboardAccountAsync(this IStorageAccoun throw new ArgumentNullException("provider"); } - return provider.GetAccountAsync(ConnectionStringNames.Dashboard, cancellationToken); + return provider.TryGetAccountAsync(ConnectionStringNames.Dashboard, cancellationToken); } - public static Task GetStorageAccountAsync(this IStorageAccountProvider provider, CancellationToken cancellationToken) + public static async Task GetStorageAccountAsync(this IStorageAccountProvider provider, CancellationToken cancellationToken) { if (provider == null) { throw new ArgumentNullException("provider"); } - return provider.GetAccountAsync(ConnectionStringNames.Storage, cancellationToken); + IStorageAccount account = await provider.TryGetAccountAsync(ConnectionStringNames.Storage, cancellationToken); + ValidateStorageAccount(account, ConnectionStringNames.Storage); + return account; } - public static Task GetStorageAccountAsync(this IStorageAccountProvider provider, ParameterInfo parameter, CancellationToken cancellationToken, INameResolver nameResolver = null) + public static async Task GetAccountAsync(this IStorageAccountProvider provider, string connectionStringName, CancellationToken cancellationToken) + { + if (provider == null) + { + throw new ArgumentNullException("provider"); + } + + IStorageAccount account = await provider.TryGetAccountAsync(connectionStringName, cancellationToken); + ValidateStorageAccount(account, connectionStringName); + return account; + } + public static async Task GetStorageAccountAsync(this IStorageAccountProvider provider, ParameterInfo parameter, CancellationToken cancellationToken, INameResolver nameResolver = null) { if (provider == null) { @@ -54,7 +67,18 @@ public static Task GetStorageAccountAsync(this IStorageAccountP } } - return provider.GetAccountAsync(connectionStringName, cancellationToken); + IStorageAccount account = await provider.TryGetAccountAsync(connectionStringName, cancellationToken); + ValidateStorageAccount(account, connectionStringName); + return account; + } + + private static void ValidateStorageAccount(IStorageAccount account, string connectionStringName) + { + if (account == null) + { + string message = StorageAccountParser.FormatParseAccountErrorMessage(StorageAccountParseResult.MissingOrEmptyConnectionStringError, connectionStringName); + throw new InvalidOperationException(message); + } } /// @@ -68,7 +92,6 @@ internal static string GetAccountOverrideOrNull(ParameterInfo parameter) { return attribute.Account; } - return null; } } diff --git a/src/Microsoft.Azure.WebJobs.Host/FuncConverter.cs b/src/Microsoft.Azure.WebJobs.Host/FuncConverter.cs index 57673c782..71db44326 100644 --- a/src/Microsoft.Azure.WebJobs.Host/FuncConverter.cs +++ b/src/Microsoft.Azure.WebJobs.Host/FuncConverter.cs @@ -15,6 +15,7 @@ namespace Microsoft.Azure.WebJobs /// source /// attribute /// binding context that may have additional parameters to influence conversion. + [Obsolete("Not ready for public consumption.")] public delegate TDestination FuncConverter(TSource src, TAttribute attribute, ValueBindingContext context) where TAttribute : Attribute; } \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Host/GlobalSuppressions.cs b/src/Microsoft.Azure.WebJobs.Host/GlobalSuppressions.cs index e54a4ab4f..2140e7e34 100644 --- a/src/Microsoft.Azure.WebJobs.Host/GlobalSuppressions.cs +++ b/src/Microsoft.Azure.WebJobs.Host/GlobalSuppressions.cs @@ -21,4 +21,32 @@ [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Usage", "CA1801:ReviewUnusedParameters", MessageId = "converter", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.GenericAsyncCollectorBindingProvider`1+IdentityConverterManager.#AddConverter`3(System.Func`3)")] [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1062:Validate arguments of public methods", MessageId = "0", Scope = "member", Target = "Microsoft.Azure.WebJobs.IConverterManagerExtensions.#AddConverter`2(Microsoft.Azure.WebJobs.IConverterManager,System.Func`2)")] [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1062:Validate arguments of public methods", MessageId = "0", Scope = "member", Target = "Microsoft.Azure.WebJobs.IConverterManagerExtensions.#AddConverter`3(Microsoft.Azure.WebJobs.IConverterManager,System.Func`3)")] -[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1005:AvoidExcessiveParametersOnGenericTypes", Scope = "type", Target = "Microsoft.Azure.WebJobs.FuncConverter`3")] \ No newline at end of file +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1005:AvoidExcessiveParametersOnGenericTypes", Scope = "type", Target = "Microsoft.Azure.WebJobs.FuncConverter`3")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA2204:Literals should be spelled correctly", MessageId = "WindowsAzure", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.AzureStorageDeploymentValidator.#Validate()")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.IConverterManager.#AddConverter2`3(System.Func`2>)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1800:DoNotCastUnnecessarily", Scope = "member", Target = "Microsoft.Azure.WebJobs.ConverterManager.#TryGetConverter`3()")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1800:DoNotCastUnnecessarily", Scope = "member", Target = "Microsoft.Azure.WebJobs.ConverterManager.#TryGetConverter`3(System.Type)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Tables.TableAttributeBindingProvider.#BuildIQueryable`1(Microsoft.Azure.WebJobs.Host.Storage.Table.IStorageTable)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.IConverterManager.#AddConverter2`3(System.Func`3>)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Tables.TableAttributeBindingProvider+TableQueryableOpenType.#IsValid(System.Type)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1800:DoNotCastUnnecessarily", Scope = "member", Target = "Microsoft.Azure.WebJobs.ConverterManager.#TryGetConverter`3(System.Type,System.Type)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA2204:Literals should be spelled correctly", MessageId = "IsValid", Scope = "member", Target = "Microsoft.Azure.WebJobs.ConverterManager.#GetChecker`1()")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.IConverterManager.#AddConverterBuilder`3(System.Func`3>)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.IConverterManagerExtensions.#AddConverterBuilder`3(Microsoft.Azure.WebJobs.IConverterManager,System.Type,System.Object[])")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.IConverterManagerExtensions.#AddConverterBuilder`3(Microsoft.Azure.WebJobs.IConverterManager,System.Object)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Tables.TableAttributeBindingProvider+Table2IQueryableConverter`1.#Convert(Microsoft.Azure.WebJobs.Host.Storage.Table.IStorageTable)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Tables.TableAttributeBindingProvider+Object2ITableEntityConverter`1.#Convert(!0)")][assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA2204:Literals should be spelled correctly", MessageId = "WindowsAzure", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.AzureStorageDeploymentValidator.#Validate()")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1062:Validate arguments of public methods", MessageId = "1", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.AsyncCollectorBinding`2.#BuildAsync(!0,Microsoft.Azure.WebJobs.Host.Bindings.BindingContext)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1062:Validate arguments of public methods", MessageId = "1", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.ExactTypeBindingProvider`2+ExactBinding.#BuildAsync(!0,Microsoft.Azure.WebJobs.Host.Bindings.BindingContext)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1062:Validate arguments of public methods", MessageId = "1", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.GenericItemBindingProvider`1+Binding.#BuildAsync(!0,Microsoft.Azure.WebJobs.Host.Bindings.BindingContext)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1062:Validate arguments of public methods", MessageId = "1", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.ItemBindingProvider`1+Binding.#BuildAsync(!0,Microsoft.Azure.WebJobs.Host.Bindings.BindingContext)")][assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Tables.TableAttributeBindingProvider+Object2ITableEntityConverter`1.#Convert(!0)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.BindingFactory.#BindToInput`2(System.Boolean,System.Type,System.Object[])")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.BindingFactory.#BindToInput`2(System.Boolean,System.Object)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.BindingFactory.#BindToInput`2(System.Object)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.BindingFactory.#BindToInput`2(System.Type,System.Object[])")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.IConverterManagerExtensions.#AddConverter`3(Microsoft.Azure.WebJobs.IConverterManager,System.Type,System.Object[])")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.IConverterManagerExtensions.#AddConverter`3(Microsoft.Azure.WebJobs.IConverterManager,System.Object)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.IConverterManager.#AddConverter`3(System.Func`3>)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.IConverterManagerExtensions.#AddConverter`3(Microsoft.Azure.WebJobs.IConverterManager,System.Type,System.Object[])")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.IConverterManagerExtensions.#AddConverter`3(Microsoft.Azure.WebJobs.IConverterManager,System.Object)")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter", Scope = "member", Target = "Microsoft.Azure.WebJobs.Host.Bindings.BindingFactory.#BindToCollector`2(System.Type,System.Object[])")] \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Host/IConverterManager.cs b/src/Microsoft.Azure.WebJobs.Host/IConverterManager.cs index d2b6822bb..7c36de310 100644 --- a/src/Microsoft.Azure.WebJobs.Host/IConverterManager.cs +++ b/src/Microsoft.Azure.WebJobs.Host/IConverterManager.cs @@ -2,6 +2,8 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Reflection; +using Microsoft.Azure.WebJobs.Host.Bindings; namespace Microsoft.Azure.WebJobs { @@ -9,6 +11,7 @@ namespace Microsoft.Azure.WebJobs /// General service for converting between types for parameter bindings. /// Parameter bindings call this to convert from user parameter types to underlying binding types. /// + [Obsolete("Not ready for public consumption.")] public interface IConverterManager { /// @@ -36,13 +39,30 @@ FuncConverter GetConverterthe converter function for this combination of type parameters. void AddConverter(FuncConverter converter) where TAttribute : Attribute; + + /// + /// Add a builder function that returns a converter. This can use to match against an + /// open set of types. The builder can then do one time static type checking and code gen caching before + /// returning a converter function that is called on each invocation. + /// + /// Source type. + /// Destination type. + /// Attribute on the binding. + /// A function that is invoked if-and-only-if there is a compatible type match for the + /// source and destination types. It then produce a converter function that can be called many times + void AddConverter( + Func> converterBuilder) + where TAttribute : Attribute; } /// /// Convenience methods for /// + [Obsolete("Not ready for public consumption.")] public static class IConverterManagerExtensions { + private static readonly MethodInfo ConverterMethod = typeof(IConverterManagerExtensions).GetMethod("HasConverterWorker", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static); + /// /// Register a new converter function that applies for all attributes. /// If TSource is object, then this converter is applied to any attempt to convert to TDestination. @@ -72,5 +92,76 @@ public static void AddConverter(this IConvert FuncConverter func = (src, attr, context) => converter(src, attr); converterManager.AddConverter(func); } + + /// + /// Add a converter for the given Source to Destination conversion. + /// The typeConverter type is instantiated with the type arguments and constructorArgs is passed. + /// + /// Source type. + /// Destination type. + /// Attribute on the binding. + /// Instance of Converter Manager. + /// A type with conversion methods. This can be generic and will get instantiated with the + /// appropriate type parameters. + /// Constructor Arguments to pass to the constructor when instantiated. This can pass configuration and state. + public static void AddConverter( + this IConverterManager converterManager, + Type typeConverter, + params object[] constructorArgs) + where TAttribute : Attribute + { + var patternMatcher = PatternMatcher.New(typeConverter, constructorArgs); + converterManager.AddConverterBuilder(patternMatcher); + } + + /// + /// Add a converter for the given Source to Destination conversion. + /// + /// Source type. + /// Destination type. + /// Attribute on the binding. + /// Instance of Converter Manager. + /// Instance of an object with convert methods on it. + public static void AddConverter( + this IConverterManager converterManager, + object converterInstance) + where TAttribute : Attribute + { + var patternMatcher = PatternMatcher.New(converterInstance); + converterManager.AddConverterBuilder(patternMatcher); + } + + private static void AddConverterBuilder( + this IConverterManager converterManager, + PatternMatcher patternMatcher) + where TAttribute : Attribute + { + if (converterManager == null) + { + throw new ArgumentNullException("converterManager"); + } + + converterManager.AddConverter((typeSource, typeDest) => + { + var converter = patternMatcher.TryGetConverterFunc(typeSource, typeDest); + return converter; + }); + } + + private static bool HasConverterWorker(IConverterManager converterManager) + where TAttribute : Attribute + { + var func = converterManager.GetConverter(); + return func != null; + } + + // Provide late-bound access to check if a conversion exists. + internal static bool HasConverter(this IConverterManager converterManager, Type typeSource, Type typeDest) + where TAttribute : Attribute + { + var method = ConverterMethod.MakeGenericMethod(typeof(TAttribute), typeSource, typeDest); + var result = method.Invoke(null, new object[] { converterManager }); + return (bool)result; + } } } \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Host/Indexers/DefaultBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Indexers/DefaultBindingProvider.cs index 29b345411..a671fc2d8 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Indexers/DefaultBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Indexers/DefaultBindingProvider.cs @@ -18,7 +18,9 @@ namespace Microsoft.Azure.WebJobs.Host.Indexers { internal static class DefaultBindingProvider { - public static IBindingProvider Create(INameResolver nameResolver, + public static IBindingProvider Create( + INameResolver nameResolver, + IConverterManager converterManager, IStorageAccountProvider storageAccountProvider, IExtensionTypeLocator extensionTypeLocator, IContextGetter messageEnqueuedWatcherGetter, @@ -27,8 +29,12 @@ public static IBindingProvider Create(INameResolver nameResolver, { List innerProviders = new List(); + if (converterManager == null) + { + converterManager = new ConverterManager(); + } + // Wire up new bindings - IConverterManager converterManager = new ConverterManager(); var ruleQueueOutput = QueueBindingProvider.Build(storageAccountProvider, messageEnqueuedWatcherGetter, nameResolver, converterManager); innerProviders.Add(ruleQueueOutput); diff --git a/src/Microsoft.Azure.WebJobs.Host/Indexers/DefaultTriggerBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Indexers/DefaultTriggerBindingProvider.cs index 83b0db79a..d386b0f71 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Indexers/DefaultTriggerBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Indexers/DefaultTriggerBindingProvider.cs @@ -20,6 +20,7 @@ public static ITriggerBindingProvider Create(INameResolver nameResolver, IExtensionTypeLocator extensionTypeLocator, IHostIdProvider hostIdProvider, IQueueConfiguration queueConfiguration, + JobHostBlobsConfiguration blobsConfiguration, IWebJobsExceptionHandler exceptionHandler, IContextSetter messageEnqueuedWatcherSetter, IContextSetter blobWrittenWatcherSetter, @@ -33,7 +34,7 @@ public static ITriggerBindingProvider Create(INameResolver nameResolver, queueConfiguration, exceptionHandler, messageEnqueuedWatcherSetter, sharedContextProvider, trace)); innerProviders.Add(new BlobTriggerAttributeBindingProvider(nameResolver, storageAccountProvider, - extensionTypeLocator, hostIdProvider, queueConfiguration, exceptionHandler, + extensionTypeLocator, hostIdProvider, queueConfiguration, blobsConfiguration, exceptionHandler, blobWrittenWatcherSetter, messageEnqueuedWatcherSetter, sharedContextProvider, singletonManager, trace)); // add any registered extension binding providers diff --git a/src/Microsoft.Azure.WebJobs.Host/Indexers/FunctionIndexer.cs b/src/Microsoft.Azure.WebJobs.Host/Indexers/FunctionIndexer.cs index 2507d6396..ec2e58d92 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Indexers/FunctionIndexer.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Indexers/FunctionIndexer.cs @@ -254,6 +254,11 @@ internal async Task IndexMethodAsyncCore(MethodInfo method, IFunctionIndexCollec } } + if (TypeUtility.IsAsyncVoid(method)) + { + this._trace.Warning($"Function '{method.Name}' is async but does not return a Task. Your function may not run correctly."); + } + Type returnType = method.ReturnType; if (returnType != typeof(void) && returnType != typeof(Task)) diff --git a/src/Microsoft.Azure.WebJobs.Host/JobHost.cs b/src/Microsoft.Azure.WebJobs.Host/JobHost.cs index 9c9ffa710..a9f256f43 100644 --- a/src/Microsoft.Azure.WebJobs.Host/JobHost.cs +++ b/src/Microsoft.Azure.WebJobs.Host/JobHost.cs @@ -105,7 +105,7 @@ internal IListener Listener /// Starts the host. public void Start() { - StartAsync().GetAwaiter().GetResult(); + Task.Run(() => StartAsync()).GetAwaiter().GetResult(); } /// Starts the host. @@ -136,7 +136,7 @@ private async Task StartAsyncCore(CancellationToken cancellationToken) /// Stops the host. public void Stop() { - StopAsync().GetAwaiter().GetResult(); + Task.Run(() => StopAsync()).GetAwaiter().GetResult(); } /// Stops the host. @@ -195,7 +195,7 @@ public void RunAndBlock() /// The job method to call. public void Call(MethodInfo method) { - CallAsync(method).GetAwaiter().GetResult(); + Task.Run(() => CallAsync(method)).GetAwaiter().GetResult(); } /// Calls a job method. @@ -206,7 +206,7 @@ public void Call(MethodInfo method) /// public void Call(MethodInfo method, object arguments) { - CallAsync(method, arguments).GetAwaiter().GetResult(); + Task.Run(() => CallAsync(method, arguments)).GetAwaiter().GetResult(); } /// Calls a job method. @@ -215,7 +215,7 @@ public void Call(MethodInfo method, object arguments) /// In addition to parameter values, these may also include binding data values. public void Call(MethodInfo method, IDictionary arguments) { - CallAsync(method, arguments).GetAwaiter().GetResult(); + Task.Run(() => CallAsync(method, arguments)).GetAwaiter().GetResult(); } /// Calls a job method. diff --git a/src/Microsoft.Azure.WebJobs.Host/JobHostBlobsConfiguration.cs b/src/Microsoft.Azure.WebJobs.Host/JobHostBlobsConfiguration.cs new file mode 100644 index 000000000..1c4019916 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/JobHostBlobsConfiguration.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +namespace Microsoft.Azure.WebJobs.Host +{ + /// + /// Represents configuration for . + /// + public class JobHostBlobsConfiguration + { + /// + /// Constructs a new instance. + /// + public JobHostBlobsConfiguration() + { + CentralizedPoisonQueue = false; + } + + /// + /// Gets or sets a value indicating whether a single centralized + /// poison queue for poison blobs should be used (in the primary + /// storage account) or whether the poison queue for a blob triggered + /// function should be co-located with the target blob container. + /// This comes into play only when using multiple storage accounts via + /// . The default is false. + /// + public bool CentralizedPoisonQueue { get; set; } + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/JobHostConfiguration.cs b/src/Microsoft.Azure.WebJobs.Host/JobHostConfiguration.cs index b6186d5d3..420c14411 100644 --- a/src/Microsoft.Azure.WebJobs.Host/JobHostConfiguration.cs +++ b/src/Microsoft.Azure.WebJobs.Host/JobHostConfiguration.cs @@ -22,6 +22,7 @@ public sealed class JobHostConfiguration : IServiceProvider private readonly DefaultStorageAccountProvider _storageAccountProvider; private readonly JobHostQueuesConfiguration _queueConfiguration = new JobHostQueuesConfiguration(); + private readonly JobHostBlobsConfiguration _blobsConfiguration = new JobHostBlobsConfiguration(); private readonly JobHostTraceConfiguration _traceConfiguration = new JobHostTraceConfiguration(); private readonly ConcurrentDictionary _services = new ConcurrentDictionary(); private IJobHostContextFactory _contextFactory; @@ -195,6 +196,18 @@ public INameResolver NameResolver } } + /// + /// Get the converter manager, which can be used to register additional conversions for + /// customizing model binding. + /// + public IConverterManager ConverterManager + { + get + { + return GetService(); + } + } + /// /// Gets a helper object for constructing common binding rules for extensions. /// @@ -215,6 +228,14 @@ public JobHostQueuesConfiguration Queues get { return _queueConfiguration; } } + /// + /// Gets the configuration used by . + /// + public JobHostBlobsConfiguration Blobs + { + get { return _blobsConfiguration; } + } + /// /// Gets the configuration used by . /// diff --git a/src/Microsoft.Azure.WebJobs.Host/JobHostQueuesConfiguration.cs b/src/Microsoft.Azure.WebJobs.Host/JobHostQueuesConfiguration.cs index b402fb6f2..e5acac9c9 100644 --- a/src/Microsoft.Azure.WebJobs.Host/JobHostQueuesConfiguration.cs +++ b/src/Microsoft.Azure.WebJobs.Host/JobHostQueuesConfiguration.cs @@ -8,7 +8,9 @@ namespace Microsoft.Azure.WebJobs.Host { - /// Represents configuration for . + /// + /// Represents configuration for . + /// public sealed class JobHostQueuesConfiguration : IQueueConfiguration { private const int DefaultMaxDequeueCount = 5; @@ -21,6 +23,7 @@ public sealed class JobHostQueuesConfiguration : IQueueConfiguration private int _batchSize = DefaultBatchSize; private int _newBatchThreshold; private TimeSpan _maxPollingInterval = QueuePollingIntervals.DefaultMaximum; + private TimeSpan _visibilityTimeout = TimeSpan.Zero; private int _maxDequeueCount = DefaultMaxDequeueCount; /// @@ -126,6 +129,28 @@ public int MaxDequeueCount } } + /// + /// Gets or sets the default message visibility timeout that will be used + /// for messages that fail processing. The default is TimeSpan.Zero. To increase + /// the time delay between retries, increase this value. + /// + /// + /// When message processing fails, the message will remain in the queue and + /// its visibility will be updated with this value. The message will then be + /// available for reprocessing after this timeout expires. + /// + public TimeSpan VisibilityTimeout + { + get + { + return _visibilityTimeout; + } + set + { + _visibilityTimeout = value; + } + } + /// /// Gets or sets the that will be used to create /// instances that will be used to process messages. diff --git a/src/Microsoft.Azure.WebJobs.Host/Listeners/HostListenerFactory.cs b/src/Microsoft.Azure.WebJobs.Host/Listeners/HostListenerFactory.cs index 1a64440eb..518976cff 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Listeners/HostListenerFactory.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Listeners/HostListenerFactory.cs @@ -58,7 +58,7 @@ public async Task CreateAsync(CancellationToken cancellationToken) SingletonAttribute singletonAttribute = SingletonManager.GetListenerSingletonOrNull(listener.GetType(), method); if (singletonAttribute != null) { - listener = new SingletonListener(method, singletonAttribute, _singletonManager, listener); + listener = new SingletonListener(method, singletonAttribute, _singletonManager, listener, _trace); } listeners.Add(listener); diff --git a/src/Microsoft.Azure.WebJobs.Host/Loggers/FunctionInstanceLogEntry.cs b/src/Microsoft.Azure.WebJobs.Host/Loggers/FunctionInstanceLogEntry.cs index 75851a005..7236d904a 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Loggers/FunctionInstanceLogEntry.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Loggers/FunctionInstanceLogEntry.cs @@ -13,6 +13,7 @@ namespace Microsoft.Azure.WebJobs.Host.Loggers /// Represent a function invocation starting or finishing. /// A host can register an IAsyncCollector on the JobHostConfiguration to receive these notifications to do their own logging. /// + [Obsolete("Not ready for public consumption.")] public class FunctionInstanceLogEntry { /// diff --git a/src/Microsoft.Azure.WebJobs.Host/Loggers/TraceWriterFunctionInstanceLogger.cs b/src/Microsoft.Azure.WebJobs.Host/Loggers/TraceWriterFunctionInstanceLogger.cs index 111104939..739464769 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Loggers/TraceWriterFunctionInstanceLogger.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Loggers/TraceWriterFunctionInstanceLogger.cs @@ -25,7 +25,7 @@ public TraceWriterFunctionInstanceLogger(TraceWriter trace) public Task LogFunctionStartedAsync(FunctionStartedMessage message, CancellationToken cancellationToken) { - string traceMessage = string.Format(CultureInfo.InvariantCulture, "Executing: '{0}' - Reason: '{1}'", message.Function.ShortName, message.FormatReason()); + string traceMessage = string.Format(CultureInfo.InvariantCulture, "Executing '{0}' (Reason='{1}', Id={2})", message.Function.ShortName, message.FormatReason(), message.FunctionInstanceId); Trace(TraceLevel.Info, message.HostInstanceId, message.Function, message.FunctionInstanceId, traceMessage, TraceSource.Execution); return Task.FromResult(null); } @@ -34,12 +34,12 @@ public Task LogFunctionCompletedAsync(FunctionCompletedMessage message, Cancella { if (message.Succeeded) { - string traceMessage = string.Format(CultureInfo.InvariantCulture, "Executed: '{0}' (Succeeded)", message.Function.ShortName); + string traceMessage = string.Format(CultureInfo.InvariantCulture, "Executed '{0}' (Succeeded, Id={1})", message.Function.ShortName, message.FunctionInstanceId); Trace(TraceLevel.Info, message.HostInstanceId, message.Function, message.FunctionInstanceId, traceMessage, TraceSource.Execution); } else { - string traceMessage = string.Format(CultureInfo.InvariantCulture, "Executed: '{0}' (Failed)", message.Function.ShortName); + string traceMessage = string.Format(CultureInfo.InvariantCulture, "Executed '{0}' (Failed, Id={1})", message.Function.ShortName, message.FunctionInstanceId); Trace(TraceLevel.Error, message.HostInstanceId, message.Function, message.FunctionInstanceId, traceMessage, TraceSource.Execution, message.Failure.Exception); // Also log the eror message using TraceSource.Host, to ensure diff --git a/src/Microsoft.Azure.WebJobs.Host/Queues/Bindings/QueueBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Queues/Bindings/QueueBindingProvider.cs index 1d1b3dc68..580224ca7 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Queues/Bindings/QueueBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Queues/Bindings/QueueBindingProvider.cs @@ -63,10 +63,15 @@ private IBindingProvider New( var bindingFactory = new BindingFactory(nameResolver, converterManager); - var bindAsyncCollector = bindingFactory.BindToAsyncCollector(BuildFromQueueAttribute, ToWriteParameterDescriptorForCollector, CollectAttributeInfo); - var bindClient = bindingFactory.BindToExactAsyncType(BuildClientFromQueueAttributeAsync, ToReadWriteParameterDescriptorForCollector, CollectAttributeInfo); - var bindSdkClient = bindingFactory.BindToExactAsyncType(BuildRealClientFromQueueAttributeAsync, ToReadWriteParameterDescriptorForCollector, CollectAttributeInfo); - + var bindAsyncCollector = bindingFactory.BindToCollector(BuildFromQueueAttribute) + .SetPostResolveHook(ToWriteParameterDescriptorForCollector, CollectAttributeInfo); + + var bindClient = bindingFactory.BindToInput(typeof(QueueBuilder)) + .SetPostResolveHook(ToReadWriteParameterDescriptorForCollector, CollectAttributeInfo); + + var bindSdkClient = bindingFactory.BindToInput(typeof(QueueBuilder)) + .SetPostResolveHook(ToReadWriteParameterDescriptorForCollector, CollectAttributeInfo); + var bindingProvider = new GenericCompositeBindingProvider( ValidateQueueAttribute, nameResolver, bindClient, bindSdkClient, bindAsyncCollector); @@ -120,11 +125,6 @@ private ParameterDescriptor ToParameterDescriptorForCollector(QueueAttribute att _accountProvider.GetStorageAccountAsync(parameter, CancellationToken.None, nameResolver)); IStorageAccount account = t.GetAwaiter().GetResult(); - if (account == null) - { - throw new InvalidOperationException("Unable to bind Queue because no storage account has been configured."); - } - string accountName = account.Credentials.AccountName; return new QueueParameterDescriptor @@ -189,19 +189,6 @@ private IStorageQueueMessage ConvertStringToCloudQueueMessage(string arg, QueueA return msg; } - private async Task BuildRealClientFromQueueAttributeAsync(QueueAttribute attrResolved) - { - var queue = await this.BuildClientFromQueueAttributeAsync(attrResolved); - return queue.SdkObject; - } - - private async Task BuildClientFromQueueAttributeAsync(QueueAttribute attrResolved) - { - IStorageQueue queue = GetQueue(attrResolved); - await queue.CreateIfNotExistsAsync(CancellationToken.None); - return queue; - } - private IAsyncCollector BuildFromQueueAttribute(QueueAttribute attrResolved) { IStorageQueue queue = GetQueue(attrResolved); @@ -215,6 +202,29 @@ private static IStorageQueue GetQueue(QueueAttribute attrResolved) return queue; } + private class QueueBuilder : + IAsyncConverter, + IAsyncConverter + { + async Task IAsyncConverter.ConvertAsync( + QueueAttribute attrResolved, + CancellationToken cancellation) + { + IStorageQueue queue = GetQueue(attrResolved); + await queue.CreateIfNotExistsAsync(CancellationToken.None); + return queue; + } + + async Task IAsyncConverter.ConvertAsync( + QueueAttribute attrResolved, + CancellationToken cancellation) + { + IAsyncConverter convert = this; + var queue = await convert.ConvertAsync(attrResolved, cancellation); + return queue.SdkObject; + } + } + // Queue attributes can optionally be paired with a separate [StorageAccount]. // Consolidate the information from both attributes into a single attribute. // New extensions should just place everything in the attribute or the configuration and so shouldn't need to do this. diff --git a/src/Microsoft.Azure.WebJobs.Host/Queues/IQueueConfiguration.cs b/src/Microsoft.Azure.WebJobs.Host/Queues/IQueueConfiguration.cs index 9d3799d9d..8f92c5ff0 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Queues/IQueueConfiguration.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Queues/IQueueConfiguration.cs @@ -15,6 +15,8 @@ internal interface IQueueConfiguration int MaxDequeueCount { get; } + TimeSpan VisibilityTimeout { get; } + IQueueProcessorFactory QueueProcessorFactory { get; } } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/HostMessageListenerFactory.cs b/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/HostMessageListenerFactory.cs index fb9245228..83fd4e616 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/HostMessageListenerFactory.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/HostMessageListenerFactory.cs @@ -15,9 +15,6 @@ namespace Microsoft.Azure.WebJobs.Host.Queues.Listeners { internal class HostMessageListenerFactory : IListenerFactory { - private static readonly TimeSpan Minimum = QueuePollingIntervals.Minimum; - private static readonly TimeSpan DefaultMaximum = QueuePollingIntervals.DefaultMaximum; - private readonly IStorageQueue _queue; private readonly IQueueConfiguration _queueConfiguration; private readonly IWebJobsExceptionHandler _exceptionHandler; @@ -83,20 +80,18 @@ public Task CreateAsync(CancellationToken cancellationToken) { ITriggerExecutor triggerExecutor = new HostMessageExecutor(_executor, _functionLookup, _functionInstanceLogger); - TimeSpan configuredMaximum = _queueConfiguration.MaxPollingInterval; // Provide an upper bound on the maximum polling interval for run/abort from dashboard. - // Use the default maximum for host polling (1 minute) unless the configured overall maximum is even faster. - TimeSpan maximum = configuredMaximum < DefaultMaximum ? configuredMaximum : DefaultMaximum; - IDelayStrategy delayStrategy = new RandomizedExponentialBackoffStrategy(Minimum, maximum); + // This ensures that if users have customized this value the Dashboard will remain responsive. + TimeSpan maxPollingInterval = QueuePollingIntervals.DefaultMaximum; IListener listener = new QueueListener(_queue, poisonQueue: null, triggerExecutor: triggerExecutor, - delayStrategy: delayStrategy, exceptionHandler: _exceptionHandler, trace: _trace, sharedWatcher: null, - queueConfiguration: _queueConfiguration); + queueConfiguration: _queueConfiguration, + maxPollingInterval: maxPollingInterval); return Task.FromResult(listener); } diff --git a/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/QueueListener.cs b/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/QueueListener.cs index f5625c1fd..1990fe464 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/QueueListener.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/QueueListener.cs @@ -30,6 +30,7 @@ internal sealed class QueueListener : IListener, ITaskSeriesCommand, INotificati private readonly object _stopWaitingTaskSourceLock = new object(); private readonly IQueueConfiguration _queueConfiguration; private readonly QueueProcessor _queueProcessor; + private readonly TimeSpan _visibilityTimeout; private bool _foundMessageSinceLastDelay; private bool _disposed; @@ -38,11 +39,12 @@ internal sealed class QueueListener : IListener, ITaskSeriesCommand, INotificati public QueueListener(IStorageQueue queue, IStorageQueue poisonQueue, ITriggerExecutor triggerExecutor, - IDelayStrategy delayStrategy, IWebJobsExceptionHandler exceptionHandler, TraceWriter trace, SharedQueueWatcher sharedWatcher, - IQueueConfiguration queueConfiguration) + IQueueConfiguration queueConfiguration, + QueueProcessor queueProcessor = null, + TimeSpan? maxPollingInterval = null) { if (trace == null) { @@ -68,11 +70,14 @@ public QueueListener(IStorageQueue queue, _queue = queue; _poisonQueue = poisonQueue; _triggerExecutor = triggerExecutor; - _delayStrategy = delayStrategy; _exceptionHandler = exceptionHandler; _trace = trace; _queueConfiguration = queueConfiguration; + // if the function runs longer than this, the invisibility will be updated + // on a timer periodically for the duration of the function execution + _visibilityTimeout = TimeSpan.FromMinutes(10); + if (sharedWatcher != null) { // Call Notify whenever a function adds a message to this queue. @@ -80,10 +85,19 @@ public QueueListener(IStorageQueue queue, _sharedWatcher = sharedWatcher; } - EventHandler poisonMessageEventHandler = _sharedWatcher != null ? OnMessageAddedToPoisonQueue : (EventHandler)null; - _queueProcessor = CreateQueueProcessor( + EventHandler poisonMessageEventHandler = _sharedWatcher != null ? OnMessageAddedToPoisonQueue : (EventHandler)null; + _queueProcessor = queueProcessor ?? CreateQueueProcessor( _queue.SdkObject, _poisonQueue != null ? _poisonQueue.SdkObject : null, _trace, _queueConfiguration, poisonMessageEventHandler); + + TimeSpan maximumInterval = _queueProcessor.MaxPollingInterval; + if (maxPollingInterval.HasValue && maximumInterval > maxPollingInterval.Value) + { + // enforce the maximum polling interval if specified + maximumInterval = maxPollingInterval.Value; + } + + _delayStrategy = new RandomizedExponentialBackoffStrategy(QueuePollingIntervals.Minimum, maximumInterval); } public void Cancel() @@ -134,14 +148,11 @@ public async Task ExecuteAsync(CancellationToken cancel return CreateBackoffResult(); } - // What if job takes longer. Call CloudQueue.UpdateMessage - TimeSpan visibilityTimeout = TimeSpan.FromMinutes(10); // long enough to process the job IEnumerable batch; - try { batch = await _queue.GetMessagesAsync(_queueProcessor.BatchSize, - visibilityTimeout, + _visibilityTimeout, options: null, operationContext: null, cancellationToken: cancellationToken); @@ -181,7 +192,7 @@ public async Task ExecuteAsync(CancellationToken cancel // of the cancellation token contract. However, the timer implementation would not dispose of the // cancellation token source until it has stopped and perhaps also disposed, and we wait for all // outstanding tasks to complete before stopping the timer. - Task task = ProcessMessageAsync(message, visibilityTimeout, cancellationToken); + Task task = ProcessMessageAsync(message, _visibilityTimeout, cancellationToken); // Having both WaitForNewBatchThreshold and this method mutate _processing is safe because the timer // contract is serial: it only calls ExecuteAsync once the wait expires (and the wait won't expire until @@ -271,9 +282,11 @@ internal async Task ProcessMessageAsync(IStorageQueueMessage message, TimeSpan v } } - private void OnMessageAddedToPoisonQueue(object sender, EventArgs e) + private void OnMessageAddedToPoisonQueue(object sender, PoisonMessageEventArgs e) { - _sharedWatcher.Notify(_poisonQueue.Name); + // TODO: this is assuming that the poison queue is in the same + // storage account + _sharedWatcher.Notify(e.PoisonQueue.Name); } private static ITaskSeriesTimer CreateUpdateMessageVisibilityTimer(IStorageQueue queue, @@ -296,7 +309,7 @@ private void ThrowIfDisposed() } } - internal static QueueProcessor CreateQueueProcessor(CloudQueue queue, CloudQueue poisonQueue, TraceWriter trace, IQueueConfiguration queueConfig, EventHandler poisonQueueMessageAddedHandler) + internal static QueueProcessor CreateQueueProcessor(CloudQueue queue, CloudQueue poisonQueue, TraceWriter trace, IQueueConfiguration queueConfig, EventHandler poisonQueueMessageAddedHandler) { QueueProcessorFactoryContext context = new QueueProcessorFactoryContext(queue, trace, queueConfig, poisonQueue); diff --git a/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/QueueListenerFactory.cs b/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/QueueListenerFactory.cs index ac6484235..d7c2ab50b 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/QueueListenerFactory.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Queues/Listeners/QueueListenerFactory.cs @@ -83,12 +83,10 @@ public Task CreateAsync(CancellationToken cancellationToken) { QueueTriggerExecutor triggerExecutor = new QueueTriggerExecutor(_executor); - IDelayStrategy delayStrategy = new RandomizedExponentialBackoffStrategy(QueuePollingIntervals.Minimum, _queueConfiguration.MaxPollingInterval); - SharedQueueWatcher sharedWatcher = _sharedContextProvider.GetOrCreateInstance( new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter)); - IListener listener = new QueueListener(_queue, _poisonQueue, triggerExecutor, delayStrategy, + IListener listener = new QueueListener(_queue, _poisonQueue, triggerExecutor, _exceptionHandler, _trace, sharedWatcher, _queueConfiguration); return Task.FromResult(listener); diff --git a/src/Microsoft.Azure.WebJobs.Host/Queues/PoisonQueueMessageEventArgs.cs b/src/Microsoft.Azure.WebJobs.Host/Queues/PoisonQueueMessageEventArgs.cs new file mode 100644 index 000000000..23252e629 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Queues/PoisonQueueMessageEventArgs.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using Microsoft.WindowsAzure.Storage.Queue; + +namespace Microsoft.Azure.WebJobs.Host.Queues +{ + /// + /// Event argument class for when poison messages + /// are added to a poison queue. + /// + [CLSCompliant(false)] + public class PoisonMessageEventArgs : EventArgs + { + /// + /// Constructs a new instance. + /// + /// The poison message + /// The poison queue + public PoisonMessageEventArgs(CloudQueueMessage message, CloudQueue poisonQueue) + { + Message = message; + PoisonQueue = poisonQueue; + } + + /// + /// The poison message + /// + public CloudQueueMessage Message { get; private set; } + + /// + /// The poison queue + /// + public CloudQueue PoisonQueue { get; private set; } + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/Queues/QueueProcessor.cs b/src/Microsoft.Azure.WebJobs.Host/Queues/QueueProcessor.cs index 449c212bf..5cedc685a 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Queues/QueueProcessor.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Queues/QueueProcessor.cs @@ -26,7 +26,6 @@ public class QueueProcessor private readonly CloudQueue _queue; private readonly CloudQueue _poisonQueue; private readonly TraceWriter _trace; - private readonly int _maxDequeueCount; /// /// Constructs a new instance. @@ -42,27 +41,46 @@ public QueueProcessor(QueueProcessorFactoryContext context) _queue = context.Queue; _poisonQueue = context.PoisonQueue; _trace = context.Trace; - _maxDequeueCount = context.MaxDequeueCount; + MaxDequeueCount = context.MaxDequeueCount; BatchSize = context.BatchSize; NewBatchThreshold = context.NewBatchThreshold; + VisibilityTimeout = context.VisibilityTimeout; + MaxPollingInterval = context.MaxPollingInterval; } /// /// Event raised when a message is added to the poison queue. /// - public event EventHandler MessageAddedToPoisonQueue; + public event EventHandler MessageAddedToPoisonQueue; /// /// Gets or sets the number of queue messages to retrieve and process in parallel. /// public int BatchSize { get; protected set; } + /// + /// Gets or sets the number of times to try processing a message before moving it to the poison queue. + /// + public int MaxDequeueCount { get; protected set; } + /// /// Gets or sets the threshold at which a new batch of messages will be fetched. /// public int NewBatchThreshold { get; protected set; } + /// + /// Gets or sets the longest period of time to wait before checking for a message to arrive when a queue remains + /// empty. + /// + public TimeSpan MaxPollingInterval { get; protected set; } + + /// + /// Gets or sets the default message visibility timeout that will be used + /// for messages that fail processing. + /// + public TimeSpan VisibilityTimeout { get; protected set; } + /// /// This method is called when there is a new message to process, before the job function is invoked. /// This allows any preprocessing to take place on the message before processing begins. @@ -95,14 +113,14 @@ public virtual async Task CompleteProcessingMessageAsync(CloudQueueMessage messa } else if (_poisonQueue != null) { - if (message.DequeueCount >= _maxDequeueCount) + if (message.DequeueCount >= MaxDequeueCount) { - await CopyMessageToPoisonQueueAsync(message, cancellationToken); + await CopyMessageToPoisonQueueAsync(message, _poisonQueue, cancellationToken); await DeleteMessageAsync(message, cancellationToken); } else { - await ReleaseMessageAsync(message, result, TimeSpan.Zero, cancellationToken); + await ReleaseMessageAsync(message, result, VisibilityTimeout, cancellationToken); } } else @@ -117,15 +135,17 @@ public virtual async Task CompleteProcessingMessageAsync(CloudQueueMessage messa /// Moves the specified message to the poison queue. /// /// The poison message + /// The poison queue to copy the message to /// The to use /// - protected virtual async Task CopyMessageToPoisonQueueAsync(CloudQueueMessage message, CancellationToken cancellationToken) + protected virtual async Task CopyMessageToPoisonQueueAsync(CloudQueueMessage message, CloudQueue poisonQueue, CancellationToken cancellationToken) { - _trace.Warning(string.Format(CultureInfo.InvariantCulture, "Message has reached MaxDequeueCount of {0}. Moving message to queue '{1}'.", _maxDequeueCount, _poisonQueue.Name), TraceSource.Execution); + _trace.Warning(string.Format(CultureInfo.InvariantCulture, "Message has reached MaxDequeueCount of {0}. Moving message to queue '{1}'.", MaxDequeueCount, poisonQueue.Name), TraceSource.Execution); - await AddMessageAndCreateIfNotExistsAsync(_poisonQueue, message, cancellationToken); + await AddMessageAndCreateIfNotExistsAsync(poisonQueue, message, cancellationToken); - OnMessageAddedToPoisonQueue(EventArgs.Empty); + var eventArgs = new PoisonMessageEventArgs(message, poisonQueue); + OnMessageAddedToPoisonQueue(eventArgs); } /// @@ -200,13 +220,9 @@ protected virtual async Task DeleteMessageAsync(CloudQueueMessage message, Cance /// Called to raise the MessageAddedToPoisonQueue event /// /// The event arguments - protected internal virtual void OnMessageAddedToPoisonQueue(EventArgs e) + protected internal virtual void OnMessageAddedToPoisonQueue(PoisonMessageEventArgs e) { - EventHandler handler = MessageAddedToPoisonQueue; - if (handler != null) - { - handler(this, e); - } + MessageAddedToPoisonQueue?.Invoke(this, e); } private static async Task AddMessageAndCreateIfNotExistsAsync(CloudQueue queue, CloudQueueMessage message, CancellationToken cancellationToken) diff --git a/src/Microsoft.Azure.WebJobs.Host/Queues/QueueProcessorFactoryContext.cs b/src/Microsoft.Azure.WebJobs.Host/Queues/QueueProcessorFactoryContext.cs index d3ba86f4f..f2d6a2568 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Queues/QueueProcessorFactoryContext.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Queues/QueueProcessorFactoryContext.cs @@ -47,6 +47,8 @@ internal QueueProcessorFactoryContext(CloudQueue queue, TraceWriter trace, IQueu BatchSize = queueConfiguration.BatchSize; MaxDequeueCount = queueConfiguration.MaxDequeueCount; NewBatchThreshold = queueConfiguration.NewBatchThreshold; + VisibilityTimeout = queueConfiguration.VisibilityTimeout; + MaxPollingInterval = queueConfiguration.MaxPollingInterval; } /// @@ -80,5 +82,17 @@ internal QueueProcessorFactoryContext(CloudQueue queue, TraceWriter trace, IQueu /// Gets or sets the threshold at which a new batch of messages will be fetched. /// public int NewBatchThreshold { get; set; } + + /// + /// Gets or sets the longest period of time to wait before checking for a message to arrive when a queue remains + /// empty. + /// + public TimeSpan MaxPollingInterval { get; set; } + + /// + /// Gets or sets the message visibility that will be used for messages that + /// fail processing. + /// + public TimeSpan VisibilityTimeout { get; set; } } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Queues/Triggers/QueueMessageValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Queues/Triggers/QueueMessageValueProvider.cs index 58ff64c8a..760d7cbb5 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Queues/Triggers/QueueMessageValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Queues/Triggers/QueueMessageValueProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; using Microsoft.Azure.WebJobs.Host.Storage.Queue; @@ -30,9 +31,9 @@ public Type Type get { return _valueType; } } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonListener.cs b/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonListener.cs index 10db255d5..42a1a80ee 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonListener.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonListener.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Globalization; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -15,11 +16,12 @@ internal class SingletonListener : IListener private readonly SingletonManager _singletonManager; private readonly SingletonConfiguration _singletonConfig; private readonly IListener _innerListener; + private readonly TraceWriter _trace; private string _lockId; private object _lockHandle; private bool _isListening; - public SingletonListener(MethodInfo method, SingletonAttribute attribute, SingletonManager singletonManager, IListener innerListener) + public SingletonListener(MethodInfo method, SingletonAttribute attribute, SingletonManager singletonManager, IListener innerListener, TraceWriter trace) { _attribute = attribute; _singletonManager = singletonManager; @@ -29,6 +31,8 @@ public SingletonListener(MethodInfo method, SingletonAttribute attribute, Single string boundScopeId = _singletonManager.GetBoundScopeId(_attribute.ScopeId); _lockId = singletonManager.FormatLockId(method, _attribute.Scope, boundScopeId); _lockId += ".Listener"; + + _trace = trace; } // exposed for testing @@ -43,6 +47,8 @@ public async Task StartAsync(CancellationToken cancellationToken) if (_lockHandle == null) { + _trace.Verbose(string.Format(CultureInfo.InvariantCulture, "Unable to acquire Singleton lock ({0}).", _lockId), source: TraceSource.Execution); + // If we're unable to acquire the lock, it means another listener // has it so we return w/o starting our listener. // @@ -76,7 +82,7 @@ public async Task StopAsync(CancellationToken cancellationToken) { await _innerListener.StopAsync(cancellationToken); _isListening = false; - } + } } public void Cancel() @@ -120,7 +126,7 @@ internal async Task TryAcquireLock() LockTimer.Dispose(); LockTimer = null; } - + await _innerListener.StartAsync(CancellationToken.None); _isListening = true; diff --git a/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonManager.cs b/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonManager.cs index 340a7b141..801df64cd 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonManager.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonManager.cs @@ -74,8 +74,8 @@ internal string HostId } // for testing - internal TimeSpan MinimumLeaseRenewalInterval - { + internal TimeSpan MinimumLeaseRenewalInterval + { get { return _minimumLeaseRenewalInterval; @@ -92,8 +92,8 @@ public async virtual Task LockAsync(string lockId, string functionInstan if (lockHandle == null) { - TimeSpan acquisitionTimeout = attribute.LockAcquisitionTimeout != null - ? TimeSpan.FromSeconds(attribute.LockAcquisitionTimeout.Value) : + TimeSpan acquisitionTimeout = attribute.LockAcquisitionTimeout != null + ? TimeSpan.FromSeconds(attribute.LockAcquisitionTimeout.Value) : _config.LockAcquisitionTimeout; throw new TimeoutException(string.Format("Unable to acquire singleton lock blob lease for blob '{0}' (timeout of {1} exceeded).", lockId, acquisitionTimeout.ToString("g"))); } @@ -103,8 +103,6 @@ public async virtual Task LockAsync(string lockId, string functionInstan public async virtual Task TryLockAsync(string lockId, string functionInstanceId, SingletonAttribute attribute, CancellationToken cancellationToken, bool retry = true) { - _trace.Verbose(string.Format(CultureInfo.InvariantCulture, "Waiting for Singleton lock ({0})", lockId), source: TraceSource.Execution); - IStorageBlobDirectory lockDirectory = GetLockDirectory(attribute.Account); IStorageBlockBlob lockBlob = lockDirectory.GetBlockBlobReference(lockId); TimeSpan lockPeriod = GetLockPeriod(attribute, _config); @@ -128,7 +126,6 @@ public async virtual Task TryLockAsync(string lockId, string functionIns if (string.IsNullOrEmpty(leaseId)) { - _trace.Verbose(string.Format(CultureInfo.InvariantCulture, "Unable to acquire Singleton lock ({0}).", lockId), source: TraceSource.Execution); return null; } @@ -254,7 +251,7 @@ public SingletonListener CreateHostSingletonListener(IListener innerListener, st { Mode = SingletonMode.Listener }; - return new SingletonListener(null, singletonAttribute, this, innerListener); + return new SingletonListener(null, singletonAttribute, this, innerListener, _trace); } public static SingletonAttribute GetListenerSingletonOrNull(Type listenerType, MethodInfo method) @@ -347,18 +344,18 @@ internal static TimeSpan GetLockPeriod(SingletonAttribute attribute, SingletonCo config.ListenerLockPeriod : config.LockPeriod; } - private ITaskSeriesTimer CreateLeaseRenewalTimer(IStorageBlockBlob leaseBlob, string leaseId, string lockId, TimeSpan leasePeriod, + private ITaskSeriesTimer CreateLeaseRenewalTimer(IStorageBlockBlob leaseBlob, string leaseId, string lockId, TimeSpan leasePeriod, IWebJobsExceptionHandler exceptionHandler) { // renew the lease when it is halfway to expiring TimeSpan normalUpdateInterval = new TimeSpan(leasePeriod.Ticks / 2); IDelayStrategy speedupStrategy = new LinearSpeedupStrategy(normalUpdateInterval, MinimumLeaseRenewalInterval); - ITaskSeriesCommand command = new RenewLeaseCommand(leaseBlob, leaseId, lockId, speedupStrategy, _trace); + ITaskSeriesCommand command = new RenewLeaseCommand(leaseBlob, leaseId, lockId, speedupStrategy, _trace, leasePeriod); return new TaskSeriesTimer(command, exceptionHandler, Task.Delay(normalUpdateInterval)); } - private async Task TryAcquireLeaseAsync(IStorageBlockBlob blob, TimeSpan leasePeriod, CancellationToken cancellationToken) + private static async Task TryAcquireLeaseAsync(IStorageBlockBlob blob, TimeSpan leasePeriod, CancellationToken cancellationToken) { bool blobDoesNotExist = false; try @@ -415,7 +412,7 @@ private async Task TryAcquireLeaseAsync(IStorageBlockBlob blob, TimeSpan return null; } - private async Task ReleaseLeaseAsync(IStorageBlockBlob blob, string leaseId, CancellationToken cancellationToken) + private static async Task ReleaseLeaseAsync(IStorageBlockBlob blob, string leaseId, CancellationToken cancellationToken) { try { @@ -450,7 +447,7 @@ await blob.ReleaseLeaseAsync( } } - private async Task TryCreateAsync(IStorageBlockBlob blob, CancellationToken cancellationToken) + private static async Task TryCreateAsync(IStorageBlockBlob blob, CancellationToken cancellationToken) { bool isContainerNotFoundException = false; @@ -467,7 +464,7 @@ private async Task TryCreateAsync(IStorageBlockBlob blob, CancellationToke { isContainerNotFoundException = true; } - else if (exception.RequestInformation.HttpStatusCode == 409 || + else if (exception.RequestInformation.HttpStatusCode == 409 || exception.RequestInformation.HttpStatusCode == 412) { // The blob already exists, or is leased by someone else @@ -507,7 +504,7 @@ private async Task TryCreateAsync(IStorageBlockBlob blob, CancellationToke } } - private async Task WriteLeaseBlobMetadata(IStorageBlockBlob blob, string leaseId, string functionInstanceId, CancellationToken cancellationToken) + private static async Task WriteLeaseBlobMetadata(IStorageBlockBlob blob, string leaseId, string functionInstanceId, CancellationToken cancellationToken) { blob.Metadata.Add(FunctionInstanceMetadataKey, functionInstanceId); @@ -518,7 +515,7 @@ await blob.SetMetadataAsync( cancellationToken: cancellationToken); } - private async Task ReadLeaseBlobMetadata(IStorageBlockBlob blob, CancellationToken cancellationToken) + private static async Task ReadLeaseBlobMetadata(IStorageBlockBlob blob, CancellationToken cancellationToken) { try { @@ -553,14 +550,19 @@ internal class RenewLeaseCommand : ITaskSeriesCommand private readonly string _lockId; private readonly IDelayStrategy _speedupStrategy; private readonly TraceWriter _trace; + private DateTimeOffset _lastRenewal; + private TimeSpan _lastRenewalLatency; + private TimeSpan _leasePeriod; - public RenewLeaseCommand(IStorageBlockBlob leaseBlob, string leaseId, string lockId, IDelayStrategy speedupStrategy, TraceWriter trace) + public RenewLeaseCommand(IStorageBlockBlob leaseBlob, string leaseId, string lockId, IDelayStrategy speedupStrategy, TraceWriter trace, TimeSpan leasePeriod) { + _lastRenewal = DateTimeOffset.UtcNow; _leaseBlob = leaseBlob; _leaseId = leaseId; _lockId = lockId; _speedupStrategy = speedupStrategy; _trace = trace; + _leasePeriod = leasePeriod; } public async Task ExecuteAsync(CancellationToken cancellationToken) @@ -569,13 +571,14 @@ public async Task ExecuteAsync(CancellationToken cancel try { - _trace.Verbose(string.Format(CultureInfo.InvariantCulture, "Renewing Singleton lock ({0})", _lockId), source: TraceSource.Execution); - AccessCondition condition = new AccessCondition { LeaseId = _leaseId }; + DateTimeOffset requestStart = DateTimeOffset.UtcNow; await _leaseBlob.RenewLeaseAsync(condition, null, null, cancellationToken); + _lastRenewal = DateTime.UtcNow; + _lastRenewalLatency = _lastRenewal - requestStart; // The next execution should occur after a normal delay. delay = _speedupStrategy.GetNextDelay(executionSucceeded: true); @@ -586,17 +589,47 @@ public async Task ExecuteAsync(CancellationToken cancel { // The next execution should occur more quickly (try to renew the lease before it expires). delay = _speedupStrategy.GetNextDelay(executionSucceeded: false); + _trace.Warning(string.Format(CultureInfo.InvariantCulture, "Singleton lock renewal failed for blob '{0}' with error code {1}. Retry renewal in {2} milliseconds.", + _lockId, FormatErrorCode(exception), delay.TotalMilliseconds), source: TraceSource.Execution); } else { - // If we've lost the lease or cannot restablish it, we want to fail any + // Log the details we've been accumulating to help with debugging this scenario + int leasePeriodMilliseconds = (int)_leasePeriod.TotalMilliseconds; + string lastRenewalFormatted = _lastRenewal.ToString("yyyy-MM-ddTHH:mm:ss.FFFZ", CultureInfo.InvariantCulture); + int millisecondsSinceLastSuccess = (int)(DateTime.UtcNow - _lastRenewal).TotalMilliseconds; + int lastRenewalMilliseconds = (int)_lastRenewalLatency.TotalMilliseconds; + + _trace.Error(string.Format(CultureInfo.InvariantCulture, "Singleton lock renewal failed for blob '{0}' with error code {1}. The last successful renewal completed at {2} ({3} milliseconds ago) with a duration of {4} milliseconds. The lease period was {5} milliseconds.", + _lockId, FormatErrorCode(exception), lastRenewalFormatted, millisecondsSinceLastSuccess, lastRenewalMilliseconds, leasePeriodMilliseconds)); + + // If we've lost the lease or cannot re-establish it, we want to fail any // in progress function execution throw; } } - return new TaskSeriesCommandResult(wait: Task.Delay(delay)); } + + private static string FormatErrorCode(StorageException exception) + { + int statusCode; + if (!exception.TryGetStatusCode(out statusCode)) + { + return "''"; + } + + string message = statusCode.ToString(CultureInfo.InvariantCulture); + + string errorCode = exception.GetErrorCode(); + + if (errorCode != null) + { + message += ": " + errorCode; + } + + return message; + } } } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonValueProvider.cs index 52883fc77..0f4c162b4 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Singleton/SingletonValueProvider.cs @@ -32,7 +32,7 @@ public SingletonValueProvider(MethodInfo method, string scopeId, string function public Type Type { - get + get { return typeof(SingletonLock); } @@ -40,15 +40,15 @@ public Type Type public IWatcher Watcher { - get + get { - return _watcher; + return _watcher; } } - public object GetValue() + public Task GetValueAsync() { - return _singletonLock; + return Task.FromResult(_singletonLock); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/AsyncCollectorArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/AsyncCollectorArgumentBindingProvider.cs deleted file mode 100644 index 530a967fd..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/AsyncCollectorArgumentBindingProvider.cs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.IO; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Storage.Table; -using Microsoft.WindowsAzure.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class AsyncCollectorArgumentBindingProvider : IStorageTableArgumentBindingProvider - { - public IStorageTableArgumentBinding TryCreate(ParameterInfo parameter) - { - if (!parameter.ParameterType.IsGenericType || - (parameter.ParameterType.GetGenericTypeDefinition() != typeof(IAsyncCollector<>))) - { - return null; - } - - Type entityType = GetCollectorItemType(parameter.ParameterType); - - if (!TableClient.ImplementsOrEqualsITableEntity(entityType)) - { - TableClient.VerifyContainsProperty(entityType, "RowKey"); - TableClient.VerifyContainsProperty(entityType, "PartitionKey"); - } - - return CreateBinding(entityType); - } - - private static Type GetCollectorItemType(Type queryableType) - { - Type[] genericArguments = queryableType.GetGenericArguments(); - var itemType = genericArguments[0]; - return itemType; - } - - private static IStorageTableArgumentBinding CreateBinding(Type entityType) - { - if (TableClient.ImplementsOrEqualsITableEntity(entityType)) - { - Type genericType = typeof(TableEntityAsyncCollectorArgumentBinding<>).MakeGenericType(entityType); - return (IStorageTableArgumentBinding)Activator.CreateInstance(genericType); - } - else - { - Type genericType = typeof(PocoEntityAsyncCollectorArgumentBinding<>).MakeGenericType(entityType); - return (IStorageTableArgumentBinding)Activator.CreateInstance(genericType); - } - } - - private class TableEntityAsyncCollectorArgumentBinding : IStorageTableArgumentBinding - where TElement : ITableEntity - { - public FileAccess Access - { - get { return FileAccess.Write; } - } - - public Type ValueType - { - get { return typeof(IAsyncCollector); } - } - - public Task BindAsync(IStorageTable value, ValueBindingContext context) - { - TableEntityWriter tableWriter = new TableEntityWriter(value); - IValueProvider provider = new TableEntityCollectorBinder(value, tableWriter, typeof(IAsyncCollector)); - return Task.FromResult(provider); - } - } - - private class PocoEntityAsyncCollectorArgumentBinding : IStorageTableArgumentBinding - { - public FileAccess Access - { - get { return FileAccess.Write; } - } - - public Type ValueType - { - get { return typeof(IAsyncCollector); } - } - - public Task BindAsync(IStorageTable value, ValueBindingContext context) - { - PocoEntityWriter collector = new PocoEntityWriter(value); - IValueProvider provider = new PocoEntityCollectorBinder(value, collector, typeof(IAsyncCollector)); - return Task.FromResult(provider); - } - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/BindableTablePath.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/BindableTablePath.cs deleted file mode 100644 index a8ea6f835..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/BindableTablePath.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Bindings.Path; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal static class BindableTablePath - { - public static IBindableTablePath Create(string tableNamePattern) - { - BindingTemplate template = BindingTemplate.FromString(tableNamePattern); - - if (template.ParameterNames.Count() > 0) - { - return new ParameterizedTablePath(template); - } - - TableClient.ValidateAzureTableName(tableNamePattern); - return new BoundTablePath(tableNamePattern); - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/BoundTablePath.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/BoundTablePath.cs deleted file mode 100644 index b45528f65..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/BoundTablePath.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Collections.Generic; -using System.Linq; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class BoundTablePath : IBindableTablePath - { - private readonly string _tableName; - - public BoundTablePath(string tableName) - { - _tableName = tableName; - } - - public string TableNamePattern - { - get { return _tableName; } - } - - public bool IsBound - { - get { return true; } - } - - public IEnumerable ParameterNames - { - get { return Enumerable.Empty(); } - } - - public string Bind(IReadOnlyDictionary bindingData) - { - return _tableName; - } - - public static string Validate(string value) - { - TableClient.ValidateAzureTableName(value); - return value; - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/CloudQueueToStorageQueueConverter.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/CloudQueueToStorageQueueConverter.cs deleted file mode 100644 index d989dadd0..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/CloudQueueToStorageQueueConverter.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using Microsoft.Azure.WebJobs.Host.Converters; -using Microsoft.Azure.WebJobs.Host.Storage.Table; -using Microsoft.WindowsAzure.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class CloudTableToStorageTableConverter : IConverter - { - public IStorageTable Convert(CloudTable input) - { - if (input == null) - { - return null; - } - - return new StorageTable(input); - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/CloudTableArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/CloudTableArgumentBindingProvider.cs deleted file mode 100644 index 7a7bf091c..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/CloudTableArgumentBindingProvider.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.IO; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Storage.Table; -using Microsoft.WindowsAzure.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class CloudTableArgumentBindingProvider : IStorageTableArgumentBindingProvider - { - public IStorageTableArgumentBinding TryCreate(ParameterInfo parameter) - { - if (parameter.ParameterType != typeof(CloudTable)) - { - return null; - } - - return new CloudTableArgumentBinding(); - } - - private class CloudTableArgumentBinding : IStorageTableArgumentBinding - { - public Type ValueType - { - get { return typeof(CloudTable); } - } - - public FileAccess Access - { - get - { - return FileAccess.ReadWrite; - } - } - - public async Task BindAsync(IStorageTable value, ValueBindingContext context) - { - await value.CreateIfNotExistsAsync(context.CancellationToken); - return new TableValueProvider(value, value.SdkObject, typeof(CloudTable)); - } - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/CollectorArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/CollectorArgumentBindingProvider.cs deleted file mode 100644 index 579d9b673..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/CollectorArgumentBindingProvider.cs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.IO; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Storage.Table; -using Microsoft.WindowsAzure.Storage.Table; -using Newtonsoft.Json.Linq; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class CollectorArgumentBindingProvider : IStorageTableArgumentBindingProvider - { - public IStorageTableArgumentBinding TryCreate(ParameterInfo parameter) - { - if (!parameter.ParameterType.IsGenericType || - (parameter.ParameterType.GetGenericTypeDefinition() != typeof(ICollector<>))) - { - return null; - } - - Type entityType = GetCollectorItemType(parameter.ParameterType); - - if (!TableClient.ImplementsOrEqualsITableEntity(entityType) && !TypeUtility.IsJObject(entityType)) - { - TableClient.VerifyContainsProperty(entityType, "RowKey"); - TableClient.VerifyContainsProperty(entityType, "PartitionKey"); - } - - return CreateBinding(entityType); - } - - private static Type GetCollectorItemType(Type queryableType) - { - Type[] genericArguments = queryableType.GetGenericArguments(); - var itemType = genericArguments[0]; - return itemType; - } - - private static IStorageTableArgumentBinding CreateBinding(Type entityType) - { - if (TableClient.ImplementsOrEqualsITableEntity(entityType)) - { - Type genericType = typeof(TableEntityCollectorArgumentBinding<>).MakeGenericType(entityType); - return (IStorageTableArgumentBinding)Activator.CreateInstance(genericType); - } - else - { - Type genericType = typeof(PocoEntityCollectorArgumentBinding<>).MakeGenericType(entityType); - return (IStorageTableArgumentBinding)Activator.CreateInstance(genericType); - } - } - - private class TableEntityCollectorArgumentBinding : IStorageTableArgumentBinding - where TElement : ITableEntity - { - public FileAccess Access - { - get { return FileAccess.Write; } - } - - public Type ValueType - { - get { return typeof(ICollector); } - } - - public Task BindAsync(IStorageTable value, ValueBindingContext context) - { - TableEntityWriter tableWriter = new TableEntityWriter(value); - IValueProvider provider = new TableEntityCollectorBinder(value, tableWriter, typeof(ICollector)); - return Task.FromResult(provider); - } - } - - private class PocoEntityCollectorArgumentBinding : IStorageTableArgumentBinding - { - public FileAccess Access - { - get { return FileAccess.Write; } - } - - public Type ValueType - { - get { return typeof(ICollector); } - } - - public Task BindAsync(IStorageTable value, ValueBindingContext context) - { - PocoEntityWriter collector = new PocoEntityWriter(value); - IValueProvider provider = new PocoEntityCollectorBinder(value, collector, typeof(ICollector)); - return Task.FromResult(provider); - } - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/CompositeArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/CompositeArgumentBindingProvider.cs deleted file mode 100644 index 2d5caeb7b..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/CompositeArgumentBindingProvider.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Collections.Generic; -using System.Reflection; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class CompositeArgumentBindingProvider : IStorageTableArgumentBindingProvider - { - private readonly IEnumerable _providers; - - public CompositeArgumentBindingProvider(params IStorageTableArgumentBindingProvider[] providers) - { - _providers = providers; - } - - public IStorageTableArgumentBinding TryCreate(ParameterInfo parameter) - { - foreach (IStorageTableArgumentBindingProvider provider in _providers) - { - IStorageTableArgumentBinding binding = provider.TryCreate(parameter); - - if (binding != null) - { - return binding; - } - } - - return null; - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/IBindableTablePath.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/IBindableTablePath.cs deleted file mode 100644 index d797e3e8e..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/IBindableTablePath.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using Microsoft.Azure.WebJobs.Host.Bindings; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal interface IBindableTablePath : IBindablePath - { - string TableNamePattern { get; } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/IStorageTableArgumentBinding.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/IStorageTableArgumentBinding.cs deleted file mode 100644 index 438ba2f36..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/IStorageTableArgumentBinding.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.IO; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - /// - /// Defines an argument binding for arguments. - /// - /// is our own internal abstraction used for testing, - /// and is not exposed publically. - internal interface IStorageTableArgumentBinding : IArgumentBinding - { - /// - /// Gets the that defines the storage operations the - /// binding supports. - /// - FileAccess Access { get; } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/IStorageTableArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/IStorageTableArgumentBindingProvider.cs deleted file mode 100644 index 766944c0a..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/IStorageTableArgumentBindingProvider.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Reflection; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal interface IStorageTableArgumentBindingProvider - { - IStorageTableArgumentBinding TryCreate(ParameterInfo parameter); - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/ITableArgumentBinding.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/ITableArgumentBinding.cs deleted file mode 100644 index 9746c9eca..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/ITableArgumentBinding.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.IO; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.WindowsAzure.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - /// - /// Defines an argument binding for arguments. - /// - [CLSCompliant(false)] - public interface ITableArgumentBinding : IArgumentBinding - { - /// - /// Gets the that defines the storage operations the - /// binding supports. - /// - FileAccess Access { get; } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/NullEntityValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/NullEntityValueProvider.cs index 22a6e2326..18f4391e9 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/NullEntityValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Tables/NullEntityValueProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; namespace Microsoft.Azure.WebJobs.Host.Tables @@ -20,9 +21,9 @@ public Type Type get { return typeof(TElement); } } - public object GetValue() + public Task GetValueAsync() { - return default(TElement); + return Task.FromResult((object)default(TElement)); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/ParameterizedTablePath.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/ParameterizedTablePath.cs deleted file mode 100644 index 31a694242..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/ParameterizedTablePath.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Bindings.Path; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class ParameterizedTablePath : IBindableTablePath - { - private readonly BindingTemplate _template; - - public ParameterizedTablePath(BindingTemplate template) - { - Debug.Assert(template.ParameterNames.Count() > 0); - - _template = template; - } - - public string TableNamePattern - { - get { return _template.Pattern; } - } - - public bool IsBound - { - get { return false; } - } - - public IEnumerable ParameterNames - { - get { return _template.ParameterNames; } - } - - public string Bind(IReadOnlyDictionary bindingData) - { - IReadOnlyDictionary parameters = BindingDataPathHelper.ConvertParameters(bindingData); - string tableName = _template.Bind(parameters); - - TableClient.ValidateAzureTableName(tableName); - - return tableName; - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/PocoEntityCollectorBinder.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/PocoEntityCollectorBinder.cs index da8e23498..e9a671e46 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/PocoEntityCollectorBinder.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Tables/PocoEntityCollectorBinder.cs @@ -41,9 +41,9 @@ public IWatcher Watcher } } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/PocoEntityValueBinder.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/PocoEntityValueBinder.cs index bf0acd81b..e89614e33 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/PocoEntityValueBinder.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Tables/PocoEntityValueBinder.cs @@ -3,11 +3,9 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Converters; using Microsoft.Azure.WebJobs.Host.Protocols; using Microsoft.Azure.WebJobs.Host.Storage.Table; using Microsoft.WindowsAzure.Storage.Table; @@ -51,9 +49,9 @@ public bool HasChanged } } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public Task SetValueAsync(object value, CancellationToken cancellationToken) diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/QueryableArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/QueryableArgumentBindingProvider.cs deleted file mode 100644 index fa27af357..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/QueryableArgumentBindingProvider.cs +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Storage.Table; -using Microsoft.WindowsAzure.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class QueryableArgumentBindingProvider : IStorageTableArgumentBindingProvider - { - public IStorageTableArgumentBinding TryCreate(ParameterInfo parameter) - { - if (!parameter.ParameterType.IsGenericType || parameter.ParameterType.GetGenericTypeDefinition() != typeof(IQueryable<>)) - { - return null; - } - - Type entityType = GetQueryableItemType(parameter.ParameterType); - - if (!TableClient.ImplementsITableEntity(entityType)) - { - throw new InvalidOperationException("IQueryable is only supported on types that implement ITableEntity."); - } - - TableClient.VerifyDefaultConstructor(entityType); - - return CreateBinding(entityType); - } - - private static Type GetQueryableItemType(Type queryableType) - { - Type[] genericArguments = queryableType.GetGenericArguments(); - var itemType = genericArguments[0]; - return itemType; - } - - private static IStorageTableArgumentBinding CreateBinding(Type entityType) - { - Type genericType = typeof(QueryableArgumentBinding<>).MakeGenericType(entityType); - return (IStorageTableArgumentBinding)Activator.CreateInstance(genericType); - } - - private class QueryableArgumentBinding : IStorageTableArgumentBinding - where TElement : ITableEntity, new() - { - public Type ValueType - { - get { return typeof(IQueryable); } - } - - public FileAccess Access - { - get - { - return FileAccess.Read; - } - } - - public async Task BindAsync(IStorageTable value, ValueBindingContext context) - { - IQueryable queryable; - - if (!await value.ExistsAsync(context.CancellationToken)) - { - queryable = Enumerable.Empty().AsQueryable(); - } - else - { - queryable = value.CreateQuery(); - } - - return new TableValueProvider(value, queryable, typeof(IQueryable)); - } - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/StorageTableArgumentBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/StorageTableArgumentBindingProvider.cs deleted file mode 100644 index e55a39d31..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/StorageTableArgumentBindingProvider.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.IO; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class StorageTableArgumentBindingProvider : IStorageTableArgumentBindingProvider - { - public IStorageTableArgumentBinding TryCreate(ParameterInfo parameter) - { - if (parameter.ParameterType != typeof(IStorageTable)) - { - return null; - } - - return new StorageTableArgumentBinding(); - } - - private class StorageTableArgumentBinding : IStorageTableArgumentBinding - { - public Type ValueType - { - get { return typeof(IStorageTable); } - } - - public FileAccess Access - { - get - { - return FileAccess.ReadWrite; - } - } - - public Task BindAsync(IStorageTable value, ValueBindingContext context) - { - IValueProvider valueProvider = new TableValueProvider(value, value, typeof(IStorageTable)); - return Task.FromResult(valueProvider); - } - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/StringToStorageTableConverter.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/StringToStorageTableConverter.cs deleted file mode 100644 index c8d84abe1..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/StringToStorageTableConverter.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using Microsoft.Azure.WebJobs.Host.Converters; -using Microsoft.Azure.WebJobs.Host.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class StringToStorageTableConverter : IConverter - { - private readonly IStorageTableClient _client; - private readonly IBindableTablePath _defaultPath; - - public StringToStorageTableConverter(IStorageTableClient client, IBindableTablePath defaultPath) - { - _client = client; - _defaultPath = defaultPath; - } - - public IStorageTable Convert(string input) - { - string tableName; - - // For convenience, treat an an empty string as a request for the default value (when valid). - if (String.IsNullOrEmpty(input) && _defaultPath.IsBound) - { - tableName = _defaultPath.Bind(null); - } - else - { - tableName = BoundTablePath.Validate(input); - } - - return _client.GetTableReference(tableName); - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/TableArgumentBindingExtensionProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/TableArgumentBindingExtensionProvider.cs deleted file mode 100644 index 54bd5d56d..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/TableArgumentBindingExtensionProvider.cs +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Storage.Table; -using Microsoft.WindowsAzure.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - /// - /// This binding provider loads any instances - /// registered with the . When it binds, it delegates to those - /// providers. - /// - internal class TableArgumentBindingExtensionProvider : IStorageTableArgumentBindingProvider - { - private IEnumerable> _bindingExtensionsProviders; - - public TableArgumentBindingExtensionProvider(IExtensionRegistry extensions) - { - _bindingExtensionsProviders = extensions.GetExtensions>(); - } - - public IStorageTableArgumentBinding TryCreate(ParameterInfo parameter) - { - // see if there are any registered binding extension providers that can - // bind to this parameter - foreach (IArgumentBindingProvider provider in _bindingExtensionsProviders) - { - ITableArgumentBinding bindingExtension = provider.TryCreate(parameter); - if (bindingExtension != null) - { - // if an extension is able to bind, wrap the binding - return new TableArgumentBindingExtension(bindingExtension); - } - } - - return null; - } - - /// - /// This binding wraps the actual extension binding and delegates to it. It exists to map - /// from from the internal interface into the - /// public ITableArgumentBinding interface. - /// - internal class TableArgumentBindingExtension : IStorageTableArgumentBinding - { - private ITableArgumentBinding _binding; - - public TableArgumentBindingExtension(ITableArgumentBinding binding) - { - _binding = binding; - } - - public FileAccess Access - { - get { return _binding.Access; } - } - - public Type ValueType - { - get { return _binding.ValueType; } - } - - public Task BindAsync(IStorageTable value, ValueBindingContext context) - { - CloudTable table = null; - if (value != null) - { - table = value.SdkObject; - } - return _binding.BindAsync(table, context); - } - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/TableAttributeBindingProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/TableAttributeBindingProvider.cs index d0cdce483..0d54ce465 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/TableAttributeBindingProvider.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Tables/TableAttributeBindingProvider.cs @@ -2,12 +2,15 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.IO; using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; +using Microsoft.Azure.WebJobs.Host.Converters; using Microsoft.Azure.WebJobs.Host.Executors; +using Microsoft.Azure.WebJobs.Host.Protocols; using Microsoft.Azure.WebJobs.Host.Storage; using Microsoft.Azure.WebJobs.Host.Storage.Table; using Microsoft.WindowsAzure.Storage.Table; @@ -17,7 +20,6 @@ namespace Microsoft.Azure.WebJobs.Host.Tables { internal class TableAttributeBindingProvider : IBindingProvider { - private readonly IStorageTableArgumentBindingProvider _tableBindingProvider; private readonly ITableEntityArgumentBindingProvider _entityBindingProvider; private readonly INameResolver _nameResolver; @@ -38,21 +40,13 @@ private TableAttributeBindingProvider(INameResolver nameResolver, IStorageAccoun _nameResolver = nameResolver; _accountProvider = accountProvider; - _tableBindingProvider = new CompositeArgumentBindingProvider( - new StorageTableArgumentBindingProvider(), - new CloudTableArgumentBindingProvider(), - new QueryableArgumentBindingProvider(), - new CollectorArgumentBindingProvider(), - new AsyncCollectorArgumentBindingProvider(), - new TableArgumentBindingExtensionProvider(extensions)); - _entityBindingProvider = new CompositeEntityArgumentBindingProvider( new TableEntityArgumentBindingProvider(), new PocoEntityArgumentBindingProvider()); // Supports all types; must come after other providers } - // [Table] has some pre-existing behavior where the storage account can be specified outside of the [Queue] attribute. + // [Table] has some pre-existing behavior where the storage account can be specified outside of the [Table] attribute. // The storage account is pulled from the ParameterInfo (which could pull in a [Storage] attribute on the container class) // Resolve everything back down to a single attribute so we can use the binding helpers. // This pattern should be rare since other extensions can just keep everything directly on the primary attribute. @@ -68,106 +62,79 @@ private async Task CollectAttributeInfo(TableAttribute attrResol return new ResolvedTableAttribute(attrResolved, client); } - + public static IBindingProvider Build(INameResolver nameResolver, IConverterManager converterManager, IStorageAccountProvider accountProvider, IExtensionRegistry extensions) { var original = new TableAttributeBindingProvider(nameResolver, accountProvider, extensions); - converterManager.AddConverter(original.JObjectToTableEntityConverterFunc); + converterManager.AddConverter(original.JObjectToTableEntityConverterFunc); + converterManager.AddConverter(typeof(ObjectToITableEntityConverter<>)); + // IStorageTable --> IQueryable + converterManager.AddConverter, TableAttribute>(typeof(TableToIQueryableConverter<>)); + var bindingFactory = new BindingFactory(nameResolver, converterManager); - var bindAsyncCollector = bindingFactory.BindToAsyncCollector(original.BuildFromTableAttribute, null, original.CollectAttributeInfo); - var bindToJobject = bindingFactory.BindToExactAsyncType(original.BuildJObject, null, original.CollectAttributeInfo); - var bindToJArray = bindingFactory.BindToExactAsyncType(original.BuildJArray, null, original.CollectAttributeInfo); + // Includes converter manager, which provides access to IQueryable + var bindToExactCloudTable = bindingFactory.BindToInput(typeof(JObjectBuilder)) + .SetPostResolveHook(original.ToParameterDescriptorForCollector, original.CollectAttributeInfo); + + var bindToExactTestCloudTable = bindingFactory.BindToInput(typeof(JObjectBuilder)) + .SetPostResolveHook(original.ToParameterDescriptorForCollector, original.CollectAttributeInfo); - // Filter to just support JObject, and use legacy bindings for everything else. - // Once we have ITableEntity converters for pocos, we can remove the filter. - // https://github.com/Azure/azure-webjobs-sdk/issues/887 - bindAsyncCollector = bindingFactory.AddFilter( - (attr, type) => (type == typeof(IAsyncCollector) || type == typeof(ICollector)), - bindAsyncCollector); + var bindAsyncCollector = bindingFactory.BindToCollector(original.BuildFromTableAttribute) + .SetPostResolveHook(null, original.CollectAttributeInfo); + + var bindToJobject = bindingFactory.BindToInput(typeof(JObjectBuilder)) + .SetPostResolveHook(null, original.CollectAttributeInfo); + + var bindToJArray = bindingFactory.BindToInput(typeof(JObjectBuilder)) + .SetPostResolveHook(null, original.CollectAttributeInfo); var bindingProvider = new GenericCompositeBindingProvider( - new IBindingProvider[] { bindToJArray, bindToJobject, bindAsyncCollector, original }); + ValidateAttribute, nameResolver, + new IBindingProvider[] + { + bindAsyncCollector, + AllowMultipleRows(bindingFactory, bindToExactCloudTable), + AllowMultipleRows(bindingFactory, bindToExactTestCloudTable), + bindToJArray, + bindToJobject, + original + }); return bindingProvider; } - private async Task BuildJObject(TableAttribute attribute) + // Binding rule only allowed on attributes that don't specify the RowKey. + private static IBindingProvider AllowMultipleRows(BindingFactory bf, IBindingProvider innerProvider) { - IStorageTable table = GetTable(attribute); - - IStorageTableOperation retrieve = table.CreateRetrieveOperation( - attribute.PartitionKey, attribute.RowKey); - TableResult result = await table.ExecuteAsync(retrieve, CancellationToken.None); - DynamicTableEntity entity = (DynamicTableEntity)result.Result; - if (entity == null) - { - return null; - } - else - { - var obj = ConvertEntityToJObject(entity); - return obj; - } + return bf.AddFilter((attr, type) => attr.RowKey == null, innerProvider); } - // Build a JArray. - // Used as an alternative to binding to IQueryable. - private async Task BuildJArray(TableAttribute attribute) + private ParameterDescriptor ToParameterDescriptorForCollector(TableAttribute attribute, ParameterInfo parameter, INameResolver nameResolver) { - var table = GetTable(attribute).SdkObject; - - string finalQuery = attribute.Filter; - if (!string.IsNullOrEmpty(attribute.PartitionKey)) - { - var partitionKeyPredicate = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, attribute.PartitionKey); - if (!string.IsNullOrEmpty(attribute.Filter)) - { - finalQuery = TableQuery.CombineFilters(attribute.Filter, TableOperators.And, partitionKeyPredicate); - } - else - { - finalQuery = partitionKeyPredicate; - } - } + Task t = Task.Run(() => + _accountProvider.GetStorageAccountAsync(parameter, CancellationToken.None, nameResolver)); + IStorageAccount account = t.GetAwaiter().GetResult(); + string accountName = account.Credentials.AccountName; - TableQuery tableQuery = new TableQuery + return new TableParameterDescriptor { - FilterString = finalQuery + Name = parameter.Name, + AccountName = accountName, + TableName = Resolve(attribute.TableName), + Access = FileAccess.ReadWrite }; - if (attribute.Take > 0) - { - tableQuery.TakeCount = attribute.Take; - } - int countRemaining = attribute.Take; - - JArray entityArray = new JArray(); - TableContinuationToken token = null; + } - do + private static void ValidateAttribute(TableAttribute attribute, Type parameterType) + { + // Queue pre-existing behavior: if there are { }in the path, then defer validation until runtime. + if (!attribute.TableName.Contains("{")) { - var segment = await table.ExecuteQuerySegmentedAsync(tableQuery, token); - var entities = segment.Results; - - token = segment.ContinuationToken; - - foreach (var entity in entities) - { - countRemaining--; - entityArray.Add(ConvertEntityToJObject(entity)); - - if (countRemaining == 0) - { - token = null; - break; - } - } + TableClient.ValidateAzureTableName(attribute.TableName); } - while (token != null); - - return entityArray; } private IAsyncCollector BuildFromTableAttribute(TableAttribute attribute) @@ -218,18 +185,10 @@ public async Task TryCreateAsync(BindingProviderContext context) if (bindsToEntireTable) { - IBindableTablePath path = BindableTablePath.Create(tableName); - path.ValidateContractCompatibility(context.BindingDataContract); - - IStorageTableArgumentBinding argumentBinding = _tableBindingProvider.TryCreate(parameter); - - if (argumentBinding == null) - { - throw new InvalidOperationException("Can't bind Table to type '" + parameter.ParameterType + "'."); - } - - binding = new TableBinding(parameter.Name, argumentBinding, client, path); - } + // This should have been caught by the other rule-based binders. + // We never expect this to get thrown. + throw new InvalidOperationException("Can't bind Table to type '" + parameter.ParameterType + "'."); + } else { string partitionKey = Resolve(tableAttribute.PartitionKey); @@ -365,5 +324,162 @@ public ResolvedTableAttribute(TableAttribute inner, IStorageTableClient client) internal IStorageTableClient Client { get; private set; } } + + // IStorageTable --> IQueryable + // ConverterManager's pattern matcher will figure out TElement. + private class TableToIQueryableConverter : + IAsyncConverter> + where TElement : ITableEntity, new() + { + public TableToIQueryableConverter() + { + // We're now commited to an IQueryable. Verify other constraints. + Type entityType = typeof(TElement); + + if (!TableClient.ImplementsITableEntity(entityType)) + { + throw new InvalidOperationException("IQueryable is only supported on types that implement ITableEntity."); + } + + TableClient.VerifyDefaultConstructor(entityType); + } + + public async Task> ConvertAsync(IStorageTable value, CancellationToken cancellation) + { + // If Table does not exist, treat it like have zero rows. + // This means return an non-null but empty enumerable. + // SDK doesn't do that, so we need to explicitly check. + bool exists = await value.ExistsAsync(CancellationToken.None); + + if (!exists) + { + return Enumerable.Empty().AsQueryable(); + } + else + { + return value.CreateQuery(); + } + } + } + + // Convert from T --> ITableEntity + private class ObjectToITableEntityConverter + : IConverter + { + private static readonly IConverter Converter = PocoToTableEntityConverter.Create(); + + public ObjectToITableEntityConverter() + { + // JObject case should have been claimed by another converter. + // So we can statically enforce an ITableEntity compatible contract + var t = typeof(TElement); + TableClient.VerifyContainsProperty(t, "RowKey"); + TableClient.VerifyContainsProperty(t, "PartitionKey"); + } + + public ITableEntity Convert(TElement item) + { + return Converter.Convert(item); + } + } + + // Provide some common builder rules. + private class JObjectBuilder : + IAsyncConverter, + IConverter, + IAsyncConverter, + IAsyncConverter + { + IStorageTable IConverter.Convert(TableAttribute attribute) + { + IStorageTable table = GetTable(attribute); + return table; + } + + async Task IAsyncConverter.ConvertAsync(TableAttribute attribute, CancellationToken cancellation) + { + IStorageTable table = GetTable(attribute); + await table.CreateIfNotExistsAsync(CancellationToken.None); + + var sdkTable = table.SdkObject; + return sdkTable; + } + + async Task IAsyncConverter.ConvertAsync(TableAttribute attribute, CancellationToken cancellation) + { + IStorageTable table = GetTable(attribute); + + IStorageTableOperation retrieve = table.CreateRetrieveOperation( + attribute.PartitionKey, attribute.RowKey); + TableResult result = await table.ExecuteAsync(retrieve, CancellationToken.None); + DynamicTableEntity entity = (DynamicTableEntity)result.Result; + if (entity == null) + { + return null; + } + else + { + var obj = ConvertEntityToJObject(entity); + return obj; + } + } + + // Build a JArray. + // Used as an alternative to binding to IQueryable. + async Task IAsyncConverter.ConvertAsync(TableAttribute attribute, CancellationToken cancellation) + { + var table = GetTable(attribute).SdkObject; + + string finalQuery = attribute.Filter; + if (!string.IsNullOrEmpty(attribute.PartitionKey)) + { + var partitionKeyPredicate = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, attribute.PartitionKey); + if (!string.IsNullOrEmpty(attribute.Filter)) + { + finalQuery = TableQuery.CombineFilters(attribute.Filter, TableOperators.And, partitionKeyPredicate); + } + else + { + finalQuery = partitionKeyPredicate; + } + } + + TableQuery tableQuery = new TableQuery + { + FilterString = finalQuery + }; + if (attribute.Take > 0) + { + tableQuery.TakeCount = attribute.Take; + } + int countRemaining = attribute.Take; + + JArray entityArray = new JArray(); + TableContinuationToken token = null; + + do + { + var segment = await table.ExecuteQuerySegmentedAsync(tableQuery, token); + var entities = segment.Results; + + token = segment.ContinuationToken; + + foreach (var entity in entities) + { + countRemaining--; + entityArray.Add(ConvertEntityToJObject(entity)); + + if (countRemaining == 0) + { + token = null; + break; + } + } + } + while (token != null); + + return entityArray; + } + } } } diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/TableBinding.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/TableBinding.cs deleted file mode 100644 index 154bfaa7d..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/TableBinding.cs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.IO; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Converters; -using Microsoft.Azure.WebJobs.Host.Protocols; -using Microsoft.Azure.WebJobs.Host.Storage.Table; -using Microsoft.WindowsAzure.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal class TableBinding : IBinding - { - private readonly string _parameterName; - private readonly IStorageTableArgumentBinding _argumentBinding; - private readonly IStorageTableClient _client; - private readonly string _accountName; - private readonly IBindableTablePath _path; - private readonly IObjectToTypeConverter _converter; - - public TableBinding(string parameterName, IStorageTableArgumentBinding argumentBinding, IStorageTableClient client, IBindableTablePath path) - { - _parameterName = parameterName; - _argumentBinding = argumentBinding; - _client = client; - _accountName = TableClient.GetAccountName(client); - _path = path; - _converter = CreateConverter(client, path); - } - - public bool FromAttribute - { - get { return true; } - } - - public string TableName - { - get { return _path.TableNamePattern; } - } - - private FileAccess Access - { - get - { - return _argumentBinding.Access; - } - } - - private static IObjectToTypeConverter CreateConverter(IStorageTableClient client, IBindableTablePath path) - { - return new CompositeObjectToTypeConverter( - new OutputConverter(new IdentityConverter()), - new OutputConverter(new CloudTableToStorageTableConverter()), - new OutputConverter(new StringToStorageTableConverter(client, path))); - } - - public Task BindAsync(BindingContext context) - { - if (context == null) - { - throw new ArgumentNullException("context"); - } - - string boundTableName = _path.Bind(context.BindingData); - IStorageTable table = _client.GetTableReference(boundTableName); - - return BindTableAsync(table, context.ValueContext); - } - - private Task BindTableAsync(IStorageTable value, ValueBindingContext context) - { - return _argumentBinding.BindAsync(value, context); - } - - public Task BindAsync(object value, ValueBindingContext context) - { - IStorageTable table = null; - - if (!_converter.TryConvert(value, out table)) - { - throw new InvalidOperationException("Unable to convert value to IStorageTable."); - } - - return BindTableAsync(table, context); - } - - public ParameterDescriptor ToParameterDescriptor() - { - return new TableParameterDescriptor - { - Name = _parameterName, - AccountName = _accountName, - TableName = _path.TableNamePattern, - Access = Access - }; - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/TableEntityCollectorBinder.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/TableEntityCollectorBinder.cs index 211cf8494..b275e89ac 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/TableEntityCollectorBinder.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Tables/TableEntityCollectorBinder.cs @@ -43,9 +43,9 @@ public IWatcher Watcher } } - public object GetValue() + public Task GetValueAsync() { - return _tableWriter; + return Task.FromResult(_tableWriter); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/TableEntityValueBinder.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/TableEntityValueBinder.cs index d41ab970b..3e1ffb6ae 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/TableEntityValueBinder.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Tables/TableEntityValueBinder.cs @@ -47,9 +47,9 @@ public bool HasChanged } } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public Task SetValueAsync(object value, CancellationToken cancellationToken) diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/TableFilterFormatter.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/TableFilterFormatter.cs new file mode 100644 index 000000000..080fcd601 --- /dev/null +++ b/src/Microsoft.Azure.WebJobs.Host/Tables/TableFilterFormatter.cs @@ -0,0 +1,111 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Azure.WebJobs.Host.Bindings; +using Microsoft.Azure.WebJobs.Host.Bindings.Path; + +namespace Microsoft.Azure.WebJobs.Host.Tables +{ + internal static class TableFilterFormatter + { + public static string Format(BindingTemplate template, IReadOnlyDictionary bindingData) + { + if (!template.ParameterNames.Any()) + { + return template.Pattern; + } + + if (template.ParameterNames.Count() == 1) + { + // Special case where the entire filter expression + // is a single parameter. We let this go through as is + string parameterName = template.ParameterNames.Single(); + if (template.Pattern == $"{{{parameterName}}}") + { + return template.Bind(bindingData); + } + } + + // each distinct parameter can occur one or more times in the template + // so group by parameter name + var parameterGroups = template.ParameterNames.GroupBy(p => p); + + // for each parameter, classify it as a string literal or other + // and perform value validation + var convertedBindingData = BindingDataPathHelper.ConvertParameters(bindingData); + foreach (var parameterGroup in parameterGroups) + { + // perform any OData specific formatting on the values + string parameterName = parameterGroup.Key; + object originalValue; + if (bindingData.TryGetValue(parameterName, out originalValue)) + { + if (originalValue is DateTime) + { + // OData DateTime literals should be ISO 8601 formatted (e.g. 2009-03-18T04:25:03Z) + convertedBindingData[parameterName] = ((DateTime)originalValue).ToUniversalTime().ToString("o"); + } + else if (originalValue is DateTimeOffset) + { + convertedBindingData[parameterName] = ((DateTimeOffset)originalValue).UtcDateTime.ToString("o"); + } + } + + // to classify as a string literal, ALL occurrences in the template + // must be string literals (e.g. of the form '{p}') + // note that this will also capture OData expressions of the form + // datetime'{p}', guid'{p}', X'{p}' which is fine, because single quotes + // aren't valid for those values anyways. + bool isStringLiteral = true; + string stringParameterFormat = $"'{{{parameterName}}}'"; + int count = 0, idx = 0; + while (idx >= 0 && idx < template.Pattern.Length && count++ < parameterGroup.Count()) + { + idx = template.Pattern.IndexOf(stringParameterFormat, idx, StringComparison.OrdinalIgnoreCase); + if (idx < 0) + { + isStringLiteral = false; + break; + } + idx++; + } + + // validate and format the value based on its classification + string value = null; + if (convertedBindingData.TryGetValue(parameterName, out value)) + { + if (isStringLiteral) + { + convertedBindingData[parameterName] = value.Replace("'", "''"); + } + else if (!TryValidateNonStringLiteral(value)) + { + throw new InvalidOperationException($"An invalid parameter value was specified for filter parameter '{parameterName}'."); + } + } + } + + return template.Bind(convertedBindingData); + } + + internal static bool TryValidateNonStringLiteral(string value) + { + // value must be one of the odata supported non string literal types: + // bool, int, long, double + bool boolValue; + long longValue; + double doubleValue; + if (bool.TryParse(value, out boolValue) || + long.TryParse(value, out longValue) || + double.TryParse(value, out doubleValue)) + { + return true; + } + + return false; + } + } +} diff --git a/src/Microsoft.Azure.WebJobs.Host/Tables/TableValueProvider.cs b/src/Microsoft.Azure.WebJobs.Host/Tables/TableValueProvider.cs deleted file mode 100644 index d6f6b240b..000000000 --- a/src/Microsoft.Azure.WebJobs.Host/Tables/TableValueProvider.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Storage.Table; - -namespace Microsoft.Azure.WebJobs.Host.Tables -{ - internal sealed class TableValueProvider : IValueProvider - { - private readonly IStorageTable _table; - private readonly object _value; - private readonly Type _valueType; - - public TableValueProvider(IStorageTable table, object value, Type valueType) - { - if (value != null && !valueType.IsAssignableFrom(value.GetType())) - { - throw new InvalidOperationException("value is not of the correct type."); - } - - _table = table; - _value = value; - _valueType = valueType; - } - - public Type Type - { - get { return _valueType; } - } - - public object GetValue() - { - return _value; - } - - public string ToInvokeString() - { - return _table.Name; - } - } -} diff --git a/src/Microsoft.Azure.WebJobs.Host/Triggers/ITriggerBindingStrategy.cs b/src/Microsoft.Azure.WebJobs.Host/Triggers/ITriggerBindingStrategy.cs index c8151cc6b..50be294e7 100644 --- a/src/Microsoft.Azure.WebJobs.Host/Triggers/ITriggerBindingStrategy.cs +++ b/src/Microsoft.Azure.WebJobs.Host/Triggers/ITriggerBindingStrategy.cs @@ -19,6 +19,7 @@ namespace Microsoft.Azure.WebJobs.Host.Triggers /// /// The native message type. For Azure Queues, this would be CloudQueueMessage. /// The type of the trigger object that the listener returns. This could represent a single message or a batch of messages. + [Obsolete("Not ready for public consumption.")] public interface ITriggerBindingStrategy { /// diff --git a/src/Microsoft.Azure.WebJobs.Host/TypeUtility.cs b/src/Microsoft.Azure.WebJobs.Host/TypeUtility.cs index ee4816b57..bd30b4e9c 100644 --- a/src/Microsoft.Azure.WebJobs.Host/TypeUtility.cs +++ b/src/Microsoft.Azure.WebJobs.Host/TypeUtility.cs @@ -4,6 +4,8 @@ using System; using System.Globalization; using System.Reflection; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; using Newtonsoft.Json.Linq; namespace Microsoft.Azure.WebJobs.Host @@ -32,6 +34,22 @@ internal static bool IsJObject(Type type) return type == typeof(JObject); } + // Task --> T + // Task --> void + // T --> T + internal static Type UnwrapTaskType(Type type) + { + if (type == typeof(Task)) + { + return typeof(void); + } + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Task<>)) + { + return type.GetGenericArguments()[0]; + } + return type; + } + /// /// Walk from the parameter up to the containing type, looking for an instance /// of the specified attribute type, returning it if found. @@ -74,5 +92,29 @@ internal static T GetHierarchicalAttributeOrNull(MethodInfo method) where T : return null; } + + public static bool IsAsync(MethodInfo methodInfo) + { + if (methodInfo == null) + { + throw new ArgumentNullException(nameof(methodInfo)); + } + + var stateMachineAttribute = methodInfo.GetCustomAttribute(); + if (stateMachineAttribute != null) + { + var stateMachineType = stateMachineAttribute.StateMachineType; + if (stateMachineType != null) + { + return stateMachineType.GetCustomAttribute() != null; + } + } + return false; + } + + public static bool IsAsyncVoid(MethodInfo methodInfo) + { + return IsAsync(methodInfo) && (methodInfo.ReturnType == typeof(void)); + } } } diff --git a/src/Microsoft.Azure.WebJobs.Host/WebJobs.Host.csproj b/src/Microsoft.Azure.WebJobs.Host/WebJobs.Host.csproj index a2d585182..7f261fe83 100644 --- a/src/Microsoft.Azure.WebJobs.Host/WebJobs.Host.csproj +++ b/src/Microsoft.Azure.WebJobs.Host/WebJobs.Host.csproj @@ -28,7 +28,7 @@ prompt 4 bin\Debug\Microsoft.Azure.WebJobs.Host.xml - 5 + default true true true @@ -41,7 +41,7 @@ prompt 4 bin\Release\Microsoft.Azure.WebJobs.Host.xml - 5 + default true @@ -58,15 +58,15 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True @@ -88,7 +88,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True @@ -373,21 +373,27 @@ Storage\Table\TypeEntityResolver.cs + - + + + + - - - + + + + + @@ -397,6 +403,7 @@ + @@ -419,6 +426,9 @@ + + + @@ -665,12 +675,7 @@ - - - - - @@ -711,10 +716,7 @@ - - - @@ -865,27 +867,17 @@ - - - - - - - - - - @@ -972,9 +964,9 @@ This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + - + function + Dictionary _activeFuncs = new Dictionary(); + HashSet _completedFunctions = new HashSet(); + + public Task AddAsync(FunctionInstanceLogItem item, CancellationToken cancellationToken = default(CancellationToken)) { + if (item == null) + { + throw new ArgumentNullException("item"); + } item.Validate(); item.FunctionId = FunctionId.Build(this._hostName, item.FunctionName); + // Both Start and Completed log here. Completed will overwrite a Start entry. + lock (_lock) { - lock(_lock) - { - StartBackgroundFlusher(); - if (_container == null) - { - _container = new ContainerActiveLogger(_machineName, _logTableProvider); - } - if (_instanceLogger == null) - { - int size = GetContainerSize(); - _instanceLogger = new CloudTableInstanceCountLogger(_machineName, _logTableProvider, size); - } - } - if (item.IsCompleted()) + _activeFuncs[item.FunctionInstanceId] = item; + } + + lock (_lock) + { + StartBackgroundFlusher(); + if (_container == null) { - _container.Decrement(item.FunctionInstanceId); - _instanceLogger.Decrement(item.FunctionInstanceId); + _container = new ContainerActiveLogger(_machineName, _logTableProvider); } - else + if (_instanceLogger == null) { - _container.Increment(item.FunctionInstanceId); - _instanceLogger.Increment(item.FunctionInstanceId); + int size = GetContainerSize(); + _instanceLogger = new CloudTableInstanceCountLogger(_machineName, _logTableProvider, size); } } + if (item.IsCompleted()) + { + _container.Decrement(item.FunctionInstanceId); + _instanceLogger.Decrement(item.FunctionInstanceId); + + _completedFunctions.Add(item.FunctionInstanceId); + } + else + { + _container.Increment(item.FunctionInstanceId); + _instanceLogger.Increment(item.FunctionInstanceId); + } + lock (_lock) { if (_seenFunctions.Add(item.FunctionName)) @@ -172,13 +186,6 @@ private static int GetContainerSize() _funcDefs.Add(FunctionDefinitionEntity.New(item.FunctionId, item.FunctionName)); } } - - // Both Start and Completed log here. Completed will overwrite a Start entry. - lock (_lock) - { - _instances.Add(InstanceTableEntity.New(item)); - _recents.Add(RecentPerFuncEntity.New(_machineName, item)); - } if (item.IsCompleted()) { @@ -200,13 +207,11 @@ private static int GetContainerSize() Increment(item, existingEntity); } - } + } } - // Flush every 100 items, maximize with tables. - Task t1 = FlushIntancesAsync(false); - Task t2 = FlushTimelineAggregateAsync(); - await Task.WhenAll(t1, t2); + // Results will get written on a background thread + return Task.FromResult(0); } // Could flush on a timer. @@ -238,40 +243,58 @@ private async Task FlushTimelineAggregateAsync(bool always = false) } } + + private FunctionInstanceLogItem[] Update() + { + FunctionInstanceLogItem[] items; + lock (_lock) + { + items = _activeFuncs.Values.ToArray(); + } + foreach (var item in items) + { + item.Refresh(_flushInterval); + } + return items; + } + // Could flush on a timer. - private async Task FlushIntancesAsync(bool always) + private async Task FlushIntancesAsync() { - InstanceTableEntity[] instances; - RecentPerFuncEntity[] recentInvokes; + // Before writing, give items a chance to refresh + var itemsSnapshot = Update(); + + // Write entries + var instances = Array.ConvertAll(itemsSnapshot, item => InstanceTableEntity.New(item)); + var recentInvokes = Array.ConvertAll(itemsSnapshot, item => RecentPerFuncEntity.New(_machineName, item)); + FunctionDefinitionEntity[] functionDefinitions; lock (_lock) { - if (!always) - { - if (_instances.Count < 90) - { - return; - } - } - - instances = _instances.ToArray(); - recentInvokes = _recents.ToArray(); functionDefinitions = _funcDefs.ToArray(); - _instances.Clear(); - _recents.Clear(); _funcDefs.Clear(); } Task t1 = WriteBatchAsync(instances); Task t2 = WriteBatchAsync(recentInvokes); Task t3 = WriteBatchAsync(functionDefinitions); await Task.WhenAll(t1, t2, t3); + + // After we write to table, remove all completed functions. + lock (_lock) + { + foreach (var completedId in _completedFunctions) + { + _activeFuncs.Remove(completedId); + } + _completedFunctions.Clear(); + } } private async Task FlushCoreAsync() { await FlushTimelineAggregateAsync(true); - await FlushIntancesAsync(true); + await FlushIntancesAsync(); if (_container != null) { @@ -308,73 +331,23 @@ private static void Increment(FunctionInstanceLogItem item, TimelineAggregateEnt // Limit of 100 per batch. // Parallel uploads. - private async Task WriteBatchAsync(IEnumerable e1) where T : TableEntity, IEntityWithEpoch - { - HashSet rowKeys = new HashSet(); - - int batchSize = 90; - - Dictionary batches = new Dictionary(); - Dictionary tables = new Dictionary(); - - List t = new List(); - - foreach (var e in e1) - { - if (!rowKeys.Add(e.RowKey)) - { - // Already present - } - - var epoch = e.GetEpoch(); - var instanceTable = this._logTableProvider.GetTableForDateTime(epoch); - TableBatchOperation batch; - if (!batches.TryGetValue(instanceTable.Name, out batch)) - { - tables[instanceTable.Name] = instanceTable; - batch = new TableBatchOperation(); - batches[instanceTable.Name] = batch; - } - - batch.InsertOrReplace(e); - if (batch.Count >= batchSize) - { - Task tUpload = instanceTable.SafeExecuteAsync(batch); - t.Add(tUpload); - - batch = new TableBatchOperation(); - batches[instanceTable.Name] = batch; - } - } - - foreach (var kv in batches) - { - var tableName = kv.Key; - var instanceTable = tables[tableName]; - var batch = kv.Value; - if (batch.Count > 0) - { - Task tUpload = instanceTable.SafeExecuteAsync(batch); - t.Add(tUpload); - } - } - - - await Task.WhenAll(t); + private Task WriteBatchAsync(IEnumerable e1) where T : TableEntity, IEntityWithEpoch + { + return this._logTableProvider.WriteBatchAsync(e1); } // Collection where adding in the same RowKey replaces a previous entry with that key. // This is single-threaded. Caller must lock. // All entities in this collection must have unique row keys across the partition and tables. - private class EntityCollection : IEnumerable where T : TableEntity + private class EntityCollection : IEnumerable where T : TableEntity { // Ordering doesn't matter since azure tables will order them for us. private Dictionary _map = new Dictionary(); public void Add(T entry) - { + { string row = entry.RowKey; - _map[row] = entry; + _map[row] = entry; } public int Count @@ -415,5 +388,5 @@ IEnumerator IEnumerable.GetEnumerator() return _map.Values.GetEnumerator(); } } - } + } } diff --git a/src/Microsoft.Azure.WebJobs.Logging/Internal/TimeBucket.cs b/src/Microsoft.Azure.WebJobs.Logging/Internal/TimeBucket.cs index c63abd445..8c6e0b2d3 100644 --- a/src/Microsoft.Azure.WebJobs.Logging/Internal/TimeBucket.cs +++ b/src/Microsoft.Azure.WebJobs.Logging/Internal/TimeBucket.cs @@ -11,7 +11,7 @@ namespace Microsoft.Azure.WebJobs.Logging // Use minutes since a baseline. internal static class TimeBucket { - static DateTime _baselineTime = new DateTime(2000, 1, 1); + static DateTime _baselineTime = new DateTime(2000, 1, 1, 0,0,0, DateTimeKind.Utc); public static DateTime ConvertToDateTime(long bucket) { diff --git a/src/Microsoft.Azure.WebJobs.Logging/Internal/Utility.cs b/src/Microsoft.Azure.WebJobs.Logging/Internal/Utility.cs index aaace3029..b363f9b7b 100644 --- a/src/Microsoft.Azure.WebJobs.Logging/Internal/Utility.cs +++ b/src/Microsoft.Azure.WebJobs.Logging/Internal/Utility.cs @@ -198,5 +198,63 @@ public static async Task SafeExecuteAsync(this CloudTable table, Ta await table.SafeCreateAsync(); return await table.ExecuteAsync(operation); } + + // Limit of 100 per batch. + // Parallel uploads. + public static async Task WriteBatchAsync(this ILogTableProvider logTableProvider, IEnumerable e1) where T : TableEntity, IEntityWithEpoch + { + HashSet rowKeys = new HashSet(); + + int batchSize = 90; + + // Batches must be within a single table partition, so Key is "tableName + ParitionKey". + var batches = new Dictionary>(); + + List t = new List(); + + foreach (var e in e1) + { + if (!rowKeys.Add(e.RowKey)) + { + // Already present + } + + var epoch = e.GetEpoch(); + var instanceTable = logTableProvider.GetTableForDateTime(epoch); + + string key = instanceTable.Name + "/" + e.PartitionKey; + + Tuple tuple; + if (!batches.TryGetValue(key, out tuple)) + { + tuple = Tuple.Create(instanceTable, new TableBatchOperation()); + batches[key] = tuple; + } + TableBatchOperation batch = tuple.Item2; + + batch.InsertOrMerge(e); + if (batch.Count >= batchSize) + { + Task tUpload = instanceTable.SafeExecuteAsync(batch); + t.Add(tUpload); + + batches.Remove(key); + } + } + + // Flush remaining + foreach (var tuple in batches.Values) + { + var instanceTable = tuple.Item1; + var batch = tuple.Item2; + if (batch.Count > 0) + { + Task tUpload = instanceTable.SafeExecuteAsync(batch); + t.Add(tUpload); + } + } + + await Task.WhenAll(t); + } } } diff --git a/src/Microsoft.Azure.WebJobs.Logging/WebJobs.Logging.csproj b/src/Microsoft.Azure.WebJobs.Logging/WebJobs.Logging.csproj index cb4c9aaf5..c6bba8116 100644 --- a/src/Microsoft.Azure.WebJobs.Logging/WebJobs.Logging.csproj +++ b/src/Microsoft.Azure.WebJobs.Logging/WebJobs.Logging.csproj @@ -26,7 +26,7 @@ prompt 4 bin\Debug\Microsoft.Azure.WebJobs.Logging.XML - 5 + default true @@ -37,7 +37,7 @@ prompt 4 bin\Release\Microsoft.Azure.WebJobs.Logging.XML - 5 + default true @@ -54,15 +54,15 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True @@ -80,7 +80,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True diff --git a/src/Microsoft.Azure.WebJobs.Logging/packages.config b/src/Microsoft.Azure.WebJobs.Logging/packages.config index 7b29543f3..451776d3a 100644 --- a/src/Microsoft.Azure.WebJobs.Logging/packages.config +++ b/src/Microsoft.Azure.WebJobs.Logging/packages.config @@ -1,11 +1,15 @@  - - - + + + - + + + + + \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.Protocols/WebJobs.Protocols.csproj b/src/Microsoft.Azure.WebJobs.Protocols/WebJobs.Protocols.csproj index 413d19dc2..a87aca45d 100644 --- a/src/Microsoft.Azure.WebJobs.Protocols/WebJobs.Protocols.csproj +++ b/src/Microsoft.Azure.WebJobs.Protocols/WebJobs.Protocols.csproj @@ -26,7 +26,7 @@ 4 bin\Debug\Microsoft.Azure.WebJobs.Protocols.xml false - 5 + default AnyCPU @@ -38,7 +38,7 @@ 4 bin\Release\Microsoft.Azure.WebJobs.Protocols.xml false - 5 + default true @@ -133,15 +133,15 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True @@ -160,7 +160,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True diff --git a/src/Microsoft.Azure.WebJobs.Protocols/packages.config b/src/Microsoft.Azure.WebJobs.Protocols/packages.config index 83fd8f6f8..93cf62eac 100644 --- a/src/Microsoft.Azure.WebJobs.Protocols/packages.config +++ b/src/Microsoft.Azure.WebJobs.Protocols/packages.config @@ -1,12 +1,16 @@  - - - + + + - + + + + + \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus.NuGet/WebJobs.ServiceBus.nuspec b/src/Microsoft.Azure.WebJobs.ServiceBus.NuGet/WebJobs.ServiceBus.nuspec index 1ee3953bd..fa3076c22 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus.NuGet/WebJobs.ServiceBus.nuspec +++ b/src/Microsoft.Azure.WebJobs.ServiceBus.NuGet/WebJobs.ServiceBus.nuspec @@ -16,8 +16,8 @@ - - + + \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/BrokeredMessageArgumentBinding.cs b/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/BrokeredMessageArgumentBinding.cs index d69cb9408..062ebe52d 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/BrokeredMessageArgumentBinding.cs +++ b/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/BrokeredMessageArgumentBinding.cs @@ -49,9 +49,9 @@ public Type Type get { return typeof(BrokeredMessage); } } - public object GetValue() + public Task GetValueAsync() { - return null; + return Task.FromResult(null); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/CollectorValueProvider.cs b/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/CollectorValueProvider.cs index c83e69262..9d6beee23 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/CollectorValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/CollectorValueProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; namespace Microsoft.Azure.WebJobs.ServiceBus.Bindings @@ -29,9 +30,9 @@ public Type Type get { return _valueType; } } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/ConverterValueBinder.cs b/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/ConverterValueBinder.cs index 22a4709d3..da270c987 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/ConverterValueBinder.cs +++ b/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/ConverterValueBinder.cs @@ -35,9 +35,9 @@ public Type Type get { return typeof(TInput); } } - public object GetValue() + public Task GetValueAsync() { - return default(TInput); + return Task.FromResult(default(TInput)); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/NonNullConverterValueBinder.cs b/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/NonNullConverterValueBinder.cs index c519eb345..3b4dffd72 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/NonNullConverterValueBinder.cs +++ b/src/Microsoft.Azure.WebJobs.ServiceBus/Bindings/NonNullConverterValueBinder.cs @@ -36,9 +36,9 @@ public Type Type get { return typeof(TInput); } } - public object GetValue() + public Task GetValueAsync() { - return null; + return Task.FromResult(null); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus/EventHubs/EventHubConfiguration.cs b/src/Microsoft.Azure.WebJobs.ServiceBus/EventHubs/EventHubConfiguration.cs index 86d2c57c2..33a58ca5d 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus/EventHubs/EventHubConfiguration.cs +++ b/src/Microsoft.Azure.WebJobs.ServiceBus/EventHubs/EventHubConfiguration.cs @@ -60,7 +60,8 @@ public EventHubConfiguration( if (options == null) { options = EventProcessorOptions.DefaultOptions; - options.MaxBatchSize = 1000; + options.MaxBatchSize = 64; + options.PrefetchCount = options.MaxBatchSize * 4; } _partitionOptions = partitionOptions; @@ -385,7 +386,7 @@ void IExtensionConfigProvider.Initialize(ExtensionConfigContext context) extensions.RegisterExtension(triggerBindingProvider); // register our binding provider - var ruleOutput = bf.BindToAsyncCollector(BuildFromAttribute); + var ruleOutput = bf.BindToCollector(BuildFromAttribute); extensions.RegisterBindingRules(ruleOutput); } diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageToByteArrayConverter.cs b/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageToByteArrayConverter.cs index 3c4a19f31..f8e0336e4 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageToByteArrayConverter.cs +++ b/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageToByteArrayConverter.cs @@ -1,7 +1,9 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; using System.IO; +using System.Runtime.Serialization; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Converters; @@ -32,8 +34,20 @@ public async Task ConvertAsync(BrokeredMessage input, CancellationToken } else { - return input.GetBody(); + try + { + return input.GetBody(); + } + catch (SerializationException exception) + { + // If we fail to deserialize here, it is because the message body was serialized using something other than the default + // DataContractSerializer with a binary XmlDictionaryWriter. + string contentType = input.ContentType ?? "null"; + string msg = $"The BrokeredMessage with ContentType '{contentType}' failed to deserialize to a byte[] with the message: '{exception.Message}'"; + + throw new InvalidOperationException(msg, exception); + } } } } -} +} \ No newline at end of file diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageToStringConverter.cs b/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageToStringConverter.cs index dc193db44..bcf893c74 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageToStringConverter.cs +++ b/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageToStringConverter.cs @@ -2,7 +2,9 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Globalization; using System.IO; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Converters; @@ -12,44 +14,60 @@ namespace Microsoft.Azure.WebJobs.ServiceBus.Triggers { internal class BrokeredMessageToStringConverter : IAsyncConverter { - public Task ConvertAsync(BrokeredMessage input, CancellationToken cancellationToken) + public async Task ConvertAsync(BrokeredMessage input, CancellationToken cancellationToken) { if (input == null) { throw new ArgumentNullException("input"); } + Stream stream = input.GetBody(); + if (stream == null) + { + return null; + } - if (input.ContentType == ContentTypes.TextPlain || - input.ContentType == ContentTypes.ApplicationOctetStream || - input.ContentType == ContentTypes.ApplicationJson) + TextReader reader = new StreamReader(stream, StrictEncodings.Utf8); + try { - Stream stream = input.GetBody(); - if (stream == null) + cancellationToken.ThrowIfCancellationRequested(); + try { - return Task.FromResult(null); + return await reader.ReadToEndAsync(); } + catch (DecoderFallbackException) + { + // we'll try again below + } + + // We may get here if the message is a string yet was DataContract-serialized when created. We'll + // try to deserialize it here using GetBody(). This may fail as well, in which case we'll + // provide a decent error. + // Create a clone as you cannot call GetBody twice on the same BrokeredMessage. + BrokeredMessage clonedMessage = input.Clone(); try { - using (TextReader reader = new StreamReader(stream, StrictEncodings.Utf8)) - { - stream = null; - cancellationToken.ThrowIfCancellationRequested(); - return reader.ReadToEndAsync(); - } + return clonedMessage.GetBody(); } - finally + catch (Exception exception) { - if (stream != null) - { - stream.Dispose(); - } + string contentType = input.ContentType ?? "null"; + string msg = string.Format(CultureInfo.InvariantCulture, "The BrokeredMessage with ContentType '{0}' failed to deserialize to a string with the message: '{1}'", + contentType, exception.Message); + + throw new InvalidOperationException(msg, exception); } } - else + finally { - string contents = input.GetBody(); - return Task.FromResult(contents); + if (stream != null) + { + stream.Dispose(); + } + if (reader != null) + { + reader.Dispose(); + } } } } diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageValueProvider.cs b/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageValueProvider.cs index 7b900b252..bd8cbdfb2 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageValueProvider.cs +++ b/src/Microsoft.Azure.WebJobs.ServiceBus/Triggers/BrokeredMessageValueProvider.cs @@ -35,9 +35,9 @@ public Type Type get { return _valueType; } } - public object GetValue() + public Task GetValueAsync() { - return _value; + return Task.FromResult(_value); } public string ToInvokeString() diff --git a/src/Microsoft.Azure.WebJobs.ServiceBus/WebJobs.ServiceBus.csproj b/src/Microsoft.Azure.WebJobs.ServiceBus/WebJobs.ServiceBus.csproj index 41716cf93..e65cc58d3 100644 --- a/src/Microsoft.Azure.WebJobs.ServiceBus/WebJobs.ServiceBus.csproj +++ b/src/Microsoft.Azure.WebJobs.ServiceBus/WebJobs.ServiceBus.csproj @@ -29,7 +29,7 @@ 4 bin\Debug\Microsoft.Azure.WebJobs.ServiceBus.xml false - 5 + default pdbonly @@ -40,7 +40,7 @@ 4 bin\Release\Microsoft.Azure.WebJobs.ServiceBus.xml false - 5 + default true @@ -57,23 +57,23 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True - ..\..\packages\WindowsAzure.ServiceBus.3.4.1\lib\net45-full\Microsoft.ServiceBus.dll + ..\..\packages\WindowsAzure.ServiceBus.3.4.5\lib\net45-full\Microsoft.ServiceBus.dll True - ..\..\packages\Microsoft.Azure.ServiceBus.EventProcessorHost.2.2.6\lib\net45-full\Microsoft.ServiceBus.Messaging.EventProcessorHost.dll + ..\..\packages\Microsoft.Azure.ServiceBus.EventProcessorHost.2.2.10\lib\net45-full\Microsoft.ServiceBus.Messaging.EventProcessorHost.dll True @@ -93,7 +93,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True @@ -115,15 +115,9 @@ Converters\ConversionResult.cs - - Converters\IAsyncConverter.cs - Converters\IAsyncObjectToTypeConverter.cs - - Converters\IConverter.cs - Converters\IdentityConverter.cs @@ -243,9 +237,9 @@ This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + - + + \ No newline at end of file diff --git a/test/Dashboard.UnitTests/Dashboard.UnitTests.csproj b/test/Dashboard.UnitTests/Dashboard.UnitTests.csproj index d6b828189..cde5df2b7 100644 --- a/test/Dashboard.UnitTests/Dashboard.UnitTests.csproj +++ b/test/Dashboard.UnitTests/Dashboard.UnitTests.csproj @@ -23,7 +23,7 @@ DEBUG;TRACE prompt 4 - 5 + default pdbonly @@ -32,7 +32,7 @@ TRACE prompt 4 - 5 + default true @@ -84,15 +84,15 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True @@ -107,8 +107,8 @@ ..\..\packages\WindowsAzure.Storage.7.2.1\lib\net40\Microsoft.WindowsAzure.Storage.dll True - - ..\..\packages\Moq.4.5.23\lib\net45\Moq.dll + + ..\..\packages\Moq.4.5.30\lib\net45\Moq.dll True @@ -124,7 +124,7 @@ True - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True diff --git a/test/Dashboard.UnitTests/RestApiModels/Models.cs b/test/Dashboard.UnitTests/RestApiModels/Models.cs index 4c400e102..07d2a9acd 100644 --- a/test/Dashboard.UnitTests/RestApiModels/Models.cs +++ b/test/Dashboard.UnitTests/RestApiModels/Models.cs @@ -59,7 +59,7 @@ public class TriggerReasonViewModel public string childGuid { get; set; } } - public class InvocationLogViewModel + public class InvocationLogViewModel { public string id { get; set; } public string functionId { get; set; } @@ -67,12 +67,29 @@ public class InvocationLogViewModel public string functionFullName { get; set; } public string functionDisplayTitle { get; set; } public string status { get; set; } + + // Semantics of this change depending on status. + // If Running, it's STartTime. + // If completed, it's end time. public string whenUtc { get; set; } - public double duration { get; set; } + // Null if not completed yet. + public double? duration { get; set; } public string exceptionMessage { get; set; } public string exceptionType { get; set; } } + class TimelineResponseEntry + { + public string Start { get; set; } // DateTime + public int TotalPass { get; set; } + public int TotalFail { get; set; } + public int TotalRun { get; set; } + } + + public class VersionResponse + { + public string Version { get; set; } // version string + } } \ No newline at end of file diff --git a/test/Dashboard.UnitTests/RestApiTests.cs b/test/Dashboard.UnitTests/RestApiTests.cs index 05818bd29..23935ca5b 100644 --- a/test/Dashboard.UnitTests/RestApiTests.cs +++ b/test/Dashboard.UnitTests/RestApiTests.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Net; using System.Net.Http; +using System.Reflection; using System.Threading.Tasks; using System.Web.Http; using Dashboard.UnitTests.RestProtocol; @@ -33,6 +34,33 @@ public RestApiTests(Fixture fixture) _endpoint = fixture.Endpoint; } + + // test the /verison endpoint. + [Fact] + public async Task GetVersion() + { + var response = await _client.GetJsonAsync(_endpoint + "/api/version"); + + var assembly = typeof(WebApiConfig).Assembly; + AssemblyFileVersionAttribute fileVersionAttr = assembly.GetCustomAttribute(); + + Assert.Equal(response.Version, fileVersionAttr.Version); + } + + [Fact] + public async Task GetDefinitionSkipStats() + { + var response = await _client.GetJsonAsync(_endpoint + "/api/functions/definitions?limit=100&skipstats=true"); + + // Testing against specific data that we added. + Assert.Equal(null, response.ContinuationToken); + var x = response.Entries.ToArray(); + Assert.Equal(1, x.Length); + Assert.Equal("alpha", x[0].functionName); + Assert.Equal(0, x[0].successCount); // Skipped + Assert.Equal(0, x[0].failedCount); + } + [Fact] public async Task GetDefinition() { @@ -56,6 +84,40 @@ public async Task GetDefinitionSingleHost() var x = response.Entries.ToArray(); Assert.Equal(0, x.Length); } + + [Fact] + public async Task GetTimelineInvocations() + { + // Lookup functions by name + string uri = _endpoint + "/api/functions/invocations/" + FunctionId.Build(Fixture.HostName, "alpha") + + "/timeline?limit=11&start=2001-01-01"; + var response = await _client.GetJsonAsync(uri); + + // This only includes completed / failed functions, not NeverFinished/Running. + // Important that DateTimes include the 'Z' suffix, meaning UTC timezone. + Assert.Equal(2, response.Length); + Assert.Equal("2010-03-06T18:10:00Z", response[0].Start); + Assert.Equal(1, response[0].TotalFail); + Assert.Equal(0, response[0].TotalPass); + Assert.Equal(1, response[0].TotalRun); + + Assert.Equal("2010-03-06T18:11:00Z", response[1].Start); + Assert.Equal(0, response[1].TotalFail); + Assert.Equal(1, response[1].TotalPass); + Assert.Equal(1, response[1].TotalRun); + } + + public async Task GetTimelineEmptyInvocations() + { + // Look in timeline range where there's no functions invocations. + // This verifies the time range is getting parsed by the webapi and passed through. + string uri = _endpoint + "/api/functions/invocations/" + FunctionId.Build(Fixture.HostName, "alpha") + + "/timeline?limit=11&start=2005-01-01"; + var response = await _client.GetJsonAsync(uri); + + Assert.Equal(0, response.Length); + } + [Fact] public async Task GetInvocations() @@ -64,33 +126,53 @@ public async Task GetInvocations() string uri = _endpoint + "/api/functions/definitions/" + FunctionId.Build(Fixture.HostName, "alpha") + "/invocations?limit=11"; var response = await _client.GetJsonAsync>(uri); - var item = _fixture.Data[0]; + Assert.Equal(_fixture.ExpectedItems.Count, response.entries.Length); + + for (int i = 0; i < response.entries.Length; i++) + { + var expectedItem = _fixture.ExpectedItems[i]; + var actualItem = response.entries[i]; + + AssertEqual(expectedItem, actualItem); - var x = response.entries.ToArray(); - Assert.Equal(item.FunctionInstanceId.ToString(), x[0].id); - Assert.Equal("2010-03-06T18:13:20Z", x[0].whenUtc); - Assert.Equal(120000.0, x[0].duration); - Assert.Equal("CompletedSuccess", x[0].status); - Assert.Equal("alpha", x[0].functionDisplayTitle); + Assert.Equal("alpha", actualItem.functionDisplayTitle); + } } [Fact] public async Task GetSpecificInvocation() { - var item = _fixture.Data[0]; + foreach (var expectedItem in _fixture.ExpectedItems) + { + var url = _endpoint + "/api/functions/invocations/" + expectedItem.id; + var response = await _client.GetJsonAsync(url); + + var actualItem = response.Invocation; + AssertEqual(expectedItem, actualItem); - // Lookup specific invocation + Assert.Equal("alpha ()", response.Invocation.functionDisplayTitle); - var url = _endpoint + "/api/functions/invocations/" + item.FunctionInstanceId.ToString(); - var response2 = await _client.GetJsonAsync(url); + Assert.Equal(actualItem.id, response.TriggerReason.childGuid); + } + } - Assert.Equal(item.FunctionName, response2.Invocation.functionName); - Assert.Equal(item.FunctionInstanceId.ToString(), response2.Invocation.id); - Assert.Equal("CompletedSuccess", response2.Invocation.status); - Assert.Equal(120000.0, response2.Invocation.duration); - Assert.Equal("alpha ()", response2.Invocation.functionDisplayTitle); + private static void AssertEqual(InvocationLogViewModel expectedItem, InvocationLogViewModel actualItem) + { + Assert.Equal(expectedItem.id, actualItem.id); + Assert.Equal(expectedItem.whenUtc, actualItem.whenUtc); + Assert.Equal(expectedItem.status, actualItem.status); + if (actualItem.status != "Running") + { + Assert.Equal(expectedItem.duration, actualItem.duration); + } + else + { + // Compares to current time. + var minValue = DateTime.UtcNow.AddDays(-1) - DateTime.Parse(actualItem.whenUtc); + var totalMs = minValue.TotalMilliseconds; - Assert.Equal(item.FunctionInstanceId.ToString(), response2.TriggerReason.childGuid); + Assert.True(actualItem.duration.Value > totalMs); + } } [Fact] @@ -117,10 +199,9 @@ public class Fixture : IDisposable public HttpClient Client { get; private set; } public string Endpoint { get; private set; } - public FunctionInstanceLogItem[] Data - { - get; private set; - } + + public List Data {get; private set; } + public List ExpectedItems { get; private set; } public Fixture() { @@ -133,7 +214,7 @@ public async Task Init() var tablePrefix = "logtesZZ" + Guid.NewGuid().ToString("n"); ConfigurationManager.AppSettings[FunctionLogTableAppSettingName] = tablePrefix; // tell dashboard to use it _provider = LogFactory.NewLogTableProvider(tableClient, tablePrefix); - this.Data = await WriteTestLoggingDataAsync(_provider); + await WriteTestLoggingDataAsync(_provider); var config = new HttpConfiguration(); @@ -161,31 +242,127 @@ private async Task DisposeAsync() // Write logs. Return what we wrote. // This is baseline data. REader will verify against it exactly. This helps in aggressively catching subtle breaking changes. - private async Task WriteTestLoggingDataAsync(ILogTableProvider provider) + private async Task WriteTestLoggingDataAsync(ILogTableProvider provider) { ILogWriter writer = LogFactory.NewWriter(HostName, "c1", provider); string Func1 = "alpha"; - var time = new DateTime(2010, 3, 6, 10, 11, 20); + var time = new DateTime(2010, 3, 6, 18, 11, 20, DateTimeKind.Utc); List list = new List(); - list.Add(new FunctionInstanceLogItem + List expected = new List(); + this.ExpectedItems = expected; + this.Data = list; + + // List in reverse chronology. + // Completed Success { - FunctionInstanceId = Guid.NewGuid(), - FunctionName = Func1, - StartTime = time, - EndTime = time.AddMinutes(2), - LogOutput = "one", - Status = Microsoft.Azure.WebJobs.Logging.FunctionInstanceStatus.CompletedSuccess - }); + var item = new FunctionInstanceLogItem + { + FunctionInstanceId = Guid.NewGuid(), + FunctionName = Func1, + StartTime = time, + EndTime = time.AddMinutes(2), // Completed + LogOutput = "one", + }; + list.Add(item); + expected.Add(new InvocationLogViewModel + { + id = item.FunctionInstanceId.ToString(), + status = "CompletedSuccess", + whenUtc = "2010-03-06T18:13:20Z", // since it's completed, specifies end-time + duration = 120000.0 + }); + } + + // Completed Error + { + time = time.AddMinutes(-1); + var item = new FunctionInstanceLogItem + { + FunctionInstanceId = Guid.NewGuid(), + FunctionName = Func1, + StartTime = time, + EndTime = time.AddMinutes(2), + ErrorDetails = "some failure", // signifies failure + LogOutput = "two", + }; + list.Add(item); + expected.Add(new InvocationLogViewModel + { + id = item.FunctionInstanceId.ToString(), + status = "CompletedFailed", + whenUtc = "2010-03-06T18:12:20Z", // end-time. + duration = 120000.0 + }); + } + + // Still running + { + time = time.AddMinutes(-1); + var item = new FunctionInstanceLogItem + { + FunctionInstanceId = Guid.NewGuid(), + FunctionName = Func1, + StartTime = time, // Recent heartbeat + LogOutput = "two", + }; + list.Add(item); + expected.Add(new InvocationLogViewModel + { + id = item.FunctionInstanceId.ToString(), + status = "Running", + whenUtc = "2010-03-06T18:09:20Z", // specifies start-time + }); + } + + // Never Finished + { + time = time.AddMinutes(-1); + var item = new TestFunctionInstanceLogItem + { + FunctionInstanceId = Guid.NewGuid(), + FunctionName = Func1, + StartTime = time, // Never Finished + LogOutput = "two", + OnRefresh = (me) => { me.FunctionInstanceHeartbeatExpiry = time; },// stale heartbeat + }; + list.Add(item); + expected.Add(new InvocationLogViewModel + { + id = item.FunctionInstanceId.ToString(), + status = "NeverFinished", + whenUtc = "2010-03-06T18:08:20Z", // starttime + duration = null + }); + } + + // No heartbeat (legacy example) + { + time = time.AddMinutes(-1); + var item = new TestFunctionInstanceLogItem + { + FunctionInstanceId = Guid.NewGuid(), + FunctionName = Func1, + StartTime = time, // Never Finished + LogOutput = "two", + OnRefresh = (me) => { } // No heart beat + }; + list.Add(item); + expected.Add(new InvocationLogViewModel + { + id = item.FunctionInstanceId.ToString(), + status = "Running", + whenUtc = "2010-03-06T18:07:20Z", // starttime + }); + } foreach (var item in list) { await writer.AddAsync(item); } - await writer.FlushAsync(); - return list.ToArray(); + await writer.FlushAsync(); } CloudTableClient GetNewLoggingTableClient() @@ -201,6 +378,16 @@ CloudTableClient GetNewLoggingTableClient() var client = account.CreateCloudTableClient(); return client; } + + public class TestFunctionInstanceLogItem : FunctionInstanceLogItem + { + public Action OnRefresh; + public override void Refresh(TimeSpan pollingFrequency) + { + OnRefresh(this); + } + } + } diff --git a/test/Dashboard.UnitTests/app.config b/test/Dashboard.UnitTests/app.config index 59cdc0fd3..59ba05f96 100644 --- a/test/Dashboard.UnitTests/app.config +++ b/test/Dashboard.UnitTests/app.config @@ -45,7 +45,7 @@ - + diff --git a/test/Dashboard.UnitTests/packages.config b/test/Dashboard.UnitTests/packages.config index d1042b9b9..06e7a42a2 100644 --- a/test/Dashboard.UnitTests/packages.config +++ b/test/Dashboard.UnitTests/packages.config @@ -10,14 +10,18 @@ - - - + + + - + - + + + + + diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AsyncChainEndToEndTests.cs b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AsyncChainEndToEndTests.cs index 1fb2b2019..36d3dba69 100644 --- a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AsyncChainEndToEndTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AsyncChainEndToEndTests.cs @@ -104,20 +104,20 @@ public async Task AsyncChainEndToEnd() "Microsoft.Azure.WebJobs.Host.EndToEndTests.AsyncChainEndToEndTests.TimeoutJob_Throw_NoToken", "Microsoft.Azure.WebJobs.Host.EndToEndTests.AsyncChainEndToEndTests.BlobToBlobAsync", "Microsoft.Azure.WebJobs.Host.EndToEndTests.AsyncChainEndToEndTests.ReadResultBlob", - "Microsoft.Azure.WebJobs.Host.EndToEndTests.AsyncChainEndToEndTests.RandGuidOutput", + "Microsoft.Azure.WebJobs.Host.EndToEndTests.AsyncChainEndToEndTests.SystemParameterBindingOutput", "Function 'AsyncChainEndToEndTests.DisabledJob' is disabled", "Job host started", - "Executing: 'AsyncChainEndToEndTests.WriteStartDataMessageToQueue' - Reason: 'This function was programmatically called via the host APIs.'", - "Executed: 'AsyncChainEndToEndTests.WriteStartDataMessageToQueue' (Succeeded)", - string.Format("Executing: 'AsyncChainEndToEndTests.QueueToQueueAsync' - Reason: 'New queue message detected on '{0}'.'", firstQueueName), - "Executed: 'AsyncChainEndToEndTests.QueueToQueueAsync' (Succeeded)", - string.Format("Executing: 'AsyncChainEndToEndTests.QueueToBlobAsync' - Reason: 'New queue message detected on '{0}'.'", secondQueueName), - "Executed: 'AsyncChainEndToEndTests.QueueToBlobAsync' (Succeeded)", - string.Format("Executing: 'AsyncChainEndToEndTests.BlobToBlobAsync' - Reason: 'New blob detected: {0}/Blob1'", blobContainerName), - "Executed: 'AsyncChainEndToEndTests.BlobToBlobAsync' (Succeeded)", + "Executing 'AsyncChainEndToEndTests.WriteStartDataMessageToQueue' (Reason='This function was programmatically called via the host APIs.', Id=", + "Executed 'AsyncChainEndToEndTests.WriteStartDataMessageToQueue' (Succeeded, Id=", + string.Format("Executing 'AsyncChainEndToEndTests.QueueToQueueAsync' (Reason='New queue message detected on '{0}'.', Id=", firstQueueName), + "Executed 'AsyncChainEndToEndTests.QueueToQueueAsync' (Succeeded, Id=", + string.Format("Executing 'AsyncChainEndToEndTests.QueueToBlobAsync' (Reason='New queue message detected on '{0}'.', Id=", secondQueueName), + "Executed 'AsyncChainEndToEndTests.QueueToBlobAsync' (Succeeded, Id=", + string.Format("Executing 'AsyncChainEndToEndTests.BlobToBlobAsync' (Reason='New blob detected: {0}/Blob1', Id=", blobContainerName), + "Executed 'AsyncChainEndToEndTests.BlobToBlobAsync' (Succeeded, Id=", "Job host stopped", - "Executing: 'AsyncChainEndToEndTests.ReadResultBlob' - Reason: 'This function was programmatically called via the host APIs.'", - "Executed: 'AsyncChainEndToEndTests.ReadResultBlob' (Succeeded)", + "Executing 'AsyncChainEndToEndTests.ReadResultBlob' (Reason='This function was programmatically called via the host APIs.', Id=", + "Executed 'AsyncChainEndToEndTests.ReadResultBlob' (Succeeded, Id=", "User TraceWriter log", "Another User TextWriter log", "User TextWriter log (TestParam)" @@ -126,10 +126,10 @@ public async Task AsyncChainEndToEnd() bool hasError = consoleOutputLines.Any(p => p.Contains("Function had errors")); if (!hasError) { - Assert.Equal( - string.Join(Environment.NewLine, expectedOutputLines), - string.Join(Environment.NewLine, consoleOutputLines) - ); + for (int i = 0; i < expectedOutputLines.Length; i++) + { + Assert.StartsWith(expectedOutputLines[i], consoleOutputLines[i]); + } } Console.SetOut(hold); @@ -154,11 +154,11 @@ public async Task AsyncChainEndToEnd_CustomFactories() Assert.True(queueProcessorFactory.CustomQueueProcessors.Sum(p => p.BeginProcessingCount) >= 2); Assert.True(queueProcessorFactory.CustomQueueProcessors.Sum(p => p.CompleteProcessingCount) >= 2); - Assert.Equal(17, storageClientFactory.TotalBlobClientCount); - Assert.Equal(11, storageClientFactory.TotalQueueClientCount); + Assert.Equal(19, storageClientFactory.TotalBlobClientCount); + Assert.Equal(13, storageClientFactory.TotalQueueClientCount); Assert.Equal(0, storageClientFactory.TotalTableClientCount); - Assert.Equal(6, storageClientFactory.ParameterBlobClientCount); + Assert.Equal(8, storageClientFactory.ParameterBlobClientCount); Assert.Equal(7, storageClientFactory.ParameterQueueClientCount); Assert.Equal(0, storageClientFactory.ParameterTableClientCount); } @@ -190,14 +190,12 @@ public async Task TraceWriterLogging() bool hasError = string.Join(Environment.NewLine, trace.Traces.Where(p => p.Message.Contains("Error"))).Any(); if (!hasError) { - Assert.Equal(18, trace.Traces.Count); Assert.NotNull(trace.Traces.SingleOrDefault(p => p.Message.Contains("User TraceWriter log"))); Assert.NotNull(trace.Traces.SingleOrDefault(p => p.Message.Contains("User TextWriter log (TestParam)"))); Assert.NotNull(trace.Traces.SingleOrDefault(p => p.Message.Contains("Another User TextWriter log"))); ValidateTraceProperties(trace); string[] consoleOutputLines = consoleOutput.ToString().Trim().Split(new string[] { Environment.NewLine }, StringSplitOptions.None); - Assert.Equal(27, consoleOutputLines.Length); Assert.NotNull(consoleOutputLines.SingleOrDefault(p => p.Contains("User TraceWriter log"))); Assert.NotNull(consoleOutputLines.SingleOrDefault(p => p.Contains("User TextWriter log (TestParam)"))); Assert.NotNull(consoleOutputLines.SingleOrDefault(p => p.Contains("Another User TextWriter log"))); @@ -212,7 +210,7 @@ private void ValidateTraceProperties(TestTraceWriter trace) foreach (var traceEvent in trace.Traces) { var message = traceEvent.Message; - var startedOrEndedMessage = message.StartsWith("Executing: ") || message.StartsWith("Executed: "); + var startedOrEndedMessage = message.StartsWith("Executing ") || message.StartsWith("Executed "); var userMessage = message.Contains("User TextWriter") || message.Contains("User TraceWriter"); if (startedOrEndedMessage || userMessage) @@ -267,7 +265,7 @@ public void FunctionFailures_LogsExpectedTraceEvent() } [Fact] - public void RandGuidOutput_GeneratesRandomIDs() + public void SystemParameterBindingOutput_GeneratesExpectedBlobs() { JobHost host = new JobHost(_hostConfig); @@ -281,15 +279,12 @@ public void RandGuidOutput_GeneratesRandomIDs() } } - MethodInfo methodInfo = GetType().GetMethod("RandGuidOutput"); - for (int i = 0; i < 3; i++) + MethodInfo methodInfo = GetType().GetMethod("SystemParameterBindingOutput"); + var arguments = new Dictionary { - var arguments = new Dictionary - { - { "input", i.ToString() } - }; - host.Call(methodInfo, arguments); - } + { "input", "Test Value" } + }; + host.Call(methodInfo, arguments); // We expect 3 separate blobs to have been written var blobs = container.ListBlobs().Cast().ToArray(); @@ -297,8 +292,7 @@ public void RandGuidOutput_GeneratesRandomIDs() foreach (var blob in blobs) { string content = blob.DownloadText(Encoding.UTF8); - int blobInt = int.Parse(content.Trim(new char[] { '\uFEFF', '\u200B' })); - Assert.True(blobInt >= 0 && blobInt <= 3); + Assert.Equal("Test Value", content.Trim(new char[] { '\uFEFF', '\u200B' })); } } @@ -359,7 +353,7 @@ private async Task RunTimeoutTest(IWebJobsExceptionHandler exceptionHandler, Typ TraceEvent[] traceErrors = trace.Traces.Where(p => p.Level == TraceLevel.Error).ToArray(); Assert.Equal(3, traceErrors.Length); Assert.True(traceErrors[0].Message.StartsWith(string.Format("Timeout value of 00:00:01 exceeded by function 'AsyncChainEndToEndTests.{0}'", functionName))); - Assert.True(traceErrors[1].Message.StartsWith(string.Format("Executed: 'AsyncChainEndToEndTests.{0}' (Failed)", functionName))); + Assert.True(traceErrors[1].Message.StartsWith(string.Format("Executed 'AsyncChainEndToEndTests.{0}' (Failed, Id=", functionName))); Assert.True(traceErrors[2].Message.Trim().StartsWith("Function had errors. See Azure WebJobs SDK dashboard for details.")); } @@ -401,7 +395,7 @@ public async Task FunctionTraceLevelOverride_ProducesExpectedOutput() // expect no function output TraceEvent[] traces = trace.Traces.ToArray(); - Assert.Equal(5, traces.Length); + Assert.Equal(4, traces.Length); Assert.False(traces.Any(p => p.Message.Contains("test message"))); } } @@ -435,13 +429,13 @@ public async Task FunctionTraceLevelOverride_Failure_ProducesExpectedOutput() // expect normal logs to be written (TraceLevel override is ignored) TraceEvent[] traces = trace.Traces.ToArray(); - Assert.Equal(10, traces.Length); + Assert.Equal(9, traces.Length); string output = string.Join("\r\n", traces.Select(p => p.Message)); - Assert.True(output.Contains("Executing: 'AsyncChainEndToEndTests.QueueTrigger_TraceLevelOverride'")); - Assert.True(output.Contains("Exception while executing function: AsyncChainEndToEndTests.QueueTrigger_TraceLevelOverride")); - Assert.True(output.Contains("Executed: 'AsyncChainEndToEndTests.QueueTrigger_TraceLevelOverride' (Failed)")); - Assert.True(output.Contains("Message has reached MaxDequeueCount of 1")); + Assert.Contains("Executing 'AsyncChainEndToEndTests.QueueTrigger_TraceLevelOverride' (Reason='New queue message detected", output); + Assert.Contains("Exception while executing function: AsyncChainEndToEndTests.QueueTrigger_TraceLevelOverride", output); + Assert.Contains("Executed 'AsyncChainEndToEndTests.QueueTrigger_TraceLevelOverride' (Failed, Id=", output); + Assert.Contains("Message has reached MaxDequeueCount of 1", output); } } finally @@ -469,11 +463,13 @@ public static void AlwaysFailJob() } [NoAutomaticTrigger] - public static void RandGuidOutput( + public static void SystemParameterBindingOutput( [QueueTrigger("test")] string input, - [Blob("test-output/{rand-guid}")] out string blob) + [Blob("test-output/{rand-guid}")] out string blob, + [Blob("test-output/{rand-guid:N}")] out string blob2, + [Blob("test-output/{datetime:yyyy-mm-dd}:{rand-guid:N}")] out string blob3) { - blob = input; + blob = blob2 = blob3 = input; } [Disable("Disable_DisabledJob")] @@ -619,9 +615,11 @@ public QueueProcessor Create(QueueProcessorFactoryContext context) // demonstrates how queue options can be customized context.Queue.EncodeMessage = true; - // demonstrates how batch processing behavior can be customized + // demonstrates how batch processing behavior and other knobs + // can be customized context.BatchSize = 30; context.NewBatchThreshold = 100; + context.MaxPollingInterval = TimeSpan.FromSeconds(15); CustomQueueProcessor processor = new CustomQueueProcessor(context); CustomQueueProcessors.Add(processor); diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AzureStorageEndToEndTests.cs b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AzureStorageEndToEndTests.cs index 1efa10505..048b79143 100644 --- a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AzureStorageEndToEndTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/AzureStorageEndToEndTests.cs @@ -14,6 +14,7 @@ using Microsoft.WindowsAzure.Storage.Queue; using Microsoft.WindowsAzure.Storage.Table; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using Xunit; namespace Microsoft.Azure.WebJobs.Host.EndToEndTests @@ -41,12 +42,10 @@ public class AzureStorageEndToEndTests : IClassFixture TestResult != null); + + JArray results = (JArray)TestResult; + Assert.Equal(1, results.Count); + + input = new Person { Age = 25, Location = "Tam O'Shanter" }; + json = JsonConvert.SerializeObject(input); + arguments = new { person = json }; + await host.CallAsync(methodInfo, arguments); + await TestHelpers.Await(() => TestResult != null); + results = (JArray)TestResult; + Assert.Equal(1, results.Count); + Assert.Equal("Bill", (string)results[0]["Name"]); + } + private void EndToEndTest(bool uploadBlobBeforeHostStart) { // Reinitialize the name resolver to avoid conflicts @@ -379,6 +463,13 @@ private class CustomTableEntity : TableEntity public int Number { get; set; } } + public class Person : TableEntity + { + public int Age { get; set; } + public string Location { get; set; } + public string Name { get; set; } + } + public class TestFixture : IDisposable { public TestFixture() diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/BlobTriggerEndToEndTests.cs b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/BlobTriggerEndToEndTests.cs new file mode 100644 index 000000000..6dec5aa1a --- /dev/null +++ b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/BlobTriggerEndToEndTests.cs @@ -0,0 +1,260 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.TestCommon; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Host.EndToEndTests +{ + public class BlobTriggerEndToEndTests : IDisposable + { + private const string TestArtifactPrefix = "e2etests"; + + private const string SingleTriggerContainerName = TestArtifactPrefix + "singletrigger-%rnd%"; + private const string PoisonTestContainerName = TestArtifactPrefix + "poison-%rnd%"; + private const string TestBlobName = "test"; + + private const string BlobChainContainerName = TestArtifactPrefix + "blobchain-%rnd%"; + private const string BlobChainTriggerBlobName = "blob"; + private const string BlobChainTriggerBlobPath = BlobChainContainerName + "/" + BlobChainTriggerBlobName; + private const string BlobChainCommittedQueueName = "committed"; + private const string BlobChainIntermediateBlobPath = BlobChainContainerName + "/" + "blob.middle"; + private const string BlobChainOutputBlobName = "blob.out"; + private const string BlobChainOutputBlobPath = BlobChainContainerName + "/" + BlobChainOutputBlobName; + + private static ManualResetEvent _completedEvent; + private static int _timesProcessed; + + private readonly CloudBlobContainer _testContainer; + private readonly JobHostConfiguration _hostConfiguration; + private readonly CloudStorageAccount _storageAccount; + private readonly RandomNameResolver _nameResolver; + + private static object _syncLock = new object(); + private static List poisonBlobMessages = new List(); + + public BlobTriggerEndToEndTests() + { + _timesProcessed = 0; + + _nameResolver = new RandomNameResolver(); + _hostConfiguration = new JobHostConfiguration() + { + NameResolver = _nameResolver, + TypeLocator = new FakeTypeLocator(this.GetType()), + }; + + _storageAccount = CloudStorageAccount.Parse(_hostConfiguration.StorageConnectionString); + CloudBlobClient blobClient = _storageAccount.CreateCloudBlobClient(); + _testContainer = blobClient.GetContainerReference(_nameResolver.ResolveInString(SingleTriggerContainerName)); + Assert.False(_testContainer.Exists()); + _testContainer.Create(); + } + + public static void BlobProcessorPrimary( + [BlobTrigger(PoisonTestContainerName + "/{name}")] string input) + { + // throw to generate a poison blob message + throw new Exception(); + } + + // process the poison queue for the primary storage account + public static void PoisonBlobQueueProcessorPrimary( + [QueueTrigger("webjobs-blobtrigger-poison")] JObject message) + { + lock (_syncLock) + { + string blobName = (string)message["BlobName"]; + poisonBlobMessages.Add(blobName); + } + } + + public static void BlobProcessorSecondary( + [StorageAccount("SecondaryStorage")] + [BlobTrigger(PoisonTestContainerName + "/{name}")] string input) + { + // throw to generate a poison blob message + throw new Exception(); + } + + // process the poison queue for the secondary storage account + public static void PoisonBlobQueueProcessorSecondary( + [StorageAccount("SecondaryStorage")] + [QueueTrigger("webjobs-blobtrigger-poison")] JObject message) + { + lock (_syncLock) + { + string blobName = (string)message["BlobName"]; + poisonBlobMessages.Add(blobName); + } + } + + public static void SingleBlobTrigger( + [BlobTrigger(SingleTriggerContainerName + "/{name}")] string sleepTimeInSeconds) + { + Interlocked.Increment(ref _timesProcessed); + + int sleepTime = int.Parse(sleepTimeInSeconds) * 1000; + Thread.Sleep(sleepTime); + + _completedEvent.Set(); + } + + public static void BlobChainStepOne( + [BlobTrigger(BlobChainTriggerBlobPath)] TextReader input, + [Blob(BlobChainIntermediateBlobPath)] TextWriter output) + { + string content = input.ReadToEnd(); + output.Write(content); + } + + public static void BlobChainStepTwo( + [BlobTrigger(BlobChainIntermediateBlobPath)] TextReader input, + [Blob(BlobChainOutputBlobPath)] TextWriter output, + [Queue(BlobChainCommittedQueueName)] out string committed) + { + string content = input.ReadToEnd(); + output.Write("*" + content + "*"); + committed = String.Empty; + } + + public static void BlobChainStepThree([QueueTrigger(BlobChainCommittedQueueName)] string ignore) + { + _completedEvent.Set(); + } + + [Theory] + [InlineData("AzureWebJobsSecondaryStorage")] + [InlineData("AzureWebJobsStorage")] + public async Task PoisonMessage_CreatedInCorrectStorageAccount(string storageAccountSetting) + { + poisonBlobMessages.Clear(); + + var storageAccount = CloudStorageAccount.Parse(Environment.GetEnvironmentVariable(storageAccountSetting)); + var blobClient = storageAccount.CreateCloudBlobClient(); + var containerName = _nameResolver.ResolveInString(PoisonTestContainerName); + var container = blobClient.GetContainerReference(containerName); + container.Create(); + + var blobName = Guid.NewGuid().ToString(); + CloudBlockBlob blob = container.GetBlockBlobReference(blobName); + blob.UploadText("0"); + + using (JobHost host = new JobHost(_hostConfiguration)) + { + host.Start(); + + // wait for the poison message to be handled + await TestHelpers.Await(() => + { + return poisonBlobMessages.Contains(blobName); + }); + } + } + + [Fact] + public void BlobGetsProcessedOnlyOnce_SingleHost() + { + TextWriter hold = Console.Out; + StringWriter consoleOutput = new StringWriter(); + Console.SetOut(consoleOutput); + + CloudBlockBlob blob = _testContainer.GetBlockBlobReference(TestBlobName); + blob.UploadText("0"); + + int timeToProcess; + + // Process the blob first + using (_completedEvent = new ManualResetEvent(initialState: false)) + using (JobHost host = new JobHost(_hostConfiguration)) + { + DateTime startTime = DateTime.Now; + + host.Start(); + Assert.True(_completedEvent.WaitOne(TimeSpan.FromSeconds(60))); + + timeToProcess = (int)(DateTime.Now - startTime).TotalMilliseconds; + + Console.SetOut(hold); + + Assert.Equal(1, _timesProcessed); + + string[] consoleOutputLines = consoleOutput.ToString().Trim().Split(new string[] { Environment.NewLine }, StringSplitOptions.None); + var executions = consoleOutputLines.Where(p => p.Contains("Executing")); + Assert.Equal(1, executions.Count()); + Assert.StartsWith(string.Format("Executing 'BlobTriggerEndToEndTests.SingleBlobTrigger' (Reason='New blob detected: {0}/{1}', Id=", blob.Container.Name, blob.Name), executions.Single()); + } + + // Then start again and make sure the blob doesn't get reprocessed + // wait twice the amount of time required to process first before + // deciding that it doesn't get reprocessed + using (_completedEvent = new ManualResetEvent(initialState: false)) + using (JobHost host = new JobHost(_hostConfiguration)) + { + host.Start(); + + bool blobReprocessed = _completedEvent.WaitOne(2 * timeToProcess); + + Assert.False(blobReprocessed); + } + + Assert.Equal(1, _timesProcessed); + } + + [Fact] + public void BlobChainTest() + { + // write the initial trigger blob to start the chain + var blobClient = _storageAccount.CreateCloudBlobClient(); + var container = blobClient.GetContainerReference(_nameResolver.ResolveInString(BlobChainContainerName)); + container.CreateIfNotExists(); + CloudBlockBlob blob = container.GetBlockBlobReference(BlobChainTriggerBlobName); + blob.UploadText("0"); + + using (_completedEvent = new ManualResetEvent(initialState: false)) + using (JobHost host = new JobHost(_hostConfiguration)) + { + host.Start(); + Assert.True(_completedEvent.WaitOne(TimeSpan.FromSeconds(60))); + } + } + + [Fact] + public void BlobGetsProcessedOnlyOnce_MultipleHosts() + { + _testContainer + .GetBlockBlobReference(TestBlobName) + .UploadText("10"); + + using (_completedEvent = new ManualResetEvent(initialState: false)) + using (JobHost host1 = new JobHost(_hostConfiguration)) + using (JobHost host2 = new JobHost(_hostConfiguration)) + { + host1.Start(); + host2.Start(); + + Assert.True(_completedEvent.WaitOne(TimeSpan.FromSeconds(60))); + } + + Assert.Equal(1, _timesProcessed); + } + + public void Dispose() + { + CloudBlobClient blobClient = _storageAccount.CreateCloudBlobClient(); + foreach (var testContainer in blobClient.ListContainers(TestArtifactPrefix)) + { + testContainer.Delete(); + } + } + } +} diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/BlobTriggerTests.cs b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/BlobTriggerTests.cs deleted file mode 100644 index 8ef82004f..000000000 --- a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/BlobTriggerTests.cs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.IO; -using System.Linq; -using System.Threading; -using Microsoft.Azure.WebJobs.Host.TestCommon; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using Xunit; - -namespace Microsoft.Azure.WebJobs.Host.EndToEndTests -{ - public class BlobTriggerTests : IDisposable - { - private const string TestArtifactPrefix = "e2etestsingletrigger"; - private const string ContainerName = TestArtifactPrefix + "-%rnd%"; - private const string BlobName = "test"; - - private static ManualResetEvent _blobProcessedEvent; - private static int _timesProcessed; - - private readonly CloudBlobContainer _testContainer; - private readonly JobHostConfiguration _hostConfiguration; - private readonly CloudStorageAccount _storageAccount; - - public BlobTriggerTests() - { - _timesProcessed = 0; - - RandomNameResolver nameResolver = new RandomNameResolver(); - _hostConfiguration = new JobHostConfiguration() - { - NameResolver = nameResolver, - TypeLocator = new FakeTypeLocator(typeof(BlobTriggerTests)), - }; - - _storageAccount = CloudStorageAccount.Parse(_hostConfiguration.StorageConnectionString); - CloudBlobClient blobClient = _storageAccount.CreateCloudBlobClient(); - _testContainer = blobClient.GetContainerReference(nameResolver.ResolveInString(ContainerName)); - Assert.False(_testContainer.Exists()); - _testContainer.Create(); - } - - public static void SingleBlobTrigger([BlobTrigger(ContainerName + "/{name}")] string sleepTimeInSeconds) - { - Interlocked.Increment(ref _timesProcessed); - - int sleepTime = int.Parse(sleepTimeInSeconds) * 1000; - Thread.Sleep(sleepTime); - - _blobProcessedEvent.Set(); - } - - public void Dispose() - { - CloudBlobClient blobClient = _storageAccount.CreateCloudBlobClient(); - foreach (var testContainer in blobClient.ListContainers(TestArtifactPrefix)) - { - testContainer.Delete(); - } - } - - [Fact] - public void BlobGetsProcessedOnlyOnce_SingleHost() - { - TextWriter hold = Console.Out; - StringWriter consoleOutput = new StringWriter(); - Console.SetOut(consoleOutput); - - CloudBlockBlob blob = _testContainer.GetBlockBlobReference(BlobName); - blob.UploadText("0"); - - int timeToProcess; - - // Process the blob first - using (_blobProcessedEvent = new ManualResetEvent(initialState: false)) - using (JobHost host = new JobHost(_hostConfiguration)) - { - DateTime startTime = DateTime.Now; - - host.Start(); - Assert.True(_blobProcessedEvent.WaitOne(TimeSpan.FromSeconds(60))); - - timeToProcess = (int)(DateTime.Now - startTime).TotalMilliseconds; - - host.Stop(); - - Console.SetOut(hold); - - Assert.Equal(1, _timesProcessed); - - string[] consoleOutputLines = consoleOutput.ToString().Trim().Split(new string[] { Environment.NewLine }, StringSplitOptions.None); - var executions = consoleOutputLines.Where(p => p.Contains("Executing")); - Assert.Equal(1, executions.Count()); - Assert.Equal(string.Format("Executing: 'BlobTriggerTests.SingleBlobTrigger' - Reason: 'New blob detected: {0}/{1}'", blob.Container.Name, blob.Name), executions.Single()); - } - - // Then start again and make sure the blob doesn't get reprocessed - // wait twice the amount of time required to process first before - // deciding that it doesn't get reprocessed - using (_blobProcessedEvent = new ManualResetEvent(initialState: false)) - using (JobHost host = new JobHost(_hostConfiguration)) - { - host.Start(); - - bool blobReprocessed = _blobProcessedEvent.WaitOne(2 * timeToProcess); - - host.Stop(); - - Assert.False(blobReprocessed); - } - - Assert.Equal(1, _timesProcessed); - } - - [Fact] - public void BlobGetsProcessedOnlyOnce_MultipleHosts() - { - _testContainer - .GetBlockBlobReference(BlobName) - .UploadText("10"); - - using (_blobProcessedEvent = new ManualResetEvent(initialState: false)) - using (JobHost host1 = new JobHost(_hostConfiguration)) - using (JobHost host2 = new JobHost(_hostConfiguration)) - { - host1.Start(); - host2.Start(); - - Assert.True(_blobProcessedEvent.WaitOne(TimeSpan.FromSeconds(60))); - - host1.Stop(); - host2.Stop(); - } - - Assert.Equal(1, _timesProcessed); - } - } -} diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/ServiceBusEndToEndTests.cs b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/ServiceBusEndToEndTests.cs index ed9fc6cdc..91cc77646 100644 --- a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/ServiceBusEndToEndTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/ServiceBusEndToEndTests.cs @@ -262,25 +262,25 @@ private async Task ServiceBusEndToEndInternal(Type jobContainerType, JobHost hos string.Format("{0}.SBTopicListener1", jobContainerType.FullName), string.Format("{0}.SBTopicListener2", jobContainerType.FullName), "Job host started", - string.Format("Executing: '{0}.SBQueue2SBQueue' - Reason: 'New ServiceBus message detected on '{1}'.'", jobContainerType.Name, startQueueName), - string.Format("Executed: '{0}.SBQueue2SBQueue' (Succeeded)", jobContainerType.Name), - string.Format("Executing: '{0}.SBQueue2SBTopic' - Reason: 'New ServiceBus message detected on '{1}'.'", jobContainerType.Name, secondQueueName), - string.Format("Executed: '{0}.SBQueue2SBTopic' (Succeeded)", jobContainerType.Name), - string.Format("Executing: '{0}.SBTopicListener1' - Reason: 'New ServiceBus message detected on '{1}'.'", jobContainerType.Name, firstTopicName), - string.Format("Executed: '{0}.SBTopicListener1' (Succeeded)", jobContainerType.Name), - string.Format("Executing: '{0}.SBTopicListener2' - Reason: 'New ServiceBus message detected on '{1}'.'", jobContainerType.Name, secondTopicName), - string.Format("Executed: '{0}.SBTopicListener2' (Succeeded)", jobContainerType.Name), + string.Format("Executing '{0}.SBQueue2SBQueue' (Reason='New ServiceBus message detected on '{1}'.', Id=", jobContainerType.Name, startQueueName), + string.Format("Executed '{0}.SBQueue2SBQueue' (Succeeded, Id=", jobContainerType.Name), + string.Format("Executing '{0}.SBQueue2SBTopic' (Reason='New ServiceBus message detected on '{1}'.', Id=", jobContainerType.Name, secondQueueName), + string.Format("Executed '{0}.SBQueue2SBTopic' (Succeeded, Id=", jobContainerType.Name), + string.Format("Executing '{0}.SBTopicListener1' (Reason='New ServiceBus message detected on '{1}'.', Id=", jobContainerType.Name, firstTopicName), + string.Format("Executed '{0}.SBTopicListener1' (Succeeded, Id=", jobContainerType.Name), + string.Format("Executing '{0}.SBTopicListener2' (Reason='New ServiceBus message detected on '{1}'.', Id=", jobContainerType.Name, secondTopicName), + string.Format("Executed '{0}.SBTopicListener2' (Succeeded, Id=", jobContainerType.Name), "Job host stopped" }.OrderBy(p => p).ToArray(); bool hasError = consoleOutputLines.Any(p => p.Contains("Function had errors")); if (!hasError) { - Assert.Equal( - string.Join(Environment.NewLine, expectedOutputLines), - string.Join(Environment.NewLine, consoleOutputLines) - ); - } + for (int i = 0; i < expectedOutputLines.Length; i++) + { + Assert.StartsWith(expectedOutputLines[i], consoleOutputLines[i]); + } + } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/SingletonEndToEndTests.cs b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/SingletonEndToEndTests.cs index eed5264f5..d515bf0f4 100644 --- a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/SingletonEndToEndTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/SingletonEndToEndTests.cs @@ -277,7 +277,7 @@ public async Task SingletonFunction_HostScope() host.Start(); MethodInfo method = typeof(TestJobs).GetMethod("SingletonJobA_HostScope"); - await host.CallAsync(method, new {}); + await host.CallAsync(method, new { }); VerifyLeaseState(method, SingletonScope.Host, "TestValue", LeaseState.Available, LeaseStatus.Unlocked); @@ -295,13 +295,13 @@ public async Task SingletonFunction_HostScope_InvocationsAreSerialized() List tasks = new List(); for (int i = 0; i < 5; i++) { - tasks.Add(host.CallAsync(methodA, new {})); + tasks.Add(host.CallAsync(methodA, new { })); } MethodInfo methodB = typeof(TestJobs).GetMethod("SingletonJobB_HostScope"); for (int i = 0; i < 5; i++) { - tasks.Add(host.CallAsync(methodB, new {})); + tasks.Add(host.CallAsync(methodB, new { })); } await Task.WhenAll(tasks); @@ -390,7 +390,7 @@ public async Task SingletonTriggerJob([QueueTrigger(Queue2Name)] WorkItem workIt VerifyLeaseState( GetType().GetMethod("SingletonTriggerJob"), SingletonScope.Function, - string.Format("{0}/{1}", workItem.Region, workItem.Zone), + string.Format("{0}/{1}", workItem.Region, workItem.Zone), LeaseState.Leased, LeaseStatus.Locked); // When run concurrently, this job will fail very reliably @@ -410,7 +410,7 @@ public async Task SingletonJob(WorkItem workItem) VerifyLeaseState( GetType().GetMethod("SingletonJob"), SingletonScope.Function, - "TestValue", + "TestValue", LeaseState.Leased, LeaseStatus.Locked); if (workItem.Category < 0) @@ -483,7 +483,7 @@ public async Task TriggerJob_SingletonListener([TestTrigger] string test) VerifyLeaseState( GetType().GetMethod("TriggerJob_SingletonListener"), SingletonScope.Function, - "Listener", + "Listener", LeaseState.Leased, LeaseStatus.Locked); await Task.Delay(50); @@ -588,7 +588,7 @@ private void IncrementJobInvocationCount() private static void UpdateScopeLock(string scope, bool isLocked) { bool scopeIsLocked = false; - if (ScopeLocks.TryGetValue(scope, out scopeIsLocked) + if (ScopeLocks.TryGetValue(scope, out scopeIsLocked) && scopeIsLocked && isLocked) { FailureDetected = true; @@ -732,9 +732,9 @@ public Type Type get { return typeof(string); } } - public object GetValue() + public Task GetValueAsync() { - return "Test"; + return Task.FromResult("Test"); } public string ToInvokeString() diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/WebJobs.Host.EndToEndTests.csproj b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/WebJobs.Host.EndToEndTests.csproj index 67f21ab18..04499ea17 100644 --- a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/WebJobs.Host.EndToEndTests.csproj +++ b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/WebJobs.Host.EndToEndTests.csproj @@ -24,7 +24,7 @@ prompt 4 true - 5 + default pdbonly @@ -34,7 +34,7 @@ prompt 4 true - 5 + default @@ -42,19 +42,19 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True - ..\..\packages\WindowsAzure.ServiceBus.3.4.1\lib\net45-full\Microsoft.ServiceBus.dll + ..\..\packages\WindowsAzure.ServiceBus.3.4.5\lib\net45-full\Microsoft.ServiceBus.dll True @@ -75,7 +75,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True @@ -112,7 +112,7 @@ - + diff --git a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/packages.config b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/packages.config index 44572bc47..1cacafd58 100644 --- a/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/packages.config +++ b/test/Microsoft.Azure.WebJobs.Host.EndToEndTests/packages.config @@ -1,13 +1,17 @@  - - - + + + - - + + + + + + diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/BlobTriggerTests.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/BlobTriggerTests.cs index 78ed27765..2fd260b9c 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/BlobTriggerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/BlobTriggerTests.cs @@ -3,14 +3,11 @@ using System; using System.Collections.Generic; -using System.IO; using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Blobs.Listeners; using Microsoft.Azure.WebJobs.Host.FunctionalTests.TestDoubles; using Microsoft.Azure.WebJobs.Host.Storage; using Microsoft.Azure.WebJobs.Host.Storage.Blob; using Microsoft.WindowsAzure.Storage.Blob; -using Newtonsoft.Json; using Xunit; namespace Microsoft.Azure.WebJobs.Host.FunctionalTests @@ -20,8 +17,6 @@ public class BlobTriggerTests private const string ContainerName = "container"; private const string BlobName = "blob"; private const string BlobPath = ContainerName + "/" + BlobName; - private const string OutputBlobName = "blob.out"; - private const string OutputBlobPath = ContainerName + "/" + OutputBlobName; [Fact] public void BlobTrigger_IfBoundToCloudBlob_Binds() @@ -51,93 +46,6 @@ public static void Run([BlobTrigger(BlobPath)] ICloudBlob blob) } } - [Fact] - public void BlobTrigger_IfBindingAlwaysFails_MovesToPoisonQueue() - { - // Arrange - IStorageAccount account = CreateFakeStorageAccount(); - IStorageBlobContainer container = CreateContainer(account, ContainerName); - IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName); - CloudBlockBlob expectedBlob = blob.SdkObject; - blob.UploadText("ignore"); - - // Act - string result = RunTrigger(account, typeof(PoisonBlobProgram), - (s) => PoisonBlobProgram.TaskSource = s, - new string[] { typeof(PoisonBlobProgram).FullName + ".PutInPoisonQueue" }); - - // Assert - BlobTriggerMessage message = JsonConvert.DeserializeObject(result); - Assert.NotNull(message); - Assert.Equal(typeof(PoisonBlobProgram).FullName + ".PutInPoisonQueue", message.FunctionId); - Assert.Equal(StorageBlobType.BlockBlob, message.BlobType); - Assert.Equal(ContainerName, message.ContainerName); - Assert.Equal(BlobName, message.BlobName); - Assert.NotEmpty(message.ETag); - } - - private class PoisonBlobProgram - { - public static TaskCompletionSource TaskSource { get; set; } - - public static void PutInPoisonQueue([BlobTrigger(BlobPath)] string message) - { - throw new InvalidOperationException(); - } - - public static void ReceiveFromPoisonQueue([QueueTrigger("webjobs-blobtrigger-poison")] string message) - { - TaskSource.TrySetResult(message); - } - } - - [Fact] - public void BlobTrigger_IfWritesToSecondBlobTrigger_TriggersOutputQuickly() - { - // Arrange - IStorageAccount account = CreateFakeStorageAccount(); - IStorageBlobContainer container = CreateContainer(account, ContainerName); - IStorageBlockBlob inputBlob = container.GetBlockBlobReference(BlobName); - inputBlob.UploadText("abc"); - - // Act - RunTrigger(account, typeof(BlobTriggerToBlobTriggerProgram), - (s) => BlobTriggerToBlobTriggerProgram.TaskSource = s); - - // Assert - IStorageBlockBlob outputBlob = container.GetBlockBlobReference(OutputBlobName); - string content = outputBlob.DownloadText(); - Assert.Equal("*abc*", content); - } - - private class BlobTriggerToBlobTriggerProgram - { - private const string CommittedQueueName = "committed"; - private const string IntermediateBlobPath = ContainerName + "/" + "blob.middle"; - - public static TaskCompletionSource TaskSource { get; set; } - - public static void StepOne([BlobTrigger(BlobPath)] TextReader input, - [Blob(IntermediateBlobPath)] TextWriter output) - { - string content = input.ReadToEnd(); - output.Write(content); - } - - public static void StepTwo([BlobTrigger(IntermediateBlobPath)] TextReader input, - [Blob(OutputBlobPath)] TextWriter output, [Queue(CommittedQueueName)] out string committed) - { - string content = input.ReadToEnd(); - output.Write("*" + content + "*"); - committed = String.Empty; - } - - public static void StepThree([QueueTrigger(CommittedQueueName)] string ignore) - { - TaskSource.TrySetResult(null); - } - } - private static IStorageBlobContainer CreateContainer(IStorageAccount account, string containerName) { IStorageBlobClient client = account.CreateBlobClient(); diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/FunctionalTest.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/FunctionalTest.cs index 217300d5e..b3cdcf696 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/FunctionalTest.cs +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/FunctionalTest.cs @@ -215,6 +215,7 @@ private static IServiceProvider CreateServiceProvider(IStorageAccount s IHostIdProvider hostIdProvider = new FakeHostIdProvider(); INameResolver nameResolver = null; IQueueConfiguration queueConfiguration = new FakeQueueConfiguration(storageAccountProvider); + JobHostBlobsConfiguration blobsConfiguration = new JobHostBlobsConfiguration(); IWebJobsExceptionHandler exceptionHandler = new TaskBackgroundExceptionHandler(taskSource); ContextAccessor messageEnqueuedWatcherAccessor = @@ -234,9 +235,9 @@ private static IServiceProvider CreateServiceProvider(IStorageAccount s ITriggerBindingProvider triggerBindingProvider = DefaultTriggerBindingProvider.Create(nameResolver, storageAccountProvider, extensionTypeLocator, hostIdProvider, - queueConfiguration, exceptionHandler, messageEnqueuedWatcherAccessor, + queueConfiguration, blobsConfiguration, exceptionHandler, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, sharedContextProvider, extensions, singletonManager, new TestTraceWriter(TraceLevel.Verbose)); - IBindingProvider bindingProvider = DefaultBindingProvider.Create(nameResolver, storageAccountProvider, + IBindingProvider bindingProvider = DefaultBindingProvider.Create(nameResolver, null, storageAccountProvider, extensionTypeLocator, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, extensions); @@ -301,8 +302,7 @@ public static TResult RunTrigger(IStorageAccount account, Type programT public static TResult RunTrigger(IStorageAccount account, Type programType, Action> setTaskSource, IEnumerable ignoreFailureFunctions) { - return RunTrigger(account, programType, setTaskSource, DefaultJobActivator.Instance, - ignoreFailureFunctions); + return RunTrigger(account, programType, setTaskSource, DefaultJobActivator.Instance, ignoreFailureFunctions); } public static TResult RunTrigger(IStorageAccount account, Type programType, @@ -337,8 +337,16 @@ public static TResult RunTrigger(IServiceProvider serviceProvider, Task host.Start(); // Act - completed = task.WaitUntilCompleted(25 * 1000); - + if (Debugger.IsAttached) + { + task.WaitUntilCompleted(); + completed = true; + } + else + { + completed = task.WaitUntilCompleted(25 * 1000); + } + // Assert Assert.True(completed); diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/HostCallTests.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/HostCallTests.cs index 06e391406..78837341b 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/HostCallTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/HostCallTests.cs @@ -1774,7 +1774,7 @@ public static void FuncWithPocoValueEntity([Table(TableName, "PK", "RK")] Struct private class SdkTableEntity : TableEntity { public string Value { get; set; } - } + } private class PocoTableEntity { diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueProcessorTests.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueProcessorTests.cs index e29bc4b5c..65b102a31 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueProcessorTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueProcessorTests.cs @@ -37,10 +37,22 @@ public QueueProcessorTests(TestFixture fixture) [Fact] public void Constructor_DefaultsValues() { - QueueProcessorFactoryContext context = new QueueProcessorFactoryContext(_queue, _trace, _queuesConfig); + var config = new JobHostQueuesConfiguration + { + BatchSize = 32, + MaxDequeueCount = 2, + NewBatchThreshold = 100, + VisibilityTimeout = TimeSpan.FromSeconds(30), + MaxPollingInterval = TimeSpan.FromSeconds(15) + }; + QueueProcessorFactoryContext context = new QueueProcessorFactoryContext(_queue, _trace, config); QueueProcessor localProcessor = new QueueProcessor(context); - Assert.Equal(_queuesConfig.BatchSize, localProcessor.BatchSize); - Assert.Equal(_queuesConfig.NewBatchThreshold, localProcessor.NewBatchThreshold); + + Assert.Equal(config.BatchSize, localProcessor.BatchSize); + Assert.Equal(config.MaxDequeueCount, localProcessor.MaxDequeueCount); + Assert.Equal(config.NewBatchThreshold, localProcessor.NewBatchThreshold); + Assert.Equal(config.VisibilityTimeout, localProcessor.VisibilityTimeout); + Assert.Equal(config.MaxPollingInterval, localProcessor.MaxPollingInterval); } [Fact] @@ -88,6 +100,8 @@ public async Task CompleteProcessingMessageAsync_MaxDequeueCountExceeded_MovesMe localProcessor.MessageAddedToPoisonQueue += (sender, e) => { Assert.Same(sender, localProcessor); + Assert.Same(_poisonQueue, e.PoisonQueue); + Assert.NotNull(e.Message); poisonMessageHandlerCalled = true; }; @@ -111,6 +125,29 @@ public async Task CompleteProcessingMessageAsync_MaxDequeueCountExceeded_MovesMe Assert.True(poisonMessageHandlerCalled); } + [Fact] + public async Task CompleteProcessingMessageAsync_Failure_AppliesVisibilityTimeout() + { + var queuesConfig = new JobHostQueuesConfiguration + { + // configure a non-zero visibility timeout + VisibilityTimeout = TimeSpan.FromMinutes(5) + }; + QueueProcessorFactoryContext context = new QueueProcessorFactoryContext(_queue, _trace, queuesConfig, _poisonQueue); + QueueProcessor localProcessor = new QueueProcessor(context); + + string messageContent = Guid.NewGuid().ToString(); + CloudQueueMessage message = new CloudQueueMessage(messageContent); + await _queue.AddMessageAsync(message, CancellationToken.None); + + var functionResult = new FunctionResult(false); + message = await _queue.GetMessageAsync(); + await localProcessor.CompleteProcessingMessageAsync(message, functionResult, CancellationToken.None); + + var delta = message.NextVisibleTime - DateTime.UtcNow; + Assert.True(delta.Value.TotalMinutes > 4); + } + public class TestFixture : IDisposable { private const string TestQueuePrefix = "queueprocessortests"; diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueTests.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueTests.cs index 09a91a791..9ee4d6311 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueTests.cs @@ -13,6 +13,7 @@ using Microsoft.Azure.WebJobs.Host.TestCommon; using Newtonsoft.Json; using System.Reflection; +using Microsoft.Azure.WebJobs.Host.Executors; namespace Microsoft.Azure.WebJobs.Host.FunctionalTests { @@ -187,9 +188,9 @@ public void Func([Queue(QueueName)] out string x) public void Fails_When_No_Storage_is_set() { var host = TestHelpers.NewJobHost(); // no storage account! - + string message = StorageAccountParser.FormatParseAccountErrorMessage(StorageAccountParseResult.MissingOrEmptyConnectionStringError, "Storage"); TestHelpers.AssertIndexingError(() => host.Call("Func"), - "ProgramSimple.Func", "Unable to bind Queue because no storage account has been configured."); + "ProgramSimple.Func", message); } public class ProgramBadContract diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueTriggerTests.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueTriggerTests.cs index 00a684879..616f2c041 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueTriggerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Queues/QueueTriggerTests.cs @@ -13,6 +13,7 @@ using Microsoft.WindowsAzure.Storage.Queue; using Newtonsoft.Json; using Xunit; +using System.Text.RegularExpressions; namespace Microsoft.Azure.WebJobs.Host.FunctionalTests { @@ -191,11 +192,12 @@ public void QueueTrigger_IfBoundToPocoAndMessageIsNotJson_DoesNotBind() Exception innerException = exception.InnerException; Assert.IsType(innerException); const string expectedInnerMessage = "Binding parameters to complex objects (such as 'Poco') uses " + - "Json.NET serialization. \r\n1. Bind the parameter type as 'string' instead of 'Poco' to get the raw " + - "values and avoid JSON deserialization, or\r\n2. Change the queue payload to be valid json. The JSON " + + "Json.NET serialization. 1. Bind the parameter type as 'string' instead of 'Poco' to get the raw " + + "values and avoid JSON deserialization, or2. Change the queue payload to be valid json. The JSON " + "parser failed: Unexpected character encountered while parsing value: n. Path '', line 0, position " + - "0.\r\n"; - Assert.Equal(expectedInnerMessage, innerException.Message); + "0."; + string actual = Regex.Replace(innerException.Message, @"[\n\r]", ""); + Assert.Equal(expectedInnerMessage, actual); } [Fact] @@ -218,11 +220,12 @@ public void QueueTrigger_IfBoundToPocoAndMessageIsIncompatibleJson_DoesNotBind() Exception innerException = exception.InnerException; Assert.IsType(innerException); string expectedInnerMessage = "Binding parameters to complex objects (such as 'Poco') uses Json.NET " + - "serialization. \r\n1. Bind the parameter type as 'string' instead of 'Poco' to get the raw values " + - "and avoid JSON deserialization, or\r\n2. Change the queue payload to be valid json. The JSON parser " + + "serialization. 1. Bind the parameter type as 'string' instead of 'Poco' to get the raw values " + + "and avoid JSON deserialization, or2. Change the queue payload to be valid json. The JSON parser " + "failed: Error converting value 123 to type '" + typeof(Poco).FullName + "'. Path '', line 1, " + - "position 3.\r\n"; - Assert.Equal(expectedInnerMessage, innerException.Message); + "position 3."; + string actual = Regex.Replace(innerException.Message, @"[\n\r]", ""); + Assert.Equal(expectedInnerMessage, actual); } [Fact] @@ -492,9 +495,11 @@ public void QueueTrigger_IfDequeueCountReachesMaxDequeueCount_MovesToPoisonQueue new string[] { typeof(MaxDequeueCountProgram).FullName + ".PutInPoisonQueue" }); // Assert - IStorageAccountProvider storageAccountProvider = new FakeStorageAccountProvider(); + IStorageAccountProvider storageAccountProvider = new FakeStorageAccountProvider() + { + StorageAccount = new FakeStorageAccount() + }; Assert.Equal(new FakeQueueConfiguration(storageAccountProvider).MaxDequeueCount, MaxDequeueCountProgram.DequeueCount); - } finally { diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TableTests.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TableTests.cs index dea8e37e4..452e0648b 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TableTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TableTests.cs @@ -14,11 +14,12 @@ using Microsoft.Azure.WebJobs.Host.Storage.Queue; using Microsoft.Azure.WebJobs.Host.Storage.Table; using Microsoft.Azure.WebJobs.Host.Tables; +using Microsoft.Azure.WebJobs.Host.TestCommon; using Microsoft.WindowsAzure.Storage.Queue; using Microsoft.WindowsAzure.Storage.Table; using Newtonsoft.Json; -using Xunit; using Newtonsoft.Json.Linq; +using Xunit; namespace Microsoft.Azure.WebJobs.Host.FunctionalTests { @@ -30,20 +31,90 @@ public class TableTests private const string RowKey = "RK"; private const string PropertyName = "Property"; + [Fact] + public void Table_IndexingFails() + { + // Verify we catch various indexing failures. + Utility.AssertIndexingError("Run", "'$$' is not a valid name for an Azure table"); + + // Pocos must have a default ctor. + Utility.AssertIndexingError("Run", "Table entity types must provide a default constructor."); + + // When binding to Pocos, they must be structurally compatible with ITableEntity. + Utility.AssertIndexingError("Run", "Table entity types must implement the property RowKey."); + Utility.AssertIndexingError("Run", "Table entity types must implement the property RowKey."); + Utility.AssertIndexingError("Run", "Table entity types must implement the property PartitionKey."); + } + + class BindToSingleOutProgram + { + public static void Run([Table(TableName)] out Poco x) + { + x = new Poco { PartitionKey = PartitionKey, RowKey = RowKey, Property = "1234" }; + } + } + + [Fact] + public void Table_SingleOut_Supported() + { + IStorageAccount account = new FakeStorageAccount(); + var host = TestHelpers.NewJobHost(account); + + host.Call("Run"); + + AssertStringProperty(account, "Property", "1234"); + } + + // Helper to demonstrate that TableName property can include { } pairs. + private class BindToICollectorITableEntityResolvedTableProgram + { + public static void Run( + [Table("Ta{t1}")] ICollector table1, + [Table("{t1}x{t1}")] ICollector table2) + { + table1.Add(new Poco { PartitionKey = PartitionKey, RowKey = RowKey, Property = "123" }); + table2.Add(new Poco { PartitionKey = PartitionKey, RowKey = RowKey, Property = "456" }); + } + } + + // TableName can have { } pairs. + [Fact] + public void Table_ResolvedName() + { + IStorageAccount account = new FakeStorageAccount(); + var host = TestHelpers.NewJobHost(account); + + host.Call("Run", new { t1 = "ZZ" }); + + AssertStringProperty(account, "Property", "123", "TaZZ"); + AssertStringProperty(account, "Property", "456", "ZZxZZ"); + } + + private class CustomTableBindingConverter + : IConverter> + { + public CustomTableBinding Convert(CloudTable input) + { + return new CustomTableBinding(input); + } + } + [Fact] public void Table_IfBoundToCustomTableBindingExtension_BindsCorrectly() { // Arrange IStorageAccount account = CreateFakeStorageAccount(); - IStorageQueue triggerQueue = CreateQueue(account, TriggerQueueName); - triggerQueue.AddMessage(triggerQueue.CreateMessage("ignore")); - // register our custom table binding extension provider - DefaultExtensionRegistry extensions = new DefaultExtensionRegistry(); - extensions.RegisterExtension>(new CustomTableArgumentBindingProvider()); + var config = TestHelpers.NewConfig(typeof(CustomTableBindingExtensionProgram), account); - // Act - RunTrigger(account, typeof(CustomTableBindingExtensionProgram), extensions); + IConverterManager cm = config.GetService(); + + // Add a rule for binding CloudTable --> CustomTableBinding + cm.AddConverter, TableAttribute>( + typeof(CustomTableBindingConverter<>)); + + var host = new TestJobHost(config); + host.Call("Run"); // Act // Assert Assert.Equal(TableName, CustomTableBinding.Table.Name); @@ -72,7 +143,7 @@ public void Table_IfBoundToCloudTable_BindsAndCreatesTable() } [Fact] - public void Table_IfBoundToICollectorJObject_AddInsertsEntity() + public void Table_IfBoundToICollectorJObject_AddInsertsEntity() { // Arrange const string expectedValue = "abc"; @@ -92,7 +163,24 @@ public void Table_IfBoundToICollectorJObject_AddInsertsEntity() Assert.NotNull(entity.Properties); AssertPropertyValue(entity, "ValueStr", "abcdef"); - AssertPropertyValue(entity, "ValueNum", 123); + AssertPropertyValue(entity, "ValueNum", 123); + } + + // Partition and RowKey values are in the attribute + [Fact] + public void Table_IfBoundToICollectorJObject__WithAttrKeys_AddInsertsEntity() + { + // Arrange + const string expectedValue = "abcdef"; + IStorageAccount account = CreateFakeStorageAccount(); + var config = TestHelpers.NewConfig(typeof(BindToICollectorJObjectProgramKeysInAttr), account); + + // Act + var host = new TestJobHost(config); + host.Call("Run"); + + // Assert + AssertStringProperty(account, "ValueStr", expectedValue); } [Fact] @@ -108,17 +196,7 @@ public void Table_IfBoundToICollectorITableEntity_AddInsertsEntity() RunTrigger(account, typeof(BindToICollectorITableEntityProgram)); // Assert - IStorageTableClient client = account.CreateTableClient(); - IStorageTable table = client.GetTableReference(TableName); - Assert.True(table.Exists()); - DynamicTableEntity entity = table.Retrieve(PartitionKey, RowKey); - Assert.NotNull(entity); - Assert.NotNull(entity.Properties); - Assert.True(entity.Properties.ContainsKey(PropertyName)); - EntityProperty property = entity.Properties[PropertyName]; - Assert.NotNull(property); - Assert.Equal(EdmType.String, property.PropertyType); - Assert.Equal(expectedValue, property.StringValue); + AssertStringProperty(account, PropertyName, expectedValue); } [Fact] @@ -134,17 +212,7 @@ public void Table_IfBoundToICollectorPoco_AddInsertsEntity() RunTrigger(account, typeof(BindToICollectorPocoProgram)); // Assert - IStorageTableClient client = account.CreateTableClient(); - IStorageTable table = client.GetTableReference(TableName); - Assert.True(table.Exists()); - DynamicTableEntity entity = table.Retrieve(PartitionKey, RowKey); - Assert.NotNull(entity); - Assert.NotNull(entity.Properties); - Assert.True(entity.Properties.ContainsKey(PropertyName)); - EntityProperty property = entity.Properties[PropertyName]; - Assert.NotNull(property); - Assert.Equal(EdmType.String, property.PropertyType); - Assert.Equal(expectedValue, property.StringValue); + AssertStringProperty(account, PropertyName, expectedValue); } [Fact] @@ -314,7 +382,8 @@ private static void AssertPropertyValue(DynamicTableEntity entity, string proper Assert.Equal(EdmType.Int32, property.PropertyType); Assert.Equal(expectedValue, property.Int32Value); } - else { + else + { Assert.False(true, "test bug: unsupported property type: " + expectedValue.GetType().FullName); } } @@ -348,6 +417,29 @@ private static void AssertPropertyNull(EdmType expectedType, Assert.False(actualValue.HasValue); } + // Assert the given table has the given entity with PropertyName=ExpectedValue + void AssertStringProperty( + IStorageAccount account, + string propertyName, + string expectedValue, + string tableName = TableName, + string partitionKey = PartitionKey, + string rowKey = RowKey) + { + // Assert + IStorageTableClient client = account.CreateTableClient(); + IStorageTable table = client.GetTableReference(tableName); + Assert.True(table.Exists()); + DynamicTableEntity entity = table.Retrieve(partitionKey, rowKey); + Assert.NotNull(entity); + Assert.NotNull(entity.Properties); + Assert.True(entity.Properties.ContainsKey(propertyName)); + EntityProperty property = entity.Properties[propertyName]; + Assert.NotNull(property); + Assert.Equal(EdmType.String, property.PropertyType); + Assert.Equal(expectedValue, property.StringValue); + } + private static IStorageAccount CreateFakeStorageAccount() { return new FakeStorageAccount(); @@ -383,16 +475,33 @@ public static void Run([QueueTrigger(TriggerQueueName)] CloudQueueMessage ignore } } - private class BindToICollectorJObjectProgram + private class BindToICollectorJObjectProgram { public static void Run([QueueTrigger(TriggerQueueName)] CloudQueueMessage message, [Table(TableName)] ICollector table) { - table.Add(JObject.FromObject( new { + table.Add(JObject.FromObject(new + { PartitionKey = PartitionKey, RowKey = RowKey, ValueStr = "abcdef", - ValueNum = 123 + ValueNum = 123 + })); + } + } + + // Partition and RowKey are missing from JObject, get them from the attribute. + private class BindToICollectorJObjectProgramKeysInAttr + { + [NoAutomaticTrigger] + public static void Run( + [Table(TableName, PartitionKey, RowKey)] ICollector table) + { + table.Add(JObject.FromObject(new + { + // no partition and row key! USe from attribute instead. + ValueStr = "abcdef", + ValueNum = 123 })); } } @@ -442,8 +551,7 @@ public static void Run([QueueTrigger(TriggerQueueName)] CloudQueueMessage ignore private class CustomTableBindingExtensionProgram { - public static void Run([QueueTrigger(TriggerQueueName)] CloudQueueMessage ignore, - [Table(TableName)] CustomTableBinding table) + public static void Run([Table(TableName)] CustomTableBinding table) { Poco entity = new Poco(); table.Add(entity); @@ -451,6 +559,94 @@ public static void Run([QueueTrigger(TriggerQueueName)] CloudQueueMessage ignore } } + private class BadProgramTableName + { + public static void Run([Table("$$")] ICollector output) + { + Assert.True(false, "should have gotten error at indexing time."); + } + } + + private class BadProgram1 + { + public static void Run([Table(TableName)] ICollector output) + { + Assert.True(false, "should have gotten error at indexing time."); + } + } + + private class BadProgram2 + { + public static void Run([Table(TableName)] ICollector output) + { + Assert.True(false, "should have gotten error at indexing time."); + } + } + + private class BadProgram3 + { + public static void Run([Table(TableName)] ICollector output) + { + Assert.True(false, "should have gotten error at indexing time."); + } + } + + private class BadProgram4 + { + public static void Run([Table(TableName, PartitionKey, RowKey)] BadPocoMissingDefaultCtor input) + { + Assert.True(false, "should have gotten error at indexing time."); + } + } + + // Poco that should fail at binding time: + // 1. Does not derive from ITableEntity, and + // 2. Missing PartitionKey and RowKey values, so not structurally compatible with ITableEntity + private class BadPoco + { + public string Value { get; set; } + } + + private class BadPocoMissingRowKey + { + public string PartitionKey { get; set; } + public string Value { get; set; } + } + + private class BadPocoMissingPartitionKey + { + public string RowKey { get; set; } + public string Value { get; set; } + } + + private class BadPocoMissingDefaultCtor + { + public BadPocoMissingDefaultCtor(string value) + { + this.Value = value; + } + public string Value { get; set; } + } + + private class TableOutProgram + { + public static void Run([Table(TableName, PartitionKey, RowKey)] out Poco value) + { + value = null; + Assert.True(false, "should have gotten error at indexing time."); + } + } + + private class TableOutArrayProgram + { + public static void Run([Table(TableName, PartitionKey, RowKey)] out Poco[] value) + { + value = null; + Assert.True(false, "should have gotten error at indexing time."); + } + } + + private class Poco { public string PartitionKey { get; set; } @@ -537,90 +733,5 @@ internal Task FlushAsync(CancellationToken cancellationToken) return Task.FromResult(true); } } - - /// - /// Demonstrates an example binding extension provider for Tables - /// - private class CustomTableArgumentBindingProvider : IArgumentBindingProvider - { - public ITableArgumentBinding TryCreate(ParameterInfo parameter) - { - // Determine whether the target is a Table paramter that we should bind to - TableAttribute tableAttribute = parameter.GetCustomAttribute(inherit: false); - if (tableAttribute == null || - !parameter.ParameterType.IsGenericType || - (parameter.ParameterType.GetGenericTypeDefinition() != typeof(CustomTableBinding<>))) - { - return null; - } - - // create the binding - Type elementType = GetItemType(parameter.ParameterType); - Type bindingType = typeof(CustomTableBindingExtension<>).MakeGenericType(elementType); - - return (ITableArgumentBinding)Activator.CreateInstance(bindingType); - } - - private static Type GetItemType(Type queryableType) - { - Type[] genericArguments = queryableType.GetGenericArguments(); - var itemType = genericArguments[0]; - return itemType; - } - - /// - /// Custom Table binding extension, responsible for binding a to a - /// - /// - /// - private class CustomTableBindingExtension : ITableArgumentBinding - { - public FileAccess Access - { - get { return FileAccess.ReadWrite; } - } - - public Type ValueType { get { return typeof(CustomTableBinding); } } - - public Task BindAsync(CloudTable value, ValueBindingContext context) - { - return Task.FromResult(new CustomTableValueBinder(value, ValueType)); - } - - private class CustomTableValueBinder : IValueBinder - { - private readonly CloudTable _table; - private readonly Type _valueType; - - public CustomTableValueBinder(CloudTable table, Type valueType) - { - _table = table; - _valueType = valueType; - } - - public Type Type - { - get { return typeof(CustomTableBinding); } - } - - public object GetValue() - { - return new CustomTableBinding(_table); - } - - public Task SetValueAsync(object value, CancellationToken cancellationToken) - { - // this is where any queued up storage operations can be flushed - CustomTableBinding tableBinding = value as CustomTableBinding; - return tableBinding.FlushAsync(cancellationToken); - } - - public string ToInvokeString() - { - return _table.Name; - } - } - } - } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeJobHostContextFactory.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeJobHostContextFactory.cs index f6a62ccc8..d188e9414 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeJobHostContextFactory.cs +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeJobHostContextFactory.cs @@ -49,7 +49,7 @@ public Task CreateAndLogHostStartedAsync(JobHost host, Cancellat }; return JobHostContextFactory.CreateAndLogHostStartedAsync( - host, StorageAccountProvider, QueueConfiguration, TypeLocator, DefaultJobActivator.Instance, nameResolver, + host, StorageAccountProvider, QueueConfiguration, config.Blobs, TypeLocator, DefaultJobActivator.Instance, nameResolver, ConsoleProvider, new JobHostConfiguration(), shutdownToken, cancellationToken, BackgroundExceptionDispatcher, HostIdProvider, FunctionExecutor, FunctionIndexProvider, BindingProvider, HostInstanceLoggerProvider, FunctionInstanceLoggerProvider, FunctionOutputLoggerProvider); diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeQueueConfiguration.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeQueueConfiguration.cs index a85df535f..1b09f85a3 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeQueueConfiguration.cs +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeQueueConfiguration.cs @@ -44,6 +44,11 @@ public int MaxDequeueCount get { return 3; } } + public TimeSpan VisibilityTimeout + { + get { return TimeSpan.Zero; } + } + public IQueueProcessorFactory QueueProcessorFactory { get @@ -91,14 +96,14 @@ public FakeQueueProcessor(QueueProcessorFactoryContext context, IStorageAccount } } - protected override async Task CopyMessageToPoisonQueueAsync(CloudQueueMessage message, CancellationToken cancellationToken) + protected override async Task CopyMessageToPoisonQueueAsync(CloudQueueMessage message, CloudQueue poisonQueue, CancellationToken cancellationToken) { FakeStorageQueueMessage fakeMessage = new FakeStorageQueueMessage(message); await _poisonQueue.CreateIfNotExistsAsync(cancellationToken); await _poisonQueue.AddMessageAsync(fakeMessage, cancellationToken); - OnMessageAddedToPoisonQueue(EventArgs.Empty); + OnMessageAddedToPoisonQueue(new PoisonMessageEventArgs(message, poisonQueue)); } protected override async Task ReleaseMessageAsync(CloudQueueMessage message, FunctionResult result, TimeSpan visibilityTimeout, CancellationToken cancellationToken) diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeStorageAccountProvider.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeStorageAccountProvider.cs index 3cf0db91e..b9f6d9d8f 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeStorageAccountProvider.cs +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/TestDoubles/FakeStorageAccountProvider.cs @@ -14,7 +14,7 @@ internal class FakeStorageAccountProvider : IStorageAccountProvider public IStorageAccount DashboardAccount { get; set; } - public Task GetAccountAsync(string connectionStringName, CancellationToken cancellationToken) + public Task TryGetAccountAsync(string connectionStringName, CancellationToken cancellationToken) { IStorageAccount account; diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Utility.cs b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Utility.cs new file mode 100644 index 000000000..57173dbaf --- /dev/null +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/Utility.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.WebJobs.Host.FunctionalTests.TestDoubles; +using Microsoft.Azure.WebJobs.Host.Indexers; +using Microsoft.Azure.WebJobs.Host.Storage; +using Microsoft.Azure.WebJobs.Host.TestCommon; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Host.FunctionalTests +{ + static class Utility + { + // Helper for quickly testing indexing errors + public static void AssertIndexingError(string methodName, string expectedErrorMessage) + { + // Need to pass an account to get passed initial validation checks. + IStorageAccount account = new FakeStorageAccount(); + var host = TestHelpers.NewJobHost(account); + + host.AssertIndexingError(methodName, expectedErrorMessage); + } + } +} diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/WebJobs.Host.FunctionalTests.csproj b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/WebJobs.Host.FunctionalTests.csproj index 0374f2668..fad5ad635 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/WebJobs.Host.FunctionalTests.csproj +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/WebJobs.Host.FunctionalTests.csproj @@ -14,6 +14,9 @@ 512 + ..\test.ruleset + true + false true @@ -24,7 +27,7 @@ prompt 4 true - 5 + default pdbonly @@ -34,7 +37,7 @@ prompt 4 true - 5 + default true @@ -112,6 +115,7 @@ + @@ -130,15 +134,15 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True @@ -149,8 +153,8 @@ ..\..\packages\WindowsAzure.Storage.7.2.1\lib\net40\Microsoft.WindowsAzure.Storage.dll True - - ..\..\packages\Moq.4.5.23\lib\net45\Moq.dll + + ..\..\packages\Moq.4.5.30\lib\net45\Moq.dll True @@ -161,7 +165,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True diff --git a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/packages.config b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/packages.config index c6fbf9926..3de92605c 100644 --- a/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/packages.config +++ b/test/Microsoft.Azure.WebJobs.Host.FunctionalTests/packages.config @@ -2,13 +2,17 @@ - - - + + + - + - + + + + + diff --git a/test/Microsoft.Azure.WebJobs.Host.TestCommon/JobHostFactory.cs b/test/Microsoft.Azure.WebJobs.Host.TestCommon/JobHostFactory.cs index 2c19a2873..ea51f8960 100644 --- a/test/Microsoft.Azure.WebJobs.Host.TestCommon/JobHostFactory.cs +++ b/test/Microsoft.Azure.WebJobs.Host.TestCommon/JobHostFactory.cs @@ -46,6 +46,7 @@ public static TestJobHost Create(CloudStorageAccount storage IHostIdProvider hostIdProvider = new FixedHostIdProvider("test"); INameResolver nameResolver = null; IQueueConfiguration queueConfiguration = new SimpleQueueConfiguration(maxDequeueCount); + JobHostBlobsConfiguration blobsConfiguration = new JobHostBlobsConfiguration(); ContextAccessor messageEnqueuedWatcherAccessor = new ContextAccessor(); ContextAccessor blobWrittenWatcherAccessor = @@ -63,11 +64,11 @@ public static TestJobHost Create(CloudStorageAccount storage IFunctionExecutor executor = new FunctionExecutor(new NullFunctionInstanceLogger(), outputLogger, exceptionHandler, trace); var triggerBindingProvider = DefaultTriggerBindingProvider.Create( - nameResolver, storageAccountProvider, extensionTypeLocator, hostIdProvider, queueConfiguration, + nameResolver, storageAccountProvider, extensionTypeLocator, hostIdProvider, queueConfiguration, blobsConfiguration, exceptionHandler, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, sharedContextProvider, extensions, singletonManager, new TestTraceWriter(TraceLevel.Verbose)); - var bindingProvider = DefaultBindingProvider.Create(nameResolver, storageAccountProvider, extensionTypeLocator, + var bindingProvider = DefaultBindingProvider.Create(nameResolver, null, storageAccountProvider, extensionTypeLocator, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, extensions); var functionIndexProvider = new FunctionIndexProvider(new FakeTypeLocator(typeof(TProgram)), triggerBindingProvider, bindingProvider, DefaultJobActivator.Instance, executor, new DefaultExtensionRegistry(), singletonManager, trace); diff --git a/test/Microsoft.Azure.WebJobs.Host.TestCommon/SimpleQueueConfiguration.cs b/test/Microsoft.Azure.WebJobs.Host.TestCommon/SimpleQueueConfiguration.cs index bb1303eea..cf0824ad1 100644 --- a/test/Microsoft.Azure.WebJobs.Host.TestCommon/SimpleQueueConfiguration.cs +++ b/test/Microsoft.Azure.WebJobs.Host.TestCommon/SimpleQueueConfiguration.cs @@ -38,6 +38,11 @@ public int MaxDequeueCount get { return _maxDequeueCount; } } + public TimeSpan VisibilityTimeout + { + get { return TimeSpan.Zero; } + } + public IQueueProcessorFactory QueueProcessorFactory { get diff --git a/test/Microsoft.Azure.WebJobs.Host.TestCommon/SimpleStorageAccountProvider.cs b/test/Microsoft.Azure.WebJobs.Host.TestCommon/SimpleStorageAccountProvider.cs index 72fa8b837..b39fd239f 100644 --- a/test/Microsoft.Azure.WebJobs.Host.TestCommon/SimpleStorageAccountProvider.cs +++ b/test/Microsoft.Azure.WebJobs.Host.TestCommon/SimpleStorageAccountProvider.cs @@ -23,7 +23,7 @@ public SimpleStorageAccountProvider(IServiceProvider services) public CloudStorageAccount DashboardAccount { get; set; } - Task IStorageAccountProvider.GetAccountAsync(string connectionStringName, CancellationToken cancellationToken) + Task IStorageAccountProvider.TryGetAccountAsync(string connectionStringName, CancellationToken cancellationToken) { IStorageAccount account; diff --git a/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestHelpers.cs b/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestHelpers.cs index ca41b00ca..19e720097 100644 --- a/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestHelpers.cs +++ b/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestHelpers.cs @@ -121,7 +121,7 @@ params object[] services config.JobActivator = jobActivator; continue; } - + IExtensionConfigProvider extension = obj as IExtensionConfigProvider; if (extension != null) { @@ -141,7 +141,7 @@ private class FakeStorageAccountProvider : IStorageAccountProvider public IStorageAccount DashboardAccount { get; set; } - public Task GetAccountAsync(string connectionStringName, CancellationToken cancellationToken) + public Task TryGetAccountAsync(string connectionStringName, CancellationToken cancellationToken) { IStorageAccount account; diff --git a/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestJobHost.cs b/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestJobHost.cs index f770102da..52c361aab 100644 --- a/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestJobHost.cs +++ b/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestJobHost.cs @@ -5,12 +5,14 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Indexers; +using Xunit; namespace Microsoft.Azure.WebJobs.Host.TestCommon { public class TestJobHost : JobHost { - internal TestJobHost(IServiceProvider serviceProvider) + public TestJobHost(IServiceProvider serviceProvider) : base(serviceProvider) { } @@ -40,5 +42,23 @@ public Task CallAsync(string methodName, object arguments) { return base.CallAsync(typeof(TProgram).GetMethod(methodName), arguments); } + + // Helper for quickly testing indexing errors + public void AssertIndexingError(string methodName, string expectedErrorMessage) + { + try + { + // Indexing is lazy, so must actually try a call first. + this.Call(methodName); + } + catch (FunctionIndexingException e) + { + string functionName = typeof(TProgram).Name + "." + methodName; + Assert.Equal("Error indexing method '" + functionName + "'", e.Message); + Assert.Equal(expectedErrorMessage, e.InnerException.Message); + return; + } + Assert.True(false, "Invoker should have failed"); + } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestJobHostContextFactory.cs b/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestJobHostContextFactory.cs index a16aec5ed..535dde170 100644 --- a/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestJobHostContextFactory.cs +++ b/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestJobHostContextFactory.cs @@ -36,8 +36,8 @@ public Task CreateAndLogHostStartedAsync(JobHost host, Cancellat }; return JobHostContextFactory.CreateAndLogHostStartedAsync( - host, StorageAccountProvider, Queues, typeLocator, DefaultJobActivator.Instance, nameResolver, - new NullConsoleProvider(), new JobHostConfiguration(), shutdownToken, cancellationToken, new WebJobsExceptionHandler(), + host, StorageAccountProvider, Queues, config.Blobs, typeLocator, DefaultJobActivator.Instance, nameResolver, + new NullConsoleProvider(), config, shutdownToken, cancellationToken, new WebJobsExceptionHandler(), functionIndexProvider: FunctionIndexProvider, singletonManager: SingletonManager, hostIdProvider: HostIdProvider); } } diff --git a/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestTraceWriter.cs b/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestTraceWriter.cs index aa9fa39a7..c11f69be7 100644 --- a/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestTraceWriter.cs +++ b/test/Microsoft.Azure.WebJobs.Host.TestCommon/TestTraceWriter.cs @@ -9,6 +9,7 @@ namespace Microsoft.Azure.WebJobs.Host.TestCommon public class TestTraceWriter : TraceWriter { public Collection Traces = new Collection(); + private object _syncLock = new object(); public TestTraceWriter(TraceLevel level) : base(level) { @@ -16,7 +17,10 @@ public TestTraceWriter(TraceLevel level) : base(level) public override void Trace(TraceEvent traceEvent) { - Traces.Add(traceEvent); + lock (_syncLock) + { + Traces.Add(traceEvent); + } } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.TestCommon/WebJobs.Host.TestCommon.csproj b/test/Microsoft.Azure.WebJobs.Host.TestCommon/WebJobs.Host.TestCommon.csproj index 74a994220..0d5192cc3 100644 --- a/test/Microsoft.Azure.WebJobs.Host.TestCommon/WebJobs.Host.TestCommon.csproj +++ b/test/Microsoft.Azure.WebJobs.Host.TestCommon/WebJobs.Host.TestCommon.csproj @@ -14,6 +14,9 @@ 512 + ..\test.ruleset + true + false true @@ -24,7 +27,7 @@ prompt 4 true - 5 + default pdbonly @@ -34,7 +37,7 @@ prompt 4 true - 5 + default true @@ -51,15 +54,15 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True @@ -77,7 +80,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True diff --git a/test/Microsoft.Azure.WebJobs.Host.TestCommon/packages.config b/test/Microsoft.Azure.WebJobs.Host.TestCommon/packages.config index 18c9e34a8..eb90baf1a 100644 --- a/test/Microsoft.Azure.WebJobs.Host.TestCommon/packages.config +++ b/test/Microsoft.Azure.WebJobs.Host.TestCommon/packages.config @@ -1,12 +1,16 @@  - - - + + + - + + + + + diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/AttributeClonerTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/AttributeClonerTests.cs index 6d478e6d2..76f982684 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/AttributeClonerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/AttributeClonerTests.cs @@ -4,9 +4,11 @@ using System; using System.Collections.Generic; using System.IO; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; +using Microsoft.Azure.WebJobs.Host.Bindings.Path; using Microsoft.Azure.WebJobs.Host.TestCommon; using Xunit; @@ -42,6 +44,49 @@ public Attr2(string resolvedProp2, string constantProp) public string ResolvedSetting { get; set; } } + public class AttributeWithResolutionPolicy : Attribute + { + [AutoResolve(ResolutionPolicyType = typeof(TestResolutionPolicy))] + public string PropWithPolicy { get; set; } + + [AutoResolve] + public string PropWithoutPolicy { get; set; } + + [AutoResolve(ResolutionPolicyType = typeof(WebJobs.ODataFilterResolutionPolicy))] + public string PropWithMarkerPolicy { get; set; } + + [AutoResolve(ResolutionPolicyType = typeof(AutoResolveAttribute))] + public string PropWithInvalidPolicy { get; set; } + + [AutoResolve(ResolutionPolicyType = typeof(NoDefaultConstructorResolutionPolicy))] + public string PropWithConstructorlessPolicy { get; set; } + + internal string ResolutionData { get; set; } + } + + public class TestResolutionPolicy : IResolutionPolicy + { + public string TemplateBind(PropertyInfo propInfo, Attribute attribute, BindingTemplate template, IReadOnlyDictionary bindingData) + { + // set some internal state for the binding rules to use later + ((AttributeWithResolutionPolicy)attribute).ResolutionData = "value1"; + + return template.Bind(bindingData); + } + } + + public class NoDefaultConstructorResolutionPolicy : IResolutionPolicy + { + public NoDefaultConstructorResolutionPolicy(string someValue) + { + } + + public string TemplateBind(PropertyInfo propInfo, Attribute attribute, BindingTemplate template, IReadOnlyDictionary bindingData) + { + throw new NotImplementedException(); + } + } + // Helper to easily generate a fixed binding contract. private static IReadOnlyDictionary GetBindingContract(params string[] names) { @@ -118,8 +163,7 @@ public async Task InvokeStringMultipleResolvedProperties() var cloner = new AttributeCloner(attr, GetBindingContract("p1", "p2")); - Attr2 attrResolved = cloner.ResolveFromBindings(new Dictionary { - { "p1", "v1" }, { "p2", "v2" }}); + Attr2 attrResolved = cloner.ResolveFromBindings(new Dictionary { { "p1", "v1" }, { "p2", "v2" } }); Assert.Equal("v1", attrResolved.ResolvedProp1); Assert.Equal("v2", attrResolved.ResolvedProp2); @@ -273,6 +317,73 @@ public void Fail_MissingPath() Assert.Throws(() => new AttributeCloner(attr, EmptyContract)); } + [Fact] + public void TryAutoResolveValue_UnresolvedValue_ThrowsExpectedException() + { + var resolver = new FakeNameResolver(); + var attribute = new Attr2(string.Empty, string.Empty) + { + ResolvedSetting = "MySetting" + }; + var prop = attribute.GetType().GetProperty("ResolvedSetting"); + string resolvedValue = null; + + var ex = Assert.Throws(() => AttributeCloner.TryAutoResolveValue(attribute, prop, resolver, out resolvedValue)); + Assert.Equal("Unable to resolve value for property 'Attr2.ResolvedSetting'.", ex.Message); + } + + [Fact] + public void GetPolicy_ReturnsDefault_WhenNoSpecifiedPolicy() + { + PropertyInfo propInfo = typeof(AttributeWithResolutionPolicy).GetProperty(nameof(AttributeWithResolutionPolicy.PropWithoutPolicy)); + + IResolutionPolicy policy = AttributeCloner.GetPolicy(propInfo); + + Assert.IsType(policy); + } + + [Fact] + public void GetPolicy_Returns_SpecifiedPolicy() + { + PropertyInfo propInfo = typeof(AttributeWithResolutionPolicy).GetProperty(nameof(AttributeWithResolutionPolicy.PropWithPolicy)); + + IResolutionPolicy policy = AttributeCloner.GetPolicy(propInfo); + + Assert.IsType(policy); + } + + [Fact] + public void GetPolicy_ReturnsODataFilterPolicy_ForMarkerType() + { + // This is a special-case marker type to handle TableAttribute.Filter. We cannot directly list ODataFilterResolutionPolicy + // because BindingTemplate doesn't exist in the core assembly. + PropertyInfo propInfo = typeof(AttributeWithResolutionPolicy).GetProperty(nameof(AttributeWithResolutionPolicy.PropWithMarkerPolicy)); + + IResolutionPolicy policy = AttributeCloner.GetPolicy(propInfo); + + Assert.IsType(policy); + } + + [Fact] + public void GetPolicy_Throws_IfPolicyDoesNotImplementInterface() + { + PropertyInfo propInfo = typeof(AttributeWithResolutionPolicy).GetProperty(nameof(AttributeWithResolutionPolicy.PropWithInvalidPolicy)); + + InvalidOperationException ex = Assert.Throws(() => AttributeCloner.GetPolicy(propInfo)); + + Assert.Equal($"The {nameof(AutoResolveAttribute.ResolutionPolicyType)} on {nameof(AttributeWithResolutionPolicy.PropWithInvalidPolicy)} must derive from {typeof(IResolutionPolicy).Name}.", ex.Message); + } + + [Fact] + public void GetPolicy_Throws_IfPolicyHasNoDefaultConstructor() + { + PropertyInfo propInfo = typeof(AttributeWithResolutionPolicy).GetProperty(nameof(AttributeWithResolutionPolicy.PropWithConstructorlessPolicy)); + + InvalidOperationException ex = Assert.Throws(() => AttributeCloner.GetPolicy(propInfo)); + + Assert.Equal($"The {nameof(AutoResolveAttribute.ResolutionPolicyType)} on {nameof(AttributeWithResolutionPolicy.PropWithConstructorlessPolicy)} must derive from {typeof(IResolutionPolicy).Name} and have a default constructor.", ex.Message); + } + private static BindingContext GetCtx(IReadOnlyDictionary values) { BindingContext ctx = new BindingContext( diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/BindingDataProviderTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/BindingDataProviderTests.cs index da06ce4ba..37d16ca60 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/BindingDataProviderTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/BindingDataProviderTests.cs @@ -126,6 +126,25 @@ public void GetBindingData_IfDateProperty_ReturnsValidBindingData() Assert.Equal(date, bindingData["date"]); } + [Fact] + public void GetBindingData_WithDerivedValue_ReturnsValidBindingData() + { + // Arrange + IBindingDataProvider provider = BindingDataProvider.FromType(typeof(Base)); + + Derived value = new Derived { A = 1, B = 2 }; + + // Act + var bindingData = provider.GetBindingData(value); + + // Assert + Assert.NotNull(bindingData); + + // Get binding data for the type used when creating the provider + Assert.Equal(1, bindingData.Count); + Assert.Equal(1, bindingData["a"]); + } + [Fact] public void FromTemplate_IgnoreCase_CreatesCaseInsensitiveProvider() { @@ -202,5 +221,10 @@ private class DerivedWithNew : Base { new public int A { get; set; } } + + private class Derived : Base + { + public int B { get; set; } + } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Data/DataBindingProviderTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Data/DataBindingProviderTests.cs index 13d273807..2b83a6764 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Data/DataBindingProviderTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Data/DataBindingProviderTests.cs @@ -38,13 +38,13 @@ public async Task Create_HandlesNullableTypes() }; var bindingContext = new BindingContext(valueBindingContext, bindingData); var valueProvider = await binding.BindAsync(bindingContext); - var value = valueProvider.GetValue(); + var value = await valueProvider.GetValueAsync(); Assert.Equal(123, value); bindingData["p"] = null; bindingContext = new BindingContext(valueBindingContext, bindingData); valueProvider = await binding.BindAsync(bindingContext); - value = valueProvider.GetValue(); + value = await valueProvider.GetValueAsync(); Assert.Null(value); } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Path/BindingParameterResolverTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Path/BindingParameterResolverTests.cs new file mode 100644 index 000000000..6b73439cc --- /dev/null +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Path/BindingParameterResolverTests.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using Microsoft.Azure.WebJobs.Host.Bindings.Path; +using System.Linq; +using Xunit; +using System; + +namespace Microsoft.Azure.WebJobs.Host.UnitTests.Bindings.Path +{ + public class BindingParameterResolverTests + { + [Theory] + [InlineData("rand-guid", true)] + [InlineData("RAND-GUID", true)] + [InlineData("datetime", true)] + [InlineData("foobar", false)] + public void IsSystemParameter_ReturnsExpectedValue(string value, bool expected) + { + Assert.Equal(expected, BindingParameterResolver.IsSystemParameter(value)); + } + + [Fact] + public void RandGuidResolver_ReturnsExpectedValue() + { + BindingParameterResolver resolver = null; + BindingParameterResolver.TryGetResolver("rand-guid", out resolver); + + string resolvedValue = resolver.Resolve("rand-guid"); + Assert.Equal(36, resolvedValue.Length); + Assert.Equal(4, resolvedValue.Count(p => p == '-')); + + resolvedValue = resolver.Resolve("rand-guid:"); // no format + Assert.Equal(36, resolvedValue.Length); + Assert.Equal(4, resolvedValue.Count(p => p == '-')); + + resolvedValue = resolver.Resolve("rand-guid:D"); + Assert.Equal(36, resolvedValue.Length); + Assert.Equal(4, resolvedValue.Count(p => p == '-')); + + resolvedValue = resolver.Resolve("rand-guid:N"); + Assert.Equal(32, resolvedValue.Length); + Assert.Equal(0, resolvedValue.Count(p => p == '-')); + + resolvedValue = resolver.Resolve("rand-guid:B"); + Assert.Equal(38, resolvedValue.Length); + Assert.Equal(4, resolvedValue.Count(p => p == '-')); + Assert.True(resolvedValue.StartsWith("{")); + Assert.True(resolvedValue.EndsWith("}")); + } + + [Fact] + public void IncompatibleBindingExpression_ThrowsArgumentException() + { + BindingParameterResolver resolver = null; + BindingParameterResolver.TryGetResolver("rand-guid", out resolver); + + var ex = Assert.Throws(() => + { + resolver.Resolve("datetime:mm-dd-yyyy"); + }); + + Assert.Equal("The value specified is not a 'rand-guid' binding parameter.\r\nParameter name: value", ex.Message); + } + + [Fact] + public void DateTimeResolver_ReturnsExpectedValue() + { + BindingParameterResolver resolver = null; + BindingParameterResolver.TryGetResolver("datetime", out resolver); + + string resolvedValue = resolver.Resolve("datetime"); + + resolvedValue = resolver.Resolve("datetime:G"); + Assert.NotNull(DateTime.Parse(resolvedValue)); + + resolvedValue = resolver.Resolve("datetime:MM/yyyy"); + Assert.Equal(DateTime.UtcNow.ToString("MM/yyyy"), resolvedValue); + + resolvedValue = resolver.Resolve("datetime:yyyyMMdd"); + Assert.Equal(DateTime.UtcNow.ToString("yyyyMMdd"), resolvedValue); + } + } +} diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Path/BindingTemplateParserTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Path/BindingTemplateParserTests.cs index 264811a01..2f5c47cb3 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Path/BindingTemplateParserTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Bindings/Path/BindingTemplateParserTests.cs @@ -4,12 +4,9 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings.Path; using Microsoft.Azure.WebJobs.Host.TestCommon; using Xunit; -using Xunit.Extensions; namespace Microsoft.Azure.WebJobs.Host.UnitTests.Bindings.Path { diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/BlobLogListenerTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/BlobLogListenerTests.cs new file mode 100644 index 000000000..be9dae04f --- /dev/null +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/BlobLogListenerTests.cs @@ -0,0 +1,51 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Azure.WebJobs.Host.Blobs; +using Microsoft.Azure.WebJobs.Host.Blobs.Listeners; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Host.UnitTests.Blobs.Listeners +{ + public class BlobLogListenerTests + { + [Fact] + public void GetPathsForValidBlobWrites_Returns_ValidBlobWritesOnly() + { + StorageAnalyticsLogEntry[] entries = new[] + { + // This is a valid write entry with a valid path + new StorageAnalyticsLogEntry + { + ServiceType = StorageServiceType.Blob, + OperationType = StorageServiceOperationType.PutBlob, + RequestedObjectKey = @"/storagesample/sample-container/""0x8D199A96CB71468""/sample-blob.txt" + }, + + // This is an invalid path and will be filtered out + new StorageAnalyticsLogEntry + { + ServiceType = StorageServiceType.Blob, + OperationType = StorageServiceOperationType.PutBlob, + RequestedObjectKey = "/" + }, + + // This does not constitute a write and will be filtered out + new StorageAnalyticsLogEntry + { + ServiceType = StorageServiceType.Blob, + OperationType = null, + RequestedObjectKey = @"/storagesample/sample-container/""0x8D199A96CB71468""/sample-blob.txt" + } + }; + + IEnumerable validPaths = BlobLogListener.GetPathsForValidBlobWrites(entries); + + BlobPath singlePath = validPaths.Single(); + Assert.Equal("sample-container", singlePath.ContainerName); + Assert.Equal(@"""0x8D199A96CB71468""/sample-blob.txt", singlePath.BlobName); + } + } +} diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageAnalyticsLogEntryTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageAnalyticsLogEntryTests.cs index 7981fb083..ef1352ad4 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageAnalyticsLogEntryTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Blobs/Listeners/StorageAnalyticsLogEntryTests.cs @@ -5,9 +5,7 @@ using System.Globalization; using Microsoft.Azure.WebJobs.Host.Blobs; using Microsoft.Azure.WebJobs.Host.Blobs.Listeners; -using Microsoft.Azure.WebJobs.Host.TestCommon; using Xunit; -using Xunit.Extensions; namespace Microsoft.Azure.WebJobs.Host.UnitTests.Blobs.Listeners { @@ -85,14 +83,12 @@ internal void ToBlobPath_IfValidBlobOperationEntry_ReturnsBlobPath(StorageServic [InlineData(@"/")] [InlineData(@"")] [InlineData(@"\\")] - public void ToBlobPath_IfMalformedObjectKey_ThrowsFormatException(string requestedObjectKey) + public void ToBlobPath_IfMalformedObjectKey_ReturnsNull(string requestedObjectKey) { StorageAnalyticsLogEntry entry = CreateEntry("2014-09-08T18:44:18.9681025Z", StorageServiceOperationType.PutBlob, StorageServiceType.Blob, requestedObjectKey); - ExceptionAssert.ThrowsFormat(() => entry.ToBlobPath(), "Failed to parse RequestedObjectKey property of the log entry. " + - "It should be in one of the supported formats: " + - @"""https://account.blob.core.windows.net/container/blob"", or" + - @"""/account/container/blob"""); + BlobPath blobPath = entry.ToBlobPath(); + Assert.Null(blobPath); } [Theory] @@ -106,13 +102,13 @@ public void ToBlobPath_IfMalformedUri_PropogatesUriFormatException(string reques } [Fact] - public void ToBlobPath_IfMalformedBlobPath_ThrowsFormatException() + public void ToBlobPath_IfMalformedBlobPath_ReturnsNull() { string requestedObjectKey = "/account/malformed-blob-path"; StorageAnalyticsLogEntry entry = CreateEntry("2014-09-08T18:44:18.9681025Z", StorageServiceOperationType.PutBlob, StorageServiceType.Blob, requestedObjectKey); - ExceptionAssert.ThrowsFormat(() => entry.ToBlobPath(), "Failed to parse RequestedObjectKey property of the log entry. " + - "Blob identifiers must be in the format container/blob."); + BlobPath blob = entry.ToBlobPath(); + Assert.Null(blob); } private static StorageAnalyticsLogEntry CreateEntry(string requestStartTime, StorageServiceOperationType operationType, StorageServiceType serviceType, string requestedObjectKey) diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/BindToGenericItemTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/BindToGenericItemTests.cs index ac7c93ac4..934ffbf72 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/BindToGenericItemTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/BindToGenericItemTests.cs @@ -8,96 +8,593 @@ using System.Threading.Tasks; using System.Reflection; using Microsoft.Azure.WebJobs.Host.Bindings; +using System.Threading; +using Newtonsoft.Json; namespace Microsoft.Azure.WebJobs.Host.UnitTests.Common { - // Test BindingFactory's BindToGenericItem rule. + // Test BindingFactory's BindToInput rule. + // Provide some basic types, converters, builders and make it very easy to test a + // variety of configuration permutations. + // Each Client configuration is its own test case. public class BindToGenericItemTests { - // Some custom type to bind to. - public class Widget + // Each of the TestConfigs below implement this. + interface ITest + { + void Test(TestJobHost host); + } + + // Simple case. + // Test with concrete types, no converters. + // Attr-->Widget + [Fact] + public void TestConcreteTypeNoConverter() + { + TestWorker(); + } + + public class ConfigConcreteTypeNoConverter : IExtensionConfigProvider, ITest { - public static Widget New(TestAttribute attrResolved) + public void Initialize(ExtensionConfigContext context) { - return new Widget { _value = attrResolved.Path }; + var bf = context.Config.BindingFactory; + var rule = bf.BindToInput(typeof(AlphaBuilder)); + context.RegisterBindingRules(rule); } - public string _value; + public void Test(TestJobHost host) + { + host.Call("Func", new { k = 1 }); + Assert.Equal("AlphaBuilder(1)", _log); + } + + string _log; + + // Input Rule (exact match): --> Widget + public void Func([Test("{k}")] AlphaType w) + { + _log = w._value; + } } - public class TestAttribute : Attribute + // Use OpenType (a general builder), still no converters. + [Fact] + public void TestOpenTypeNoConverters() { - public TestAttribute(string path) + TestWorker(); + } + + public class ConfigOpenTypeNoConverters : IExtensionConfigProvider, ITest + { + public void Initialize(ExtensionConfigContext context) { - this.Path = path; + var bf = context.Config.BindingFactory; + + // Replaces BindToGeneric + var rule = bf.BindToInput(typeof(GeneralBuilder<>)); + + context.RegisterBindingRules(rule); } + + public void Test(TestJobHost host) + { + host.Call("Func1", new { k = 1 }); + Assert.Equal("GeneralBuilder_AlphaType(1)", _log); - [AutoResolve] - public string Path { get; set; } + host.Call("Func2", new { k = 2 }); + Assert.Equal("GeneralBuilder_BetaType(2)", _log); + } + + string _log; + + // Input Rule (generic match): --> Widget + public void Func1([Test("{k}")] AlphaType w) + { + _log = w._value; + } + + // Input Rule (generic match): --> OtherType + public void Func2([Test("{k}")] BetaType w) + { + _log = w._value; + } } - public class FakeExtClient : IExtensionConfigProvider + [Fact] + public void TestWithConverters() + { + TestWorker(); + } + + public class ConfigWithConverters : IExtensionConfigProvider, ITest { public void Initialize(ExtensionConfigContext context) { var bf = context.Config.BindingFactory; - // Add [Test] support - var rule = bf.BindToGenericItem(Builder); + bf.ConverterManager.AddConverter(ConvertAlpha2Beta); + + // The AlphaType restriction here means that although we have a GeneralBuilder<> that *could* + // directly build a BetaType, we can only use it to build AlphaTypes, and so we must invoke the converter. + var rule = bf.BindToInput(typeof(GeneralBuilder<>)); + context.RegisterBindingRules(rule); } - private Task Builder(TestAttribute attrResolved, Type parameterType) + public void Test(TestJobHost host) + { + host.Call("Func1", new { k = 1 }); + Assert.Equal("GeneralBuilder_AlphaType(1)", _log); + + host.Call("Func2", new { k = 2 }); + Assert.Equal("A2B(GeneralBuilder_AlphaType(2))", _log); + } + + string _log; + + // Input Rule (exact match): --> Widget + public void Func1([Test("{k}")] AlphaType w) + { + _log = w._value; + } + + // Input Rule (match w/ converter) : --> Widget + // Converter: Widget --> OtherType + public void Func2([Test("{k}")] BetaType w) { - var method = parameterType.GetMethod("New", BindingFlags.Static | BindingFlags.Public); - var obj = method.Invoke(null, new object[] { attrResolved }); - return Task.FromResult(obj); + _log = w._value; } } - public class Program + // Test ordering. First rule wins. + [Fact] + public void TestMultipleRules() + { + TestWorker(); + } + + public class ConfigMultipleRules : IExtensionConfigProvider, ITest { - public string _value; + public void Initialize(ExtensionConfigContext context) + { + var bf = context.Config.BindingFactory; + var rule1 = bf.BindToInput(typeof(AlphaBuilder)); + var rule2 = bf.BindToInput(typeof(BetaBuilder)); + context.RegisterBindingRules(rule1, rule2); + } + + public void Test(TestJobHost host) + { + host.Call("Func", new { k = 1 }); + Assert.Equal("AlphaBuilder(1)", _log); + + host.Call("Func2", new { k = 1 }); + Assert.Equal("BetaBuilder(1)", _log); + } + + string _log; + + public void Func([Test("{k}")] AlphaType w) + { + _log = w._value; + } - public void Func([Test("abc-{k}")] Widget w) + // Input Rule (exact match): --> Widget + public void Func2([Test("{k}")] BetaType w) { - _value = w._value; + _log = w._value; } } + // Test binding to object with an explicit converter. [Fact] - public void Test() + public void TestExplicitObjectConverter() { - var prog = new Program(); + TestWorker(); + } + + public class ConfigExplicitObjectConverter : IExtensionConfigProvider, ITest, + IConverter + { + public void Initialize(ExtensionConfigContext context) + { + var bf = context.Config.BindingFactory; + + var cm = bf.ConverterManager; + // Have an explicit converter to object. + cm.AddConverter(this); + + var rule1 = bf.BindToInput(typeof(GeneralBuilder<>)); + context.RegisterBindingRules(rule1); + } + + public void Test(TestJobHost host) + { + // normal case + host.Call("Func", new { k = 1 }); + Assert.Equal("GeneralBuilder_AlphaType(1)", _log); + + // use 1st rule with explicit converter + host.Call("FuncObject", new { k = 1 }); + Assert.Equal("Alpha2Obj(GeneralBuilder_AlphaType(1))", _log); + } + + string _log; + + // builds AlphaDerivedType, and then applies an implicit inheritence converter. + public void Func([Test("{k}")] AlphaType w) + { + _log = w._value; + } + + // Invokes a converter + public void FuncObject([Test("{k}")] object w) + { + _log = w.ToString(); + } + + object IConverter.Convert(AlphaType input) + { + return $"Alpha2Obj({input._value})"; + } + } + + // Test binding to object. + [Fact] + public void TestObjectInheritence() + { + TestWorker(); + } + + public class ConfigObjectInheritence : IExtensionConfigProvider, ITest, IConverter + { + public void Initialize(ExtensionConfigContext context) + { + var bf = context.Config.BindingFactory; + // No explicit converters - just use implicit ones like inheritence + var rule1 = bf.BindToInput(typeof(GeneralBuilder<>)); + var rule2 = bf.BindToInput(this); // 2nd rule + context.RegisterBindingRules(rule1, rule2); + } + + public void Test(TestJobHost host) + { + // 1st rule + host.Call("FuncDerived", new { k = 1 }); + Assert.Equal("GeneralBuilder_AlphaDerivedType(1)", _log); + + // 1st rule + implicit converter + host.Call("Func", new { k = 1 }); + Assert.Equal("GeneralBuilder_AlphaDerivedType(1)", _log); + + // 2nd rule, object isn't matched in an inheritence converter + host.Call("FuncObject", new { k = 1 }); + Assert.Equal("[obj!]", _log); + } + + string _log; + + public void FuncDerived([Test("{k}")] AlphaDerivedType w) + { + _log = w._value; + } + + // builds AlphaDerivedType, and then applies an implicit inheritence converter. + public void Func([Test("{k}")] AlphaType w) + { + // Actually passed in a derived instance + Assert.IsType(w); + + _log = w._value; + } + + // Uses the direct -->object binding rule + public void FuncObject([Test("{k}")] object w) + { + var beta = Assert.IsType(w); + _log = beta._value; + } + + public object Convert(TestAttribute input) + { + return BetaType.New("[obj!]"); + } + } + + // Test collectors and object[] bindings. + // Object[] --> multiple items + [Fact] + public void TestConfigCollectorMultipleItems() + { + TestWorker>(); + } + + // Test collectors and object[] bindings. + // Object[] --> single item + [Fact] + public void TestConfigCollectorSingleItem() + { + TestWorker>(); + } + + public class ConfigCollector : + IExtensionConfigProvider, + ITest>, + IConverter> + { + public string _log; + + public IAsyncCollector Convert(TestAttribute arg) + { + return new AlphaTypeCollector { _parent = this }; + } + + public class Object2AlphaConverter : IConverter + { + public AlphaType Convert(object obj) + { + var json = JsonConvert.SerializeObject(obj); + return AlphaType.New($"Json({json})"); + } + } + + class AlphaTypeCollector : IAsyncCollector + { + public ConfigCollector _parent; + + public Task AddAsync(AlphaType item, CancellationToken cancellationToken = default(CancellationToken)) + { + _parent._log += $"Collector({item._value});"; + return Task.FromResult(0); + } + + public Task FlushAsync(CancellationToken cancellationToken = default(CancellationToken)) + { + return Task.FromResult(0); + } + } + + public void Initialize(ExtensionConfigContext context) + { + var bf = context.Config.BindingFactory; + + // The converter rule is the key switch. + // If TParam==SingleType, then that means we can only convert from non-array types to AlphaType. + // that means object[] converts to AlphaType[] (many) + // If TParam==OpenType, then we can convert any type (including arrays) to an AlphaType. + // that means object[] converts to AlphaType (one) + bf.ConverterManager.AddConverter(typeof(Object2AlphaConverter)); + + var rule1 = bf.BindToCollector(this); + context.RegisterBindingRules(rule1); + } + + public void Test(TestJobHost> host) + { + // tells you we made 2 AddAysnc calls, and invoked the converter on each item. + _log = ""; + host.Call("Func2", new { k = 1 }); + + if (typeof(TParam) == typeof(NonArrayOpenType)) + { + // Each object gets converter, so object[] gets converterd to multiple types. + Assert.Equal("Collector(Json(123));Collector(Json(\"xyz\"));", _log); + } + else + { + // the object[] gets converters to a single element to a single object + Assert.Equal("Collector(Json([123,\"xyz\"]));", _log); + } + + // 2 calls, but no converters + _log = ""; + host.Call("Func", new { k = 1 }); + Assert.Equal("Collector(v1);Collector(v2);", _log); + } + + public async Task Func([Test("{k}")] IAsyncCollector collector) + { + await collector.AddAsync(AlphaType.New("v1")); + await collector.AddAsync(AlphaType.New("v2")); + } + + public void Func2([Test("{k}")] out object[] foo) + { + foo = new object[] { + 123, + "xyz" + }; + } + } + + // Matches to 'object' but not 'object[]' + public class NonArrayOpenType : OpenType + { + public override bool IsMatch(Type type) + { + return !type.IsArray; + } + } + + // Error case. + [Fact] + public void TestError1() + { + // Test an error in configuration setup. This is targeted at the extension author. + TestWorker(); + } + + public class ConfigError1 : IExtensionConfigProvider, ITest + { + public void Initialize(ExtensionConfigContext context) + { + var bf = context.Config.BindingFactory; + + // This is an error. The rule specifies OpenType,which allows any type. + // But the builder can only produce Alpha types. + var rule = bf.BindToInput(typeof(AlphaBuilder)); + + context.RegisterBindingRules(rule); + } + + public void Test(TestJobHost host) + { + host.AssertIndexingError("Func", "No Convert method on type AlphaBuilder to convert from TestAttribute to BetaType"); + } + + // Fail to bind because: + // We only have an AlphaBuilder, and no registered converters from Alpha-->Beta + public void Func([Test("{k}")] BetaType w) + { + Assert.False(true); // Method shouldn't have been invoked. + } + } + + // Error case: verify that we don't do an arbitrary depth search + [Fact] + public void TestErrorSearch() + { + TestWorker(); + } + + public class ConfigErrorSearch : IExtensionConfigProvider, ITest + { + public void Initialize(ExtensionConfigContext context) + { + var bf = context.Config.BindingFactory; + + var cm = bf.ConverterManager; + cm.AddConverter(ConvertAlpha2Beta); + cm.AddConverter((beta) => $"Str({beta._value})" ); + var rule = bf.BindToInput(typeof(AlphaBuilder)); + + context.RegisterBindingRules(rule); + } + + public void Test(TestJobHost host) + { + host.AssertIndexingError("Func", "Can't bind Test to type 'System.String'."); + } + + // Fail to bind because: + // We don't chain multiple converters together. + // So we don't do TestAttr --> Alpha --> Beta --> string + public void Func([Test("{k}")] string w) + { + Assert.False(true); // Method shouldn't have been invoked. + } + } + + // Get standard error message for failing to bind an attribute to a given parameter type. + static string ErrorMessage(Type parameterType) + { + return $"Can't bind Test to type '{parameterType.FullName}'."; + } + + // Glue to initialize a JobHost with the correct config and invoke the Test method. + // Config also has the program on it. + private void TestWorker() where TConfig : IExtensionConfigProvider, ITest, new() + { + var prog = new TConfig(); var jobActivator = new FakeActivator(); jobActivator.Add(prog); - var host = TestHelpers.NewJobHost(jobActivator, new FakeExtClient()); - host.Call("Func", new { k = 1 }); + IExtensionConfigProvider ext = prog; + var host = TestHelpers.NewJobHost(jobActivator, ext); - // Skipped first rule, applied second - Assert.Equal(prog._value, "abc-1"); + ITest test = prog; + test.Test(host); } + + // Some custom type to bind to. + public class AlphaType + { + public static AlphaType New(string value) + { + return new AlphaType { _value = value }; + } + public string _value; + } - // Unit test that we can properly extract TMessage from a parameter type. - [Fact] - public void GetCoreType() + // Some custom type to bind to. + public class AlphaDerivedType : AlphaType + { + public static new AlphaDerivedType New(string value) + { + return new AlphaDerivedType { _value = value }; + } + } + + + // Another custom type, not related to the first type. + public class BetaType + { + public static BetaType New(string value) + { + return new BetaType { _value = value }; + } + + public string _value; + } + + static BetaType ConvertAlpha2Beta(AlphaType x) { - Assert.Equal(null, BindingFactoryHelpers.GetAsyncCollectorCoreType(typeof(Widget))); // Not an AsyncCollector type + return BetaType.New($"A2B({x._value})"); + } + + // A test attribute for binding. + public class TestAttribute : Attribute + { + public TestAttribute(string path) + { + this.Path = path; + } + + [AutoResolve] + public string Path { get; set; } + } - Assert.Equal(typeof(Widget), BindingFactoryHelpers.GetAsyncCollectorCoreType(typeof(IAsyncCollector))); - Assert.Equal(typeof(Widget), BindingFactoryHelpers.GetAsyncCollectorCoreType(typeof(ICollector))); - Assert.Equal(typeof(Widget), BindingFactoryHelpers.GetAsyncCollectorCoreType(typeof(Widget).MakeByRefType())); - Assert.Equal(typeof(Widget), BindingFactoryHelpers.GetAsyncCollectorCoreType(typeof(Widget[]).MakeByRefType())); + // Converter for building instances of RedType from an attribute + class AlphaBuilder : IConverter + { + // Test explicit interface implementation + AlphaType IConverter.Convert(TestAttribute attr) + { + return AlphaType.New("AlphaBuilder(" + attr.Path + ")"); + } + } - // Verify that 'out' takes precedence over generic. - Assert.Equal(typeof(IFoo), BindingFactoryHelpers.GetAsyncCollectorCoreType(typeof(IFoo).MakeByRefType())); + // Converter for building instances of RedType from an attribute + class BetaBuilder : IConverter + { + // Test with implicit interface implementation + public BetaType Convert(TestAttribute attr) + { + return BetaType.New("BetaBuilder(" + attr.Path + ")"); + } } - // Random generic type to use in tests. - interface IFoo + // Can build Widgets or OtherType + class GeneralBuilder : IConverter { + private readonly MethodInfo _builder; + + public GeneralBuilder() + { + _builder = typeof(T).GetMethod("New", BindingFlags.Public | BindingFlags.Static); + if (_builder == null) + { + throw new InvalidOperationException($"Type {typeof(T).Name} should have a static New() method"); + } + } + + public T Convert(TestAttribute attr) + { + var value = $"GeneralBuilder_{typeof(T).Name}({attr.Path})"; + return (T)_builder.Invoke(null, new object[] { value}); + } } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/BindingProviderFilterTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/BindingProviderFilterTests.cs index 5e3d6c8dc..3a4c4be74 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/BindingProviderFilterTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/BindingProviderFilterTests.cs @@ -56,8 +56,7 @@ public void TestSkip() // Skipped first rule, applied second Assert.Equal(prog._value, "xxx"); } - - + public class FakeExtClient : IExtensionConfigProvider { public void Initialize(ExtensionConfigContext context) @@ -65,12 +64,27 @@ public void Initialize(ExtensionConfigContext context) var bf = context.Config.BindingFactory; // Add [Test] support - var rule = bf.BindToExactType(attr => attr.Path); + var rule = bf.BindToInput(typeof(Converter1)); var ruleValidate = bf.AddFilter(Filter, rule); - var rule2 = bf.BindToExactType(attr => "xxx"); + var rule2 = bf.BindToInput(typeof(Converter2)); context.RegisterBindingRules(ruleValidate, rule2); } + class Converter1 : IConverter + { + public string Convert(TestAttribute attr) + { + return attr.Path; + } + } + class Converter2 : IConverter + { + public string Convert(TestAttribute attr) + { + return "xxx"; + } + } + public const string IndexErrorMsg = "error 12345"; private static bool Filter(TestAttribute attribute, Type parameterType) diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/CollectorTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/CollectorTests.cs index 4e0cba564..efae84363 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/CollectorTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/CollectorTests.cs @@ -1,16 +1,13 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using Microsoft.Azure.WebJobs.Host; -using Microsoft.Azure.WebJobs.Host.Config; -using Microsoft.Azure.WebJobs.Host.Indexers; -using Microsoft.Azure.WebJobs.Host.TestCommon; -using Newtonsoft.Json; using System; -using System.Collections.Generic; using System.Reflection; using System.Text; using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Indexers; +using Microsoft.Azure.WebJobs.Host.TestCommon; +using Newtonsoft.Json; using Xunit; namespace Microsoft.Azure.WebJobs.Host.UnitTests @@ -35,7 +32,7 @@ public class ErrorProgram { // Malformed // Expect the - public static void Func([FakeQueue(Prefix ="Error-{name%")] out string x) + public static void Func([FakeQueue(Prefix = "Error-{name%")] out string x) { x = "x"; } @@ -70,7 +67,7 @@ public void TestError() public class Functions { public static async Task SendDirectClient( - [FakeQueue] FakeQueueClient client) + [FakeQueue("CustomConstructor", CustomPolicy = "Custom")] FakeQueueClient client) { await client.AddAsync(new FakeQueueData { Message = "abc", ExtraPropertery = "def" }); } @@ -182,11 +179,11 @@ public void Test() { FakeQueueClient client = new FakeQueueClient(); var host = TestHelpers.NewJobHost(client); - + var p7 = Invoke(host, client, "SendDirectClient"); Assert.Equal(1, p7.Length); Assert.Equal("abc", p7[0].Message); - Assert.Equal("def", p7[0].ExtraPropertery); + Assert.Equal("def", p7[0].ExtraPropertery); var p8 = Invoke(host, client, "SendOneDerivedNative"); Assert.Equal(1, p8.Length); @@ -199,7 +196,7 @@ public void Test() // Single items var p1 = InvokeJson(host, client, "SendOnePoco"); - Assert.Equal(1, p1.Length); + Assert.Equal(1, p1.Length); Assert.Equal(123, p1[0].val1); var p2 = Invoke(host, client, "SendOneNative"); @@ -211,14 +208,14 @@ public void Test() Assert.Equal(1, p3.Length); Assert.Equal("stringvalue", p3[0].Message); - foreach (string methodName in new string[] { "SendDontQueue", "SendArrayNull", "SendArrayLen0"}) + foreach (string methodName in new string[] { "SendDontQueue", "SendArrayNull", "SendArrayLen0" }) { var p6 = Invoke(host, client, methodName); Assert.Equal(0, p6.Length); } // batching - foreach(string methodName in new string[] { + foreach (string methodName in new string[] { "SendSyncCollectorBytes", "SendArrayString", "SendSyncCollectorString", "SendAsyncCollectorString", "SendCollectorNative" }) { var p4 = Invoke(host, client, methodName); @@ -227,13 +224,13 @@ public void Test() Assert.Equal("second", p4[1].Message); } - foreach(string methodName in new string[] { "SendCollectorPoco", "SendArrayPoco" }) + foreach (string methodName in new string[] { "SendCollectorPoco", "SendArrayPoco" }) { var p5 = InvokeJson(host, client, methodName); Assert.Equal(2, p5.Length); Assert.Equal(100, p5[0].val1); Assert.Equal(200, p5[1].val1); - } + } } static FakeQueueData[] Invoke(JobHost host, FakeQueueClient client, string name) @@ -256,6 +253,6 @@ static T[] InvokeJson(JobHost host, FakeQueueClient client, string name) var obj = Array.ConvertAll(data, x => JsonConvert.DeserializeObject(x.Message)); client._items.Clear(); return obj; - } + } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/FakeItemTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/FakeItemTests.cs index 51654d73a..353147d0c 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/FakeItemTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/FakeItemTests.cs @@ -1,16 +1,16 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Config; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using System; -using Xunit; +using Microsoft.Azure.WebJobs.Host.Bindings; +using Microsoft.Azure.WebJobs.Host.Config; using Microsoft.Azure.WebJobs.Host.TestCommon; -using Newtonsoft.Json; using Microsoft.Azure.WebJobs.Host.UnitTests.Indexers; +using Newtonsoft.Json; +using Xunit; namespace Microsoft.Azure.WebJobs.Host.UnitTests { @@ -53,7 +53,7 @@ public void Test() { value = 123 }; - + var host = TestHelpers.NewJobHost(nr, client); // With out parameter @@ -72,7 +72,7 @@ public void Test() var item = (Item)client._dict["ModifyInPlace"]; Assert.Equal(124, item.value); - } + } } } @@ -98,7 +98,7 @@ private Task BuildFromAttribute(FakeItemAttribute attr, Type param } var type = typeof(MySpecialValueBinder<>).MakeGenericType(parameterType); - var result = (IValueBinder) Activator.CreateInstance(type, this, attr.Index); + var result = (IValueBinder)Activator.CreateInstance(type, this, attr.Index); return Task.FromResult(result); } @@ -121,13 +121,13 @@ public Type Type } } - public object GetValue() + public Task GetValueAsync() { // Clone to mimic real network semantics - we're not sharing in-memory objects. - var obj = _client._dict[_index]; + var obj = _client._dict[_index]; string json = JsonConvert.SerializeObject(obj); var clone = JsonConvert.DeserializeObject(json, this.Type); - return clone; + return Task.FromResult(clone); } public Task SetValueAsync(object value, CancellationToken cancellationToken) diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/HostCallTestsWithBindingData.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/HostCallTestsWithBindingData.cs index ba2eddf69..8cc25a536 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/HostCallTestsWithBindingData.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/HostCallTestsWithBindingData.cs @@ -56,7 +56,7 @@ public class TestAttribute : Attribute public string Path { get; set; } } - public class FakeExtClient : IExtensionConfigProvider + public class FakeExtClient : IExtensionConfigProvider, IConverter { public void Initialize(ExtensionConfigContext context) { @@ -64,7 +64,7 @@ public void Initialize(ExtensionConfigContext context) var bf = context.Config.BindingFactory; // Add [Test] support - var rule = bf.BindToExactType(attr => attr.Path); + var rule = bf.BindToInput(typeof(FakeExtClient)); extensions.RegisterBindingRules(rule); // Add [FakeQueueTrigger] support. @@ -74,6 +74,11 @@ public void Initialize(ExtensionConfigContext context) var triggerBindingProvider = new FakeQueueTriggerBindingProvider(new FakeQueueClient(), cm); extensions.RegisterExtension(triggerBindingProvider); } + + public string Convert(TestAttribute attr) + { + return attr.Path; + } } // Explicit bindingData takes precedence over binding data inferred from the trigger object. diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/TypedCollectorTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/TypedCollectorTests.cs index f3db827bf..a0e84a208 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/TypedCollectorTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/TypedCollectorTests.cs @@ -115,9 +115,7 @@ public class FakeQueueTypedClient : IExtensionConfigProvider // Track items that are queued. public List _items; public string _prefix; // from attribute, to test attribute automatic resolution. - - public Func Filter; - + public FakeQueueTypedClient() { _items = new List(); @@ -133,18 +131,24 @@ void IExtensionConfigProvider.Initialize(ExtensionConfigContext context) // Don't add any converters. IExtensionRegistry extensions = context.Config.GetService(); var bf = context.Config.BindingFactory; - var ruleOutput = bf.BindToGenericAsyncCollector(Builder, this.Filter); + var ruleOutput = bf.BindToCollector(typeof(Builder<>), this); extensions.RegisterBindingRules(ruleOutput); } - private object Builder(FakeQueueAttribute attrResolved, Type typeMessage) + public class Builder : IConverter> { - var client = new FakeQueueTypedClient(this, attrResolved.Prefix); + private readonly FakeQueueTypedClient _client; + public Builder(FakeQueueTypedClient client) + { + _client = client; + } - var t = typeof(TypedAsyncCollector<>).MakeGenericType(typeMessage); - var obj = Activator.CreateInstance(t, client); - return obj; + public IAsyncCollector Convert(FakeQueueAttribute attrResolved) + { + var client = new FakeQueueTypedClient(_client, attrResolved.Prefix); + return new TypedAsyncCollector(client); + } } // Collector gets dynamically instantiated to match the 'core message' type of the user's parameter. diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/ValidationTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/ValidationTests.cs index dc997a641..747d30fab 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/ValidationTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Common/ValidationTests.cs @@ -66,17 +66,22 @@ public void Good([Test(Bad= false, Path = "%k1%-{k2}")] Widget x) } } - public class FakeExtClient : IExtensionConfigProvider + public class FakeExtClient : IExtensionConfigProvider, IConverter { public void Initialize(ExtensionConfigContext context) { var bf = context.Config.BindingFactory; - // Add [Test] support - var rule = bf.BindToExactType(attr => new Widget()); + // Add [Test] support + var rule = bf.BindToInput(this); context.RegisterBindingRules(ValidateAtIndexTime, rule); } + public Widget Convert(TestAttribute attr) + { + return new Widget(); + } + private static void ValidateAtIndexTime(TestAttribute attribute, Type parameterType) { attribute.ValidateAtIndexTime(parameterType); @@ -103,20 +108,32 @@ public void TestValidatorSucceeds() } // Register [Test] with 2 rules and a local validator. - public class FakeExtClient2 : IExtensionConfigProvider + public class FakeExtClient2 : IExtensionConfigProvider, + IConverter, + IConverter { public void Initialize(ExtensionConfigContext context) { var bf = context.Config.BindingFactory; // Add [Test] support - var rule1 = bf.BindToExactType(attr => new Widget()); - var rule2 = bf.BindToExactType(attr => new Widget2()); + var rule1 = bf.BindToInput(this); + var rule2 = bf.BindToInput(this); var rule2Validator = bf.AddValidator(LocalValidator, rule2); context.RegisterBindingRules(rule2Validator, rule1); } + Widget IConverter.Convert(TestAttribute attr) + { + return new Widget(); + } + + Widget2 IConverter.Convert(TestAttribute attr) + { + return new Widget2(); + } + private void LocalValidator(TestAttribute attribute, Type parameterType) { attribute.ValidateAtIndexTime(parameterType); diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/ConverterManagerTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/ConverterManagerTests.cs index e8af18e17..9d7e77bdc 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/ConverterManagerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/ConverterManagerTests.cs @@ -8,6 +8,10 @@ using Microsoft.Azure.WebJobs.Host.Bindings; using Newtonsoft.Json.Linq; using System.Threading; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Reflection; +using System.Collections.Generic; namespace Microsoft.Azure.WebJobs.Host.UnitTests { @@ -67,12 +71,11 @@ public void UseValueBindingContext() var value = new Other { Value2 = "abc" }; Wrapper x1 = func(value, null, testContext); + // strip whitespace + string val = Regex.Replace(x1.Value, @"\s", ""); + string expected = String.Format("{{\"Value2\":\"abc\",\"$\":\"{0}\"}}", instance); - Assert.Equal(@"{ - ""Value2"": ""abc"", - ""$"": """ + instance.ToString() + @""" -}", x1.Value); - + Assert.Equal(expected, val); } // Explicit converters take precedence. @@ -197,6 +200,432 @@ public void AttributeOverloads2() Assert.Equal("[common:x]", x2); } + + // Test converter using Open generic types + class TypeConverterWithTwoGenericArgs : + IConverter + { + public TypeConverterWithTwoGenericArgs(ConverterManagerTests config) + { + // We know this is only used by a single test invoking with this combination of params. + Assert.Equal(typeof(String), typeof(TInput)); + Assert.Equal(typeof(int), typeof(TOutput)); + + config._counter++; + } + + public TOutput Convert(TInput input) + { + var str = (string)(object)input; + return (TOutput)(object)int.Parse(str); + } + } + + [Fact] + public void OpenTypeConverterWithTwoGenericArgs() + { + Assert.Equal(0, _counter); + var cm = new ConverterManager(); + + // Register a converter builder. + // Builder runs once; converter runs each time. + // Uses open type to match. + cm.AddConverter(typeof(TypeConverterWithTwoGenericArgs<,>), this); + + var converter = cm.GetConverter(); + + Assert.Equal(12, converter("12", new TestAttribute(null), null)); + Assert.Equal(34, converter("34", new TestAttribute(null), null)); + + Assert.Equal(1, _counter); // converterBuilder is only called once. + + // 'char' as src parameter doesn't match the type predicate. + Assert.Null(cm.GetConverter()); + } + + + // Test converter using Open generic types, rearranging generics + class TypeConverterWithOneGenericArg : + IConverter> + { + public IEnumerable Convert(TElement input) + { + // Trivial rule. + return new TElement[] { input, input, input }; + } + } + + [Fact] + public void OpenTypeConverterWithOneGenericArg() + { + var cm = new ConverterManager(); + + // Register a converter builder. + // Builder runs once; converter runs each time. + // Uses open type to match. + // Also test the IEnumerable pattern. + cm.AddConverter, Attribute>(typeof(TypeConverterWithOneGenericArg<>)); + + var attr = new TestAttribute(null); + + { + var converter = cm.GetConverter, Attribute>(); + Assert.Equal(new int[] { 1, 1, 1 }, converter(1, attr, null)); + } + + { + var converter = cm.GetConverter, Attribute>(); + Assert.Equal(new string[] { "a", "a", "a" }, converter("a", attr, null)); + } + } + + + class OpenArrayConverter + : IConverter + { + public string Convert(T[] input) + { + return string.Join(",", input); + } + } + + // Test OpenType[] --> converter + [Fact] + public void OpenTypeArray() + { + var cm = new ConverterManager(); + + cm.AddConverter(typeof(OpenArrayConverter<>)); + var attr = new TestAttribute(null); + + var converter = cm.GetConverter(); + Assert.Equal("1,2,3", converter(new int[] { 1, 2, 3 }, attr, null)); + } + + // Test concrete array converters. + [Fact] + public void ClosedTypeArray() + { + var cm = new ConverterManager(); + + cm.AddConverter(new OpenArrayConverter()); + var attr = new TestAttribute(null); + + var converter = cm.GetConverter(); + Assert.Equal("1,2,3", converter(new int[] { 1, 2, 3 }, attr, null)); + } + + // Counter used by tests to verify that converter ctors are only run once and then shared across + // multiple invocations. + private int _counter; + + class ConverterInstanceMethod : + IConverter + { + // Converter discovered for OpenType4 test. Used directly. + public int Convert(string input) + { + return int.Parse(input); + } + } + + [Fact] + public void OpenTypeSimpleConcreteConverter() + { + Assert.Equal(0, _counter); + var cm = new ConverterManager(); + + // Register a converter builder. + // Builder runs once; converter runs each time. + // Uses open type to match. + cm.AddConverter(new ConverterInstanceMethod()); + + var converter = cm.GetConverter(); + + Assert.Equal(12, converter("12", new TestAttribute(null), null)); + Assert.Equal(34, converter("34", new TestAttribute(null), null)); + + Assert.Equal(0, _counter); // passed in instantiated object; counter never incremented. + + // 'char' as src parameter doesn't match the type predicate. + Assert.Null(cm.GetConverter()); + } + + // Test converter using concrete types. + class TypeConverterWithConcreteTypes + : IConverter + { + public TypeConverterWithConcreteTypes(ConverterManagerTests config) + { + config._counter++; + } + + public int Convert(string input) + { + return int.Parse(input); + } + } + + [Fact] + public void OpenTypeConverterWithConcreteTypes() + { + Assert.Equal(0, _counter); + var cm = new ConverterManager(); + + // Register a converter builder. + // Builder runs once; converter runs each time. + // Uses open type to match. + cm.AddConverter(typeof(TypeConverterWithConcreteTypes), this); + + var converter = cm.GetConverter(); + + Assert.Equal(12, converter("12", new TestAttribute(null), null)); + Assert.Equal(34, converter("34", new TestAttribute(null), null)); + + Assert.Equal(1, _counter); // converterBuilder is only called once. + + // 'char' as src parameter doesn't match the type predicate. + Assert.Null(cm.GetConverter()); + } + + [Fact] + public void OpenType() + { + int count = 0; + var cm = new ConverterManager(); + + // Register a converter builder. + // Builder runs once; converter runs each time. + // Uses open type to match. + cm.AddConverter( + (typeSrc, typeDest) => + { + count++; + Assert.Equal(typeof(String), typeSrc); + Assert.Equal(typeof(int), typeDest); + + return (input) => + { + string s = (string)input; + return int.Parse(s); + }; + }); + + var converter = cm.GetConverter(); + Assert.NotNull(converter); + Assert.Equal(12, converter("12", new TestAttribute(null), null)); + Assert.Equal(34, converter("34", new TestAttribute(null), null)); + + Assert.Equal(1, count); // converterBuilder is only called once. + + // 'char' as src parameter doesn't match the type predicate. + Assert.Null(cm.GetConverter()); + } + + + // Test with async converter + public class UseAsyncConverter : IAsyncConverter + { + public Task ConvertAsync(int i, CancellationToken cancellationToken) + { + return Task.FromResult(i.ToString()); + } + } + + // Test non-generic Task, use instance match. + [Fact] + public void UseUseAsyncConverterTest() + { + var cm = new ConverterManager(); + + cm.AddConverter(new UseAsyncConverter()); + + var converter = cm.GetConverter(); + + Assert.Equal("12", converter(12, new TestAttribute(null), null)); + } + + // Test with async converter + public class UseGenericAsyncConverter : + IAsyncConverter + { + public Task ConvertAsync(int i, CancellationToken token) + { + Assert.Equal(typeof(string), typeof(T)); + return Task.FromResult((T) (object) i.ToString()); + } + } + + // Test generic Task, use typing match. + [Fact] + public void UseGenericAsyncConverterTest() + { + var cm = new ConverterManager(); + + cm.AddConverter(typeof(UseGenericAsyncConverter<>)); + + var converter = cm.GetConverter(); + + Assert.Equal("12", converter(12, new TestAttribute(null), null)); + } + + // Sample types to excercise pattern matcher + public class Foo : IConverter> + { + public IDictionary Convert(T1 input) + { + throw new NotImplementedException(); + } + + public T1[] _genericArray; + } + + // Unit tests for TestPatternMatcher.ResolveGenerics + [Fact] + public void TestPatternMatcher_ResolveGenerics() + { + var typeFoo = typeof(Foo<,>); + var int1 = typeFoo.GetInterfaces()[0]; // IConverter> + var typeFoo_T1 = typeFoo.GetGenericArguments()[0]; + var typeGenericArray = typeFoo.GetField("_genericArray").FieldType; + var typeIConverter_T1 = int1.GetGenericArguments()[0]; + var typeIConverter_IDictChar_T2 = int1.GetGenericArguments()[1]; + + var genArgs = new Dictionary + { + { "T1", typeof(int) }, + { "T2", typeof(string) } + }; + + Assert.Equal(typeof(int), PatternMatcher.ResolveGenerics(typeof(int), genArgs)); + + Assert.Equal(typeof(int[]), PatternMatcher.ResolveGenerics(typeGenericArray, genArgs)); + + var typeFooIntStr = typeof(Foo); + Assert.Equal(typeFooIntStr, PatternMatcher.ResolveGenerics(typeFooIntStr, genArgs)); + + Assert.Equal(typeof(int), PatternMatcher.ResolveGenerics(typeFoo_T1, genArgs)); + Assert.Equal(typeof(int), PatternMatcher.ResolveGenerics(typeIConverter_T1, genArgs)); + Assert.Equal(typeof(IDictionary), PatternMatcher.ResolveGenerics(typeIConverter_IDictChar_T2, genArgs)); + + Assert.Equal(typeof(Foo), PatternMatcher.ResolveGenerics(typeFoo, genArgs)); + + Assert.Equal(typeof(IConverter>), + PatternMatcher.ResolveGenerics(int1, genArgs)); + } + + public class TestConverter : + IConverter>, // binding rule converter + IConverter // general type converter + { + public IAsyncCollector Convert(Attribute input) + { + return null; + } + + byte[] IConverter.Convert(string input) + { + return null; + } + } + + [Fact] + public void PatternMatcher_Succeeds_WhenBindingRuleConverterExists() + { + var pm = PatternMatcher.New(typeof(TestConverter)); + var generalConverter = pm.TryGetConverterFunc(typeof(string), typeof(byte[])); + Assert.NotNull(generalConverter); + + var bindingRuleConverter = pm.TryGetConverterFunc(typeof(Attribute), typeof(IAsyncCollector)); + Assert.NotNull(bindingRuleConverter); + } + + private class TestConverterFakeEntity + : IConverter + { + public IFakeEntity Convert(JObject obj) + { + return new MyFakeEntity { Property = obj["Property1"].ToString() }; + } + } + + private class TestConverterFakeEntity + : IConverter + { + public IFakeEntity Convert(T item) + { + Assert.IsType(item); // test only calls this with PocoEntity + var d = (PocoEntity) (object) item; + string propValue = d.Property2; + return new MyFakeEntity { Property = propValue }; + } + } + + // Test sort of rules that we have in tables. + // Rules can overlap, so make sure that the right rule is dispatched. + // Poco is a base class of Jobject and IFakeEntity. + // Give each rule its own unique converter and ensure each converter is called. + [Fact] + public void TestConvertFakeEntity() + { + var cm = new ConverterManager(); + + // Derived --> IFakeEntity [automatic] + // JObject --> IFakeEntity + // Poco --> IFakeEntity + cm.AddConverter(typeof(TestConverterFakeEntity)); + cm.AddConverter(typeof(TestConverterFakeEntity<>)); + + { + var converter = cm.GetConverter(); + var src = new MyFakeEntity { Property = "123" }; + var dest = converter(src, null, null); + Assert.Same(src, dest); // should be exact same instance - no conversion + } + + { + var converter = cm.GetConverter(); + JObject obj = new JObject(); + obj["Property1"] = "456"; + var dest = converter(obj, null, null); + Assert.Equal("456", dest.Property); + } + + { + var converter = cm.GetConverter(); + var src = new PocoEntity { Property2 = "789" }; + var dest = converter(src, null, null); + Assert.Equal("789", dest.Property); + } + } + + // Class that implements IFakeEntity. Test conversions. + class MyFakeEntity : IFakeEntity + { + public string Property { get; set; } + } + + interface IFakeEntity + { + string Property { get; } + } + + // Poco class that can be converted to IFakeEntity, but doesn't actually implement IFakeEntity. + class PocoEntity + { + // Give a different property name so that we can enforce the exact converter. + public string Property2 { get; set; } + } + + class TypeWrapperIsString : OpenType + { + // Predicate is invoked by ConverterManager to determine if a type matches. + public override bool IsMatch(Type t) + { + return t == typeof(string); + } + } + // Custom type public class Wrapper { diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DefaultStorageAccountProviderTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DefaultStorageAccountProviderTests.cs index bac92cca3..4d15eb1fb 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DefaultStorageAccountProviderTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DefaultStorageAccountProviderTests.cs @@ -38,7 +38,7 @@ public void ConnectionStringProvider_NoDashboardConnectionString_Throws() // Act & Assert ExceptionAssert.ThrowsInvalidOperation(() => product.GetDashboardAccountAsync(CancellationToken.None).GetAwaiter().GetResult(), - "Microsoft Azure WebJobs SDK Dashboard connection string is missing or empty. The Microsoft Azure Storage account connection string can be set in the following ways:" + Environment.NewLine + + "Microsoft Azure WebJobs SDK 'Dashboard' connection string is missing or empty. The Microsoft Azure Storage account connection string can be set in the following ways:" + Environment.NewLine + "1. Set the connection string named 'AzureWebJobsDashboard' in the connectionStrings section of the .config file in the following format " + ", or" + Environment.NewLine + "2. Set the environment variable named 'AzureWebJobsDashboard', or" + Environment.NewLine + @@ -80,7 +80,7 @@ public void GetAccountAsync_WhenReadFromConfig_ReturnsParsedAccount(string conne IStorageAccountProvider provider = CreateProductUnderTest(services, connectionStringProvider, parser, validator); - IStorageAccount actualAccount = provider.GetAccountAsync( + IStorageAccount actualAccount = provider.TryGetAccountAsync( connectionStringName, CancellationToken.None).GetAwaiter().GetResult(); Assert.Same(parsedAccount, actualAccount); @@ -107,7 +107,7 @@ public void GetAccountAsync_WhenInvalidConfig_PropagatesParserException(string c IStorageAccountProvider provider = CreateProductUnderTest(services, connectionStringProvider, parser); Exception actualException = Assert.Throws( - () => provider.GetAccountAsync(connectionStringName, CancellationToken.None).GetAwaiter().GetResult()); + () => provider.TryGetAccountAsync(connectionStringName, CancellationToken.None).GetAwaiter().GetResult()); Assert.Same(expectedException, actualException); } @@ -131,7 +131,7 @@ public void GetAccountAsync_WhenInvalidCredentials_PropagatesValidatorException( IStorageAccountProvider provider = CreateProductUnderTest(services, connectionStringProvider, parser, validator); Exception actualException = Assert.Throws( - () => provider.GetAccountAsync(connectionStringName, CancellationToken.None).GetAwaiter().GetResult()); + () => provider.TryGetAccountAsync(connectionStringName, CancellationToken.None).GetAwaiter().GetResult()); Assert.Same(expectedException, actualException); } @@ -147,7 +147,7 @@ public void GetAccountAsync_WhenDashboardOverridden_ReturnsParsedAccount() IStorageCredentialsValidator validator = CreateValidator(parsedAccount); DefaultStorageAccountProvider provider = CreateProductUnderTest(services, connectionStringProvider, parser, validator); provider.DashboardConnectionString = connectionString; - IStorageAccount actualAccount = provider.GetAccountAsync( + IStorageAccount actualAccount = provider.TryGetAccountAsync( ConnectionStringNames.Dashboard, CancellationToken.None).GetAwaiter().GetResult(); Assert.Same(parsedAccount, actualAccount); @@ -165,7 +165,7 @@ public void GetAccountAsync_WhenStorageOverridden_ReturnsParsedAccount() DefaultStorageAccountProvider provider = CreateProductUnderTest(services, connectionStringProvider, parser, validator); provider.StorageConnectionString = connectionString; - IStorageAccount actualAccount = provider.GetAccountAsync( + IStorageAccount actualAccount = provider.TryGetAccountAsync( ConnectionStringNames.Storage, CancellationToken.None).GetAwaiter().GetResult(); Assert.Same(parsedAccount, actualAccount); @@ -177,7 +177,7 @@ public void GetAccountAsync_WhenDashboardOverriddenWithNull_ReturnsNull() DefaultStorageAccountProvider provider = CreateProductUnderTest(); provider.DashboardConnectionString = null; - IStorageAccount actualAccount = provider.GetAccountAsync( + IStorageAccount actualAccount = provider.TryGetAccountAsync( ConnectionStringNames.Dashboard, CancellationToken.None).GetAwaiter().GetResult(); Assert.Null(actualAccount); @@ -189,7 +189,7 @@ public async Task GetAccountAsync_WhenStorageOverriddenWithNull_Succeeds() DefaultStorageAccountProvider provider = CreateProductUnderTest(); provider.StorageConnectionString = null; - var account = await provider.GetAccountAsync(ConnectionStringNames.Storage, CancellationToken.None); + var account = await provider.TryGetAccountAsync(ConnectionStringNames.Storage, CancellationToken.None); Assert.Null(account); } @@ -200,10 +200,10 @@ public async Task GetAccountAsync_WhenNoStorage_Succeeds() provider.DashboardConnectionString = null; provider.StorageConnectionString = null; - var dashboardAccount = await provider.GetAccountAsync(ConnectionStringNames.Dashboard, CancellationToken.None); + var dashboardAccount = await provider.TryGetAccountAsync(ConnectionStringNames.Dashboard, CancellationToken.None); Assert.Null(dashboardAccount); - var storageAccount = await provider.GetAccountAsync(ConnectionStringNames.Storage, CancellationToken.None); + var storageAccount = await provider.TryGetAccountAsync(ConnectionStringNames.Storage, CancellationToken.None); Assert.Null(storageAccount); } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DefaultStorageCredentialsValidatorTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DefaultStorageCredentialsValidatorTests.cs index 564d477ab..237f16140 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DefaultStorageCredentialsValidatorTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DefaultStorageCredentialsValidatorTests.cs @@ -1,20 +1,15 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Net; using System.Threading; -using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Azure.WebJobs.Host.Storage; using Microsoft.Azure.WebJobs.Host.Storage.Blob; using Microsoft.Azure.WebJobs.Host.Storage.Queue; -using Microsoft.Azure.WebJobs.Host.TestCommon; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Auth; +using Microsoft.WindowsAzure.Storage.Shared.Protocol; using Moq; using Xunit; -using Xunit.Extensions; -using System.Net; namespace Microsoft.Azure.WebJobs.Host.UnitTests.Executors { @@ -63,7 +58,7 @@ public void StorageAccount_QueueCheckThrows_BlobOnly() .Verifiable(); blobClientMock.Setup(b => b.GetServicePropertiesAsync(It.IsAny())) - .ReturnsAsync(null); + .ReturnsAsync((ServiceProperties)null); storageMock.Setup(s => s.CreateQueueClient(null)) .Returns(queueClientMock.Object) @@ -76,7 +71,7 @@ public void StorageAccount_QueueCheckThrows_BlobOnly() .Throws(new StorageException("", new WebException("Remote name could not be resolved", WebExceptionStatus.NameResolutionFailure))); storageMock.SetupSet(s => s.Type = StorageAccountType.BlobOnly); - + validator.ValidateCredentialsAsync(storageMock.Object, It.IsAny()).GetAwaiter().GetResult(); storageMock.Verify(); @@ -100,7 +95,7 @@ public void StorageAccount_QueueCheckThrowsUnexpectedStorage() .Verifiable(); blobClientMock.Setup(b => b.GetServicePropertiesAsync(It.IsAny())) - .ReturnsAsync(null); + .ReturnsAsync((ServiceProperties)null); storageMock.Setup(s => s.CreateQueueClient(null)) .Returns(queueClientMock.Object) diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DynamicHostIdProviderTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DynamicHostIdProviderTests.cs index 20d89a63e..d8343f88b 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DynamicHostIdProviderTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Executors/DynamicHostIdProviderTests.cs @@ -24,7 +24,7 @@ public void GetHostIdAsync_IfStorageAccountProviderThrowsInvalidOperationExcepti InvalidOperationException innerException = new InvalidOperationException(); taskSource.SetException(innerException); storageAccountProviderMock - .Setup(p => p.GetAccountAsync(It.IsAny(), It.IsAny())) + .Setup(p => p.TryGetAccountAsync(It.IsAny(), It.IsAny())) .Returns(taskSource.Task); IStorageAccountProvider storageAccountProvider = storageAccountProviderMock.Object; IFunctionIndexProvider functionIndexProvider = CreateDummyFunctionIndexProvider(); diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/FakeQueue/FakeQueueAttribute.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/FakeQueue/FakeQueueAttribute.cs index f49e139e6..e8410180a 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/FakeQueue/FakeQueueAttribute.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/FakeQueue/FakeQueueAttribute.cs @@ -2,8 +2,10 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Collections.Generic; +using System.Reflection; using Microsoft.Azure.WebJobs.Host.Bindings; - +using Microsoft.Azure.WebJobs.Host.Bindings.Path; namespace Microsoft.Azure.WebJobs.Host.UnitTests { @@ -11,16 +13,54 @@ namespace Microsoft.Azure.WebJobs.Host.UnitTests // Put on a parameter to mark that it goes to a "FakeQueue". public class FakeQueueAttribute : Attribute, IAttributeInvokeDescriptor { + public FakeQueueAttribute() : this(null) + { + } + + public FakeQueueAttribute(string constructorCustomPolicy) + { + ConstructorCustomPolicy = constructorCustomPolicy; + } + [AutoResolve] public string Prefix { get; set; } + [AutoResolve(ResolutionPolicyType = typeof(CustomResolutionPolicy))] + public string CustomPolicy { get; set; } + + [AutoResolve(ResolutionPolicyType = typeof(CustomResolutionPolicy))] + public string ConstructorCustomPolicy { get; private set; } + + internal string State1 { get; set; } + + internal string State2 { get; set; } + public string ToInvokeString() { return this.Prefix; } public FakeQueueAttribute FromInvokeString(string invokeString) { - return new FakeQueueAttribute { Prefix = invokeString }; + return new FakeQueueAttribute("customPolicy") { Prefix = invokeString }; + } + private class CustomResolutionPolicy : IResolutionPolicy + { + public string TemplateBind(PropertyInfo propInfo, Attribute attribute, BindingTemplate template, IReadOnlyDictionary bindingData) + { + FakeQueueAttribute queueAttribute = (FakeQueueAttribute)attribute; + + if (propInfo.Name == nameof(CustomPolicy)) + { + queueAttribute.State1 += "value1"; + } + + if (propInfo.Name == nameof(ConstructorCustomPolicy)) + { + queueAttribute.State2 += "value2"; + } + + return template.Bind(bindingData); + } } } } \ No newline at end of file diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/FakeQueue/FakeQueueClient.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/FakeQueue/FakeQueueClient.cs index e2f59596a..5744df389 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/FakeQueue/FakeQueueClient.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/FakeQueue/FakeQueueClient.cs @@ -1,19 +1,19 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -using Microsoft.Azure.WebJobs.Host; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Config; -using Microsoft.Azure.WebJobs.Host.Triggers; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using System; +using Microsoft.Azure.WebJobs.Host.Bindings; +using Microsoft.Azure.WebJobs.Host.Config; +using Microsoft.Azure.WebJobs.Host.Triggers; +using Xunit; namespace Microsoft.Azure.WebJobs.Host.UnitTests { // For sending fake queue messages. - public class FakeQueueClient : IExtensionConfigProvider + public class FakeQueueClient : IExtensionConfigProvider, IConverter { public List _items = new List(); @@ -47,7 +47,7 @@ void IExtensionConfigProvider.Initialize(ExtensionConfigContext context) { this.SetConverters(cm); } - + cm.AddConverter(msg => msg.Message); cm.AddConverter(OtherFakeQueueData.ToEvent); @@ -56,17 +56,25 @@ void IExtensionConfigProvider.Initialize(ExtensionConfigContext context) var bf = new BindingFactory(nameResolver, cm); // Binds [FakeQueue] --> IAsyncCollector - var ruleOutput = bf.BindToAsyncCollector(BuildFromAttr); + var ruleOutput = bf.BindToCollector(BuildFromAttr); // Binds [FakeQueue] --> FakeQueueClient - var ruleClient = bf.BindToExactType((attr) => this); + var ruleClient = bf.BindToInput(this); extensions.RegisterBindingRules(ruleOutput, ruleClient); var triggerBindingProvider = new FakeQueueTriggerBindingProvider(this, cm); extensions.RegisterExtension(triggerBindingProvider); } - + + FakeQueueClient IConverter.Convert(FakeQueueAttribute attr) + { + // Ensure that you can access the state set by the custom IResolutionPolicy + Assert.Equal("value1", attr.State1); + Assert.Equal("value2", attr.State2); + + return this; + } private IAsyncCollector BuildFromAttr(FakeQueueAttribute attr) { @@ -75,7 +83,7 @@ private IAsyncCollector BuildFromAttr(FakeQueueAttribute attr) { _parent = this, _prefix = attr.Prefix - }; + }; } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/FunctionIndexerFactory.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/FunctionIndexerFactory.cs index 27761ce18..44c04f8b0 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/FunctionIndexerFactory.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/FunctionIndexerFactory.cs @@ -22,7 +22,7 @@ namespace Microsoft.Azure.WebJobs.Host.UnitTests { internal static class FunctionIndexerFactory { - public static FunctionIndexer Create(CloudStorageAccount account = null, INameResolver nameResolver = null, IExtensionRegistry extensionRegistry = null) + public static FunctionIndexer Create(CloudStorageAccount account = null, INameResolver nameResolver = null, IExtensionRegistry extensionRegistry = null, TraceWriter traceWriter = null) { Mock services = new Mock(MockBehavior.Strict); StorageClientFactory clientFactory = new StorageClientFactory(); @@ -38,30 +38,30 @@ public static FunctionIndexer Create(CloudStorageAccount account = null, INameRe ContextAccessor blobWrittenWatcherAccessor = new ContextAccessor(); ISharedContextProvider sharedContextProvider = new SharedContextProvider(); - TestTraceWriter logger = new TestTraceWriter(TraceLevel.Verbose); + TraceWriter logger = traceWriter ?? new TestTraceWriter(TraceLevel.Verbose); SingletonManager singletonManager = new SingletonManager(); IWebJobsExceptionHandler exceptionHandler = new WebJobsExceptionHandler(); + var blobsConfiguration = new JobHostBlobsConfiguration(); ITriggerBindingProvider triggerBindingProvider = DefaultTriggerBindingProvider.Create(nameResolver, storageAccountProvider, extensionTypeLocator, - new FixedHostIdProvider("test"), new SimpleQueueConfiguration(maxDequeueCount: 5), + new FixedHostIdProvider("test"), new SimpleQueueConfiguration(maxDequeueCount: 5), blobsConfiguration, exceptionHandler, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, sharedContextProvider, new DefaultExtensionRegistry(), singletonManager, logger); - IBindingProvider bindingProvider = DefaultBindingProvider.Create(nameResolver, storageAccountProvider, + IBindingProvider bindingProvider = DefaultBindingProvider.Create(nameResolver, null, storageAccountProvider, extensionTypeLocator, messageEnqueuedWatcherAccessor, blobWrittenWatcherAccessor, new DefaultExtensionRegistry()); - TraceWriter trace = new TestTraceWriter(TraceLevel.Verbose); IFunctionOutputLoggerProvider outputLoggerProvider = new NullFunctionOutputLoggerProvider(); IFunctionOutputLogger outputLogger = outputLoggerProvider.GetAsync(CancellationToken.None).Result; - IFunctionExecutor executor = new FunctionExecutor(new NullFunctionInstanceLogger(), outputLogger, exceptionHandler, trace); + IFunctionExecutor executor = new FunctionExecutor(new NullFunctionInstanceLogger(), outputLogger, exceptionHandler, logger); if (extensionRegistry == null) { extensionRegistry = new DefaultExtensionRegistry(); } - return new FunctionIndexer(triggerBindingProvider, bindingProvider, DefaultJobActivator.Instance, executor, extensionRegistry, new SingletonManager(), trace); + return new FunctionIndexer(triggerBindingProvider, bindingProvider, DefaultJobActivator.Instance, executor, extensionRegistry, new SingletonManager(), logger); } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Indexers/FunctionIndexerTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Indexers/FunctionIndexerTests.cs index 85db93d82..3f64e147e 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Indexers/FunctionIndexerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Indexers/FunctionIndexerTests.cs @@ -3,17 +3,19 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; +using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.Host.Bindings; using Microsoft.Azure.WebJobs.Host.Indexers; using Microsoft.Azure.WebJobs.Host.Protocols; +using Microsoft.Azure.WebJobs.Host.TestCommon; using Microsoft.Azure.WebJobs.Host.Triggers; using Moq; using Xunit; -using Xunit.Extensions; namespace Microsoft.Azure.WebJobs.Host.UnitTests.Indexers { @@ -114,6 +116,22 @@ public void IndexMethod_IfMethodReturnsTask_DoesNotThrow() index, CancellationToken.None).GetAwaiter().GetResult(); } + [Fact] + public async Task IndexMethod_IfMethodReturnsAsyncVoid_Throws() + { + var traceWriter = new TestTraceWriter(TraceLevel.Verbose); + + // Arrange + IFunctionIndexCollector index = CreateStubFunctionIndex(); + FunctionIndexer product = CreateProductUnderTest(traceWriter: traceWriter); + + // Act & Assert + await product.IndexMethodAsync(typeof(FunctionIndexerTests).GetMethod("ReturnAsyncVoid"), index, CancellationToken.None); + + var warning = traceWriter.Traces.First(p => p.Level == TraceLevel.Warning); + Assert.Equal("Function 'ReturnAsyncVoid' is async but does not return a Task. Your function may not run correctly.", warning.Message); + } + [Fact] public void IsJobMethod_ReturnsFalse_IfMethodHasUnresolvedGenericParameter() { @@ -268,9 +286,9 @@ private static IFunctionIndexCollector CreateDummyFunctionIndex() return new Mock(MockBehavior.Strict).Object; } - private static FunctionIndexer CreateProductUnderTest() + private static FunctionIndexer CreateProductUnderTest(TraceWriter traceWriter = null) { - return FunctionIndexerFactory.Create(); + return FunctionIndexerFactory.Create(traceWriter: traceWriter); } private static IFunctionIndexCollector CreateStubFunctionIndex() @@ -348,5 +366,11 @@ public static Task ReturnTask() { throw new NotImplementedException(); } + + [NoAutomaticTrigger] + public static async void ReturnAsyncVoid() + { + await Task.FromResult(0); + } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostConfigurationTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostConfigurationTests.cs index 6d5f4bf03..495fb7e0d 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostConfigurationTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostConfigurationTests.cs @@ -31,6 +31,7 @@ public void ConstructorDefaults() Assert.NotNull(config.Tracing); Assert.Equal(TraceLevel.Info, config.Tracing.ConsoleLevel); Assert.Equal(0, config.Tracing.Tracers.Count); + Assert.False(config.Blobs.CentralizedPoisonQueue); StorageClientFactory clientFactory = config.GetService(); Assert.NotNull(clientFactory); @@ -267,6 +268,20 @@ public void StorageClientFactory_GetterSetter() Assert.Same(customFactory, configuration.GetService()); } + [Fact] + public void ConverterManager_Getter() + { + JobHostConfiguration configuration = new JobHostConfiguration(); + + IConverterManager converterManager = configuration.ConverterManager; + Assert.NotNull(converterManager); + Assert.Same(converterManager, configuration.GetService()); + + var property = configuration.GetType().GetProperty("ConverterManager"); + Assert.True(property.CanRead); + Assert.False(property.CanWrite); // CM is read-only, although the collection itself can be mutated. + } + [Theory] [InlineData(null, false)] [InlineData("Blah", false)] diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostQueuesConfigurationTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostQueuesConfigurationTests.cs index c8b412539..774b4bd71 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostQueuesConfigurationTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostQueuesConfigurationTests.cs @@ -1,6 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; using Microsoft.Azure.WebJobs.Host.Queues; using Microsoft.Azure.WebJobs.Host.Queues.Listeners; using Xunit; @@ -12,10 +13,8 @@ public class JobHostQueuesConfigurationTests [Fact] public void Constructor_Defaults() { - // Arrange JobHostQueuesConfiguration config = new JobHostQueuesConfiguration(); - // Act & Assert Assert.Equal(16, config.BatchSize); Assert.Equal(8, config.NewBatchThreshold); Assert.Equal(typeof(DefaultQueueProcessorFactory), config.QueueProcessorFactory.GetType()); @@ -25,7 +24,6 @@ public void Constructor_Defaults() [Fact] public void NewBatchThreshold_CanSetAndGetValue() { - // Arrange JobHostQueuesConfiguration config = new JobHostQueuesConfiguration(); // Unless explicitly set, NewBatchThreshold will be computed based @@ -41,5 +39,16 @@ public void NewBatchThreshold_CanSetAndGetValue() config.BatchSize = 8; Assert.Equal(1000, config.NewBatchThreshold); } + + [Fact] + public void VisibilityTimeout_CanGetAndSetValue() + { + JobHostQueuesConfiguration config = new JobHostQueuesConfiguration(); + + Assert.Equal(TimeSpan.Zero, config.VisibilityTimeout); + + config.VisibilityTimeout = TimeSpan.FromSeconds(30); + Assert.Equal(TimeSpan.FromSeconds(30), config.VisibilityTimeout); + } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostTests.cs index 1fc5d0cfb..cb5278ef8 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/JobHostTests.cs @@ -491,7 +491,7 @@ public void IndexingExceptions_CanBeHandledByTraceWriter() Assert.Equal("BindingErrorsProgram.Invalid", fex.MethodName); // verify that the binding error was logged - Assert.Equal(5, traceWriter.Traces.Count); + Assert.Equal(4, traceWriter.Traces.Count); TraceEvent traceEvent = traceWriter.Traces[0]; Assert.Equal("Error indexing method 'BindingErrorsProgram.Invalid'", traceEvent.Message); Assert.Same(fex, traceEvent.Exception); @@ -503,7 +503,7 @@ public void IndexingExceptions_CanBeHandledByTraceWriter() Assert.True(traceEvent.Message.Contains("BindingErrorsProgram.Valid")); // verify that the job host was started successfully - traceEvent = traceWriter.Traces[4]; + traceEvent = traceWriter.Traces[3]; Assert.Equal("Job host started", traceEvent.Message); host.Stop(); @@ -569,7 +569,7 @@ public LambdaStorageAccountProvider(Func GetAccountAsync(string connectionStringName, + public Task TryGetAccountAsync(string connectionStringName, CancellationToken cancellationToken) { return _getAccountAsync.Invoke(connectionStringName, cancellationToken); @@ -661,7 +661,7 @@ public override void Trace(TraceEvent traceEvent) { fex.Handled = true; Errors.Add(fex); - } + } } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Loggers/TraceWriterFunctionInstanceLoggerTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Loggers/TraceWriterFunctionInstanceLoggerTests.cs index 0edeea548..e3ac43cef 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Loggers/TraceWriterFunctionInstanceLoggerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Loggers/TraceWriterFunctionInstanceLoggerTests.cs @@ -43,7 +43,8 @@ public async Task LogFunctionStartedAsync_CallsTraceWriter() TraceEvent traceEvent = _traceWriter.Traces[0]; Assert.Equal(TraceLevel.Info, traceEvent.Level); Assert.Equal(Host.TraceSource.Execution, traceEvent.Source); - Assert.Equal("Executing: 'TestJob' - Reason: 'TestReason'", traceEvent.Message); + + Assert.Equal(string.Format("Executing 'TestJob' (Reason='TestReason', Id={0})", message.FunctionInstanceId), traceEvent.Message); Assert.Equal(3, traceEvent.Properties.Count); Assert.Equal(message.HostInstanceId, traceEvent.Properties["MS_HostInstanceId"]); Assert.Equal(message.FunctionInstanceId, traceEvent.Properties["MS_FunctionInvocationId"]); @@ -82,7 +83,7 @@ public async Task LogFunctionCompletedAsync_CallsTraceWriter() TraceEvent traceEvent = _traceWriter.Traces[0]; Assert.Equal(TraceLevel.Info, traceEvent.Level); Assert.Equal(Host.TraceSource.Execution, traceEvent.Source); - Assert.Equal("Executed: 'TestJob' (Succeeded)", traceEvent.Message); + Assert.Equal(string.Format("Executed 'TestJob' (Succeeded, Id={0})", successMessage.FunctionInstanceId), traceEvent.Message); Assert.Equal(successMessage.HostInstanceId, traceEvent.Properties["MS_HostInstanceId"]); Assert.Equal(successMessage.FunctionInstanceId, traceEvent.Properties["MS_FunctionInvocationId"]); Assert.Same(successMessage.Function, traceEvent.Properties["MS_FunctionDescriptor"]); @@ -90,7 +91,7 @@ public async Task LogFunctionCompletedAsync_CallsTraceWriter() traceEvent = _traceWriter.Traces[1]; Assert.Equal(TraceLevel.Error, traceEvent.Level); Assert.Equal(Host.TraceSource.Execution, traceEvent.Source); - Assert.Equal("Executed: 'TestJob' (Failed)", traceEvent.Message); + Assert.Equal(string.Format("Executed 'TestJob' (Failed, Id={0})", failureMessage.FunctionInstanceId), traceEvent.Message); Assert.Same(ex, traceEvent.Exception); Assert.Equal(failureMessage.HostInstanceId, traceEvent.Properties["MS_HostInstanceId"]); Assert.Equal(failureMessage.FunctionInstanceId, traceEvent.Properties["MS_FunctionInvocationId"]); diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/PublicSurfaceTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/PublicSurfaceTests.cs index 1db68d581..76d6e649e 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/PublicSurfaceTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/PublicSurfaceTests.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; -using Microsoft.Azure.WebJobs; using Microsoft.Azure.WebJobs.Logging; using Xunit; @@ -53,20 +52,18 @@ public void LoggingPublicSurface_LimitedToSpecificTypes() { "FunctionId", "ActivationEvent", - "CloudTableInstanceCountLogger", "FunctionInstanceLogItem", - "FunctionInstanceLogItemExtensions", "FunctionInstanceStatus", "FunctionStatusExtensions", "FunctionVolumeTimelineEntry", "IAggregateEntry", "IFunctionDefinition", "IFunctionInstanceBaseEntry", + "IFunctionInstanceBaseEntryExtensions", "ILogReader", "ILogWriter", "ILogTableProvider", "InstanceCountEntity", - "InstanceCountLoggerBase", "IRecentFunctionEntry", "LogFactory", "ProjectionHelper", @@ -128,7 +125,8 @@ public void WebJobsPublicSurface_LimitedToSpecificTypes() "StorageAccountAttribute", "DisableAttribute", "TimeoutAttribute", - "TraceLevelAttribute" + "TraceLevelAttribute", + "ODataFilterResolutionPolicy" }; AssertPublicTypes(expected, assembly); @@ -143,6 +141,8 @@ public void WebJobsHostPublicSurface_LimitedToSpecificTypes() { "DefaultNameResolver", "FunctionInstanceLogEntry", + "IConverter`2", + "IAsyncConverter`2", "IConverterManager", "IConverterManagerExtensions", "FuncConverter`3", @@ -152,6 +152,7 @@ public void WebJobsHostPublicSurface_LimitedToSpecificTypes() "JobHost", "JobHostConfiguration", "JobHostQueuesConfiguration", + "JobHostBlobsConfiguration", "IJobActivator", "ITypeLocator", "INameResolver", @@ -160,6 +161,7 @@ public void WebJobsHostPublicSurface_LimitedToSpecificTypes() "BindingProviderContext", "BindingTemplate", "BindStepOrder", + "OpenType", "FunctionBindingContext", "IBinding", "IBindingProvider", @@ -194,7 +196,6 @@ public void WebJobsHostPublicSurface_LimitedToSpecificTypes() "FunctionResult", "IArgumentBinding`1", "IArgumentBindingProvider`1", - "ITableArgumentBinding", "SingletonConfiguration", "TraceWriter", "JobHostTraceConfiguration", @@ -209,7 +210,9 @@ public void WebJobsHostPublicSurface_LimitedToSpecificTypes() "Binder", "IWebJobsExceptionHandler", "WebJobsExceptionHandler", - "FunctionTimeoutException" + "FunctionTimeoutException", + "PoisonMessageEventArgs", + "IResolutionPolicy" }; AssertPublicTypes(expected, assembly); diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Queues/QueueListenerTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Queues/QueueListenerTests.cs index a4c347836..8501c4a41 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Queues/QueueListenerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Queues/QueueListenerTests.cs @@ -33,7 +33,6 @@ public QueueListenerTests() mockQueue.Setup(p => p.SdkObject).Returns(queue); _mockTriggerExecutor = new Mock>(MockBehavior.Strict); - Mock mockDelayStrategy = new Mock(MockBehavior.Strict); Mock mockExceptionDispatcher = new Mock(MockBehavior.Strict); TestTraceWriter log = new TestTraceWriter(TraceLevel.Verbose); Mock mockQueueProcessorFactory = new Mock(MockBehavior.Strict); @@ -49,7 +48,7 @@ public QueueListenerTests() mockQueueProcessorFactory.Setup(p => p.Create(It.IsAny())).Returns(_mockQueueProcessor.Object); - _listener = new QueueListener(mockQueue.Object, null, _mockTriggerExecutor.Object, mockDelayStrategy.Object, mockExceptionDispatcher.Object, log, null, queueConfig); + _listener = new QueueListener(mockQueue.Object, null, _mockTriggerExecutor.Object, mockExceptionDispatcher.Object, log, null, queueConfig); CloudQueueMessage cloudMessage = new CloudQueueMessage("TestMessage"); _storageMessage = new StorageQueueMessage(cloudMessage); @@ -61,7 +60,7 @@ public void CreateQueueProcessor_CreatesProcessorCorrectly() CloudQueue poisonQueue = null; TestTraceWriter log = new TestTraceWriter(TraceLevel.Verbose); bool poisonMessageHandlerInvoked = false; - EventHandler poisonMessageEventHandler = (sender, e) => { poisonMessageHandlerInvoked = true; }; + EventHandler poisonMessageEventHandler = (sender, e) => { poisonMessageHandlerInvoked = true; }; Mock mockQueueProcessorFactory = new Mock(MockBehavior.Strict); JobHostQueuesConfiguration queueConfig = new JobHostQueuesConfiguration { @@ -76,7 +75,7 @@ public void CreateQueueProcessor_CreatesProcessorCorrectly() QueueProcessor queueProcessor = QueueListener.CreateQueueProcessor(queue, poisonQueue, log, queueConfig, poisonMessageEventHandler); Assert.False(processorFactoryInvoked); Assert.NotSame(expectedQueueProcessor, queueProcessor); - queueProcessor.OnMessageAddedToPoisonQueue(new EventArgs()); + queueProcessor.OnMessageAddedToPoisonQueue(new PoisonMessageEventArgs(null, poisonQueue)); Assert.True(poisonMessageHandlerInvoked); QueueProcessorFactoryContext processorFactoryContext = null; @@ -113,7 +112,7 @@ public void CreateQueueProcessor_CreatesProcessorCorrectly() queueProcessor = QueueListener.CreateQueueProcessor(queue, poisonQueue, log, queueConfig, poisonMessageEventHandler); Assert.True(processorFactoryInvoked); Assert.Same(expectedQueueProcessor, queueProcessor); - queueProcessor.OnMessageAddedToPoisonQueue(new EventArgs()); + queueProcessor.OnMessageAddedToPoisonQueue(new PoisonMessageEventArgs(null, poisonQueue)); Assert.True(poisonMessageHandlerInvoked); // if poison message watcher not specified, event not subscribed to @@ -122,7 +121,7 @@ public void CreateQueueProcessor_CreatesProcessorCorrectly() queueProcessor = QueueListener.CreateQueueProcessor(queue, poisonQueue, log, queueConfig, null); Assert.True(processorFactoryInvoked); Assert.Same(expectedQueueProcessor, queueProcessor); - queueProcessor.OnMessageAddedToPoisonQueue(new EventArgs()); + queueProcessor.OnMessageAddedToPoisonQueue(new PoisonMessageEventArgs(null, poisonQueue)); Assert.False(poisonMessageHandlerInvoked); } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Queues/QueueProcessorFactoryContextTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Queues/QueueProcessorFactoryContextTests.cs index 0a7675907..f80b2bc22 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Queues/QueueProcessorFactoryContextTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Queues/QueueProcessorFactoryContextTests.cs @@ -30,6 +30,7 @@ public void Constructor_DefaultsValues() Assert.Equal(queuesConfig.BatchSize, context.BatchSize); Assert.Equal(queuesConfig.NewBatchThreshold, context.NewBatchThreshold); Assert.Equal(queuesConfig.MaxDequeueCount, context.MaxDequeueCount); + Assert.Equal(queuesConfig.MaxPollingInterval, context.MaxPollingInterval); } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonListenerTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonListenerTests.cs index b29dbfff4..be02dea00 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonListenerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonListenerTests.cs @@ -5,10 +5,11 @@ using System.Reflection; using System.Threading; using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Azure.WebJobs.Host.Listeners; +using Microsoft.Azure.WebJobs.Host.TestCommon; using Moq; using Xunit; -using Microsoft.Azure.WebJobs.Host.Executors; namespace Microsoft.Azure.WebJobs.Host.UnitTests.Singleton { @@ -24,7 +25,7 @@ public class SingletonListenerTests public SingletonListenerTests() { - MethodInfo methodInfo = this.GetType().GetMethod("TestJob", BindingFlags.Static|BindingFlags.NonPublic); + MethodInfo methodInfo = this.GetType().GetMethod("TestJob", BindingFlags.Static | BindingFlags.NonPublic); _attribute = new SingletonAttribute(); _config = new SingletonConfiguration { @@ -33,7 +34,7 @@ public SingletonListenerTests() _mockSingletonManager = new Mock(MockBehavior.Strict, null, null, null, null, new FixedHostIdProvider(TestHostId), null); _mockSingletonManager.SetupGet(p => p.Config).Returns(_config); _mockInnerListener = new Mock(MockBehavior.Strict); - _listener = new SingletonListener(methodInfo, _attribute, _mockSingletonManager.Object, _mockInnerListener.Object); + _listener = new SingletonListener(methodInfo, _attribute, _mockSingletonManager.Object, _mockInnerListener.Object, new TestTraceWriter(System.Diagnostics.TraceLevel.Verbose)); _lockId = SingletonManager.FormatLockId(methodInfo, SingletonScope.Function, TestHostId, _attribute.ScopeId) + ".Listener"; } @@ -58,8 +59,8 @@ public async Task StartAsync_StartsListener_WhenLockAcquired() public async Task StartAsync_DoesNotStartListener_WhenLockCannotBeAcquired() { CancellationToken cancellationToken = new CancellationToken(); - _mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, cancellationToken, false)) - .ReturnsAsync(null); + _mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, cancellationToken, false)) + .ReturnsAsync((object)null); await _listener.StartAsync(cancellationToken); @@ -81,7 +82,7 @@ public async Task StartAsync_DoesNotStartLockTimer_WhenPollingIntervalSetToInfin CancellationToken cancellationToken = new CancellationToken(); _mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, cancellationToken, true)) - .ReturnsAsync(null); + .ReturnsAsync((object)null); await _listener.StartAsync(cancellationToken); @@ -121,7 +122,7 @@ public async Task TryAcquireLock_LockNotAcquired_DoesNotStopLockTimer() _listener.LockTimer.Start(); _mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, CancellationToken.None, false)) - .ReturnsAsync(null); + .ReturnsAsync((object)null); Assert.True(_listener.LockTimer.Enabled); @@ -142,7 +143,7 @@ public async Task StopAsync_WhenLockNotAcquired_StopsLockTimer() { CancellationToken cancellationToken = new CancellationToken(); _mockSingletonManager.Setup(p => p.TryLockAsync(_lockId, null, _attribute, cancellationToken, false)) - .ReturnsAsync(null); + .ReturnsAsync((object)null); await _listener.StartAsync(cancellationToken); diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonManagerTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonManagerTests.cs index f996388d1..b960c84d4 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonManagerTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonManagerTests.cs @@ -60,9 +60,9 @@ public SingletonManagerTests() mockSecondaryBlobClient.Setup(p => p.GetContainerReference(HostContainerNames.Hosts)).Returns(mockSecondaryBlobContainer.Object); _mockStorageAccount.Setup(p => p.CreateBlobClient(null)).Returns(mockBlobClient.Object); _mockSecondaryStorageAccount.Setup(p => p.CreateBlobClient(null)).Returns(mockSecondaryBlobClient.Object); - _mockAccountProvider.Setup(p => p.GetAccountAsync(ConnectionStringNames.Storage, It.IsAny())) + _mockAccountProvider.Setup(p => p.TryGetAccountAsync(ConnectionStringNames.Storage, It.IsAny())) .ReturnsAsync(_mockStorageAccount.Object); - _mockAccountProvider.Setup(p => p.GetAccountAsync(Secondary, It.IsAny())) + _mockAccountProvider.Setup(p => p.TryGetAccountAsync(Secondary, It.IsAny())) .ReturnsAsync(_mockSecondaryStorageAccount.Object); _mockExceptionDispatcher = new Mock(MockBehavior.Strict); @@ -77,7 +77,7 @@ public SingletonManagerTests() TestHelpers.SetField(_singletonConfig, "_lockPeriod", TimeSpan.FromMilliseconds(500)); _singletonConfig.LockAcquisitionTimeout = TimeSpan.FromMilliseconds(200); - _nameResolver = new TestNameResolver(); + _nameResolver = new TestNameResolver(); _singletonManager = new SingletonManager(_mockAccountProvider.Object, _mockExceptionDispatcher.Object, _singletonConfig, _trace, new FixedHostIdProvider(TestHostId), _nameResolver); _singletonManager.MinimumLeaseRenewalInterval = TimeSpan.FromMilliseconds(250); @@ -166,9 +166,7 @@ public async Task TryLockAsync_CreatesBlobLease_WithAutoRenewal() await _singletonManager.ReleaseLockAsync(lockHandle, cancellationToken); // verify the traces - Assert.Equal(1, _trace.Traces.Count(p => p.ToString().Contains("Verbose Waiting for Singleton lock (testid)"))); Assert.Equal(1, _trace.Traces.Count(p => p.ToString().Contains("Verbose Singleton lock acquired (testid)"))); - Assert.Equal(renewCount, _trace.Traces.Count(p => p.ToString().Contains("Renewing Singleton lock (testid)"))); Assert.Equal(1, _trace.Traces.Count(p => p.ToString().Contains("Verbose Singleton lock released (testid)"))); renewCount = 0; diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonValueProviderTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonValueProviderTests.cs index 6991b48b5..40a02e653 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonValueProviderTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Singleton/SingletonValueProviderTests.cs @@ -4,10 +4,11 @@ using System; using System.Reflection; using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Host.Executors; using Microsoft.Azure.WebJobs.Host.Protocols; using Moq; using Xunit; -using Microsoft.Azure.WebJobs.Host.Executors; namespace Microsoft.Azure.WebJobs.Host.UnitTests.Singleton { @@ -36,9 +37,9 @@ public void Type_ReturnsExpectedValue() } [Fact] - public void GetValue_ReturnsExpectedValue() + public async Task GetValueAsync_ReturnsExpectedValue() { - SingletonLock value = (SingletonLock)_valueProvider.GetValue(); + SingletonLock value = (SingletonLock)(await _valueProvider.GetValueAsync()); Assert.Equal(_lockId, value.Id); Assert.Equal(TestInstanceId, value.FunctionId); } @@ -51,30 +52,31 @@ public void Watcher_ReturnsExpectedValue() } [Fact] - public void ToInvokeString_ReturnsExpectedValue() + public async Task ToInvokeString_ReturnsExpectedValue() { SingletonManager singletonManager = new SingletonManager(null, null, null, null, new FixedHostIdProvider(TestHostId)); SingletonAttribute attribute = new SingletonAttribute(); SingletonValueProvider localValueProvider = new SingletonValueProvider(_method, attribute.ScopeId, TestInstanceId, attribute, singletonManager); - SingletonLock singletonLock = (SingletonLock)localValueProvider.GetValue(); + SingletonLock singletonLock = (SingletonLock)(await localValueProvider.GetValueAsync()); Assert.Equal("ScopeId: (null)", localValueProvider.ToInvokeString()); attribute = new SingletonAttribute(@"{Region}\{Zone}"); localValueProvider = new SingletonValueProvider(_method, @"Central\3", TestInstanceId, attribute, singletonManager); - singletonLock = (SingletonLock)localValueProvider.GetValue(); + singletonLock = (SingletonLock)(await localValueProvider.GetValueAsync()); Assert.Equal(@"ScopeId: Central\3", localValueProvider.ToInvokeString()); } [Fact] - public void SingletonWatcher_GetStatus_ReturnsExpectedValue() + public async Task SingletonWatcher_GetStatus_ReturnsExpectedValue() { Mock mockSingletonManager = new Mock(MockBehavior.Strict, null, null, null, null, new FixedHostIdProvider(TestHostId), null); mockSingletonManager.Setup(p => p.GetLockOwnerAsync(_attribute, _lockId, CancellationToken.None)).ReturnsAsync("someotherguy"); SingletonValueProvider localValueProvider = new SingletonValueProvider(_method, _attribute.ScopeId, TestInstanceId, _attribute, mockSingletonManager.Object); - SingletonLock localSingletonLock = (SingletonLock)localValueProvider.GetValue(); + SingletonLock localSingletonLock = (SingletonLock)(await localValueProvider.GetValueAsync()); - DateTime startTime = DateTime.Now; - DateTime endTime = startTime + TimeSpan.FromSeconds(2); + // set start time before _minimumWaitForFirstOwnerCheck in SingletonValueProvider + DateTime startTime = DateTime.UtcNow - TimeSpan.FromSeconds(11); + DateTime endTime = DateTime.UtcNow + TimeSpan.FromSeconds(2); DateTime releaseTime = endTime + TimeSpan.FromSeconds(1); // before lock is called diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Tables/TableArgumentBindingExtensionProviderTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Tables/TableArgumentBindingExtensionProviderTests.cs deleted file mode 100644 index c40b7eb02..000000000 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Tables/TableArgumentBindingExtensionProviderTests.cs +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Diagnostics; -using System.IO; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Azure.WebJobs.Host.Bindings; -using Microsoft.Azure.WebJobs.Host.Storage.Table; -using Microsoft.Azure.WebJobs.Host.Tables; -using Microsoft.Azure.WebJobs.Host.TestCommon; -using Microsoft.WindowsAzure.Storage.Table; -using Xunit; - -namespace Microsoft.Azure.WebJobs.Host.UnitTests.Tables -{ - public class TableArgumentBindingExtensionProviderTests - { - private static CloudTable BoundTable = null; - - private ParameterInfo[] _parameters; - - public TableArgumentBindingExtensionProviderTests() - { - _parameters = this.GetType().GetMethod("Parameters", BindingFlags.NonPublic | BindingFlags.Static).GetParameters(); - } - - [Fact] - public void TryCreate_DelegatesToExtensions() - { - DefaultExtensionRegistry extensions = new DefaultExtensionRegistry(); - TableArgumentBindingExtensionProvider provider = new TableArgumentBindingExtensionProvider(extensions); - - // before binding extensions are registered for these types, - // the provider returns null - - Assert.Null(provider.TryCreate(_parameters[0])); - Assert.Null(provider.TryCreate(_parameters[1])); - Assert.Null(provider.TryCreate(_parameters[2])); - - // register the binding extensions - FooBarTableArgumentBindingProvider fooBarExtensionProvider = new FooBarTableArgumentBindingProvider(); - BazTableArgumentBindingProvider bazExtensionProvider = new BazTableArgumentBindingProvider(); - extensions.RegisterExtension>(fooBarExtensionProvider); - extensions.RegisterExtension>(bazExtensionProvider); - provider = new TableArgumentBindingExtensionProvider(extensions); - - IStorageTableArgumentBinding binding = provider.TryCreate(_parameters[0]); - Assert.Same(typeof(IFoo), binding.ValueType); - - binding = provider.TryCreate(_parameters[1]); - Assert.Same(typeof(IBar), binding.ValueType); - - binding = provider.TryCreate(_parameters[2]); - Assert.Same(typeof(IBaz), binding.ValueType); - } - - [Fact] - public async Task TryCreate_ReturnsTableArgumentBindingExtensionWrapper() - { - DefaultExtensionRegistry extensions = new DefaultExtensionRegistry(); - FooBarTableArgumentBindingProvider fooBarExtensionProvider = new FooBarTableArgumentBindingProvider(); - extensions.RegisterExtension>(fooBarExtensionProvider); - - TableArgumentBindingExtensionProvider provider = new TableArgumentBindingExtensionProvider(extensions); - - IStorageTableArgumentBinding binding = provider.TryCreate(_parameters[0]); - Assert.Equal(typeof(TableArgumentBindingExtensionProvider.TableArgumentBindingExtension), binding.GetType()); - - Assert.Null(BoundTable); - CloudTable table = new CloudTable(new Uri("http://localhost:10000/test/table")); - IStorageTable storageTable = new StorageTable(table); - FunctionBindingContext functionContext = new FunctionBindingContext(Guid.NewGuid(), CancellationToken.None, new TestTraceWriter(TraceLevel.Verbose)); - ValueBindingContext context = new ValueBindingContext(functionContext, CancellationToken.None); - IValueProvider valueProvider = await binding.BindAsync(storageTable, context); - Assert.NotNull(valueProvider); - Assert.Same(table, BoundTable); - } - - private static void Parameters(IFoo foo, IBar bar, IBaz baz) { } - - private interface IFoo - { - } - - private interface IBar - { - } - - private interface IBaz - { - } - - private class FooBarTableArgumentBindingProvider : IArgumentBindingProvider - { - public ITableArgumentBinding TryCreate(ParameterInfo parameter) - { - if (parameter.ParameterType == typeof(IFoo) || - parameter.ParameterType == typeof(IBar)) - { - return new FooBarTableArgumentBinding(parameter.ParameterType); - } - - return null; - } - - internal class FooBarTableArgumentBinding : ITableArgumentBinding - { - private Type _valueType; - - public FooBarTableArgumentBinding(Type valueType) - { - _valueType = valueType; - } - - public FileAccess Access - { - get { return FileAccess.ReadWrite; } - } - - public Type ValueType - { - get { return _valueType; } - } - - public static object BindValue { get; set; } - - public Task BindAsync(CloudTable value, ValueBindingContext context) - { - BoundTable = value; - return Task.FromResult(new FooBarValueProvider()); - } - } - - internal class FooBarValueProvider : IValueProvider - { - public Type Type - { - get { throw new NotImplementedException(); } - } - - public object GetValue() - { - throw new NotImplementedException(); - } - - public string ToInvokeString() - { - throw new NotImplementedException(); - } - } - } - - private class BazTableArgumentBindingProvider : IArgumentBindingProvider - { - public ITableArgumentBinding TryCreate(ParameterInfo parameter) - { - if (parameter.ParameterType == typeof(IBaz)) - { - return new BazTableArgumentBinding(); - } - - return null; - } - - internal class BazTableArgumentBinding : ITableArgumentBinding - { - public FileAccess Access - { - get { return FileAccess.ReadWrite; } - } - - public Type ValueType - { - get { return typeof(IBaz); } - } - - public Task BindAsync(CloudTable value, ValueBindingContext context) - { - throw new NotImplementedException(); - } - } - } - } -} diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/Tables/TableFilterFormatterTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Tables/TableFilterFormatterTests.cs new file mode 100644 index 000000000..87d51b459 --- /dev/null +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/Tables/TableFilterFormatterTests.cs @@ -0,0 +1,191 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Azure.WebJobs.Host.Bindings.Path; +using Microsoft.Azure.WebJobs.Host.Tables; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Host.UnitTests.Tables +{ + public class TableFilterFormatterTests + { + [Fact] + public void Format_InvalidFilter_ThrowsExpectedException() + { + // verify an invalid integer value + var template = BindingTemplate.FromString("Age gt {Age}"); + var bindingData = new Dictionary + { + { "Age", "25 or true" } + }; + var ex = Assert.Throws(() => TableFilterFormatter.Format(template, (IReadOnlyDictionary)bindingData)); + Assert.Equal($"An invalid parameter value was specified for filter parameter 'Age'.", ex.Message); + + // verify the same template reversed + template = BindingTemplate.FromString("{Age} lt Age"); + bindingData = new Dictionary + { + { "Age", "25 or true" } + }; + ex = Assert.Throws(() => TableFilterFormatter.Format(template, (IReadOnlyDictionary)bindingData)); + Assert.Equal($"An invalid parameter value was specified for filter parameter 'Age'.", ex.Message); + + // verify datetime day operator value + template = BindingTemplate.FromString("day({BirthDate}) eq 8"); + bindingData = new Dictionary + { + { "BirthDate", "BirthDate) neq 8 or day(BirthDate" } + }; + ex = Assert.Throws(() => TableFilterFormatter.Format(template, (IReadOnlyDictionary)bindingData)); + Assert.Equal($"An invalid parameter value was specified for filter parameter 'BirthDate'.", ex.Message); + + // verify ambiguous values are handled correctly + template = BindingTemplate.FromString("Age eq '{Age}' or Age eq {Age}"); + bindingData = new Dictionary + { + { "Age", "25 or true" } + }; + ex = Assert.Throws(() => TableFilterFormatter.Format(template, (IReadOnlyDictionary)bindingData)); + Assert.Equal($"An invalid parameter value was specified for filter parameter 'Age'.", ex.Message); + } + + [Fact] + public void Format_StringLiteral_ReturnsExpectedValue() + { + // verify single quotes are escaped + var bindingData = new Dictionary + { + { "Name", "x' or 'x' eq 'x" } + }; + string result = GetFormattedValue("Name eq '{name}'", bindingData); + Assert.Equal("Name eq 'x'' or ''x'' eq ''x'", result); + + // verify the same template reversed + bindingData = new Dictionary + { + { "Name", "x' or 'x' eq 'x" } + }; + result = GetFormattedValue("'{name}' eq Name", bindingData); + Assert.Equal("'x'' or ''x'' eq ''x' eq Name", result); + + // verify a multipart filter + bindingData = new Dictionary + { + { "Name", "O'Malley" }, + { "Age", "35"} + }; + result = GetFormattedValue("(Age gt {Age}) and (Name eq '{Name}')", bindingData); + Assert.Equal("(Age gt 35) and (Name eq 'O''Malley')", result); + + // datetime filter range + bindingData = new Dictionary + { + { "D1", "2000-01-10" }, + { "D2", "2017-01-10" } + }; + result = GetFormattedValue("(BirthDate gt datetime'{D1}') and (BirthDate lt datetime'{D2}')", bindingData); + Assert.Equal("(BirthDate gt datetime'2000-01-10') and (BirthDate lt datetime'2017-01-10')", result); + + // guid filter + bindingData = new Dictionary + { + { "Category", "Electronics" }, + { "ID", "110C91D4-5412-4DAC-A960-EF0BCB8BAFEB" } + }; + result = GetFormattedValue("Category eq '{Category}' and ID eq guid'{ID}'", bindingData); + Assert.Equal("Category eq 'Electronics' and ID eq guid'110C91D4-5412-4DAC-A960-EF0BCB8BAFEB'", result); + + // invalid guid filter containing a quote + // we double quote it, but the value will fail to parse + // in table service + bindingData = new Dictionary + { + { "ID", "invalid'value" } + }; + result = GetFormattedValue("ID eq guid'{ID}'", bindingData); + Assert.Equal("ID eq guid'invalid''value'", result); + + // binary filter + bindingData = new Dictionary + { + { "Value", "TWFyeSBoYWQgYSBsaXR0bGUgbGFtYiE=" } + }; + result = GetFormattedValue("Binary eq X'{Value}'", bindingData); + Assert.Equal("Binary eq X'TWFyeSBoYWQgYSBsaXR0bGUgbGFtYiE='", result); + + // verify that the entire filter can be specified as an expression + bindingData = new Dictionary + { + { "Filter", "Name eq 'Curly'" } + }; + result = GetFormattedValue("{Filter}", bindingData); + Assert.Equal("Name eq 'Curly'", result); + } + + [Fact] + public void Format_MissingParameter_ThrowsExpectedException() + { + var bindingData = new Dictionary + { + { "Category", "Electronics" } + }; + var ex = Assert.Throws(() => + { + GetFormattedValue("Category eq '{Category}' and ID eq guid'{ID}'", bindingData); + }); + Assert.Equal("No value for named parameter 'ID'.", ex.Message); + } + + [Fact] + public void Format_SpecialTypes_ReturnsExpectedResult() + { + DateTime dateTime = new DateTime(2017, 1, 26, 19, 30, 00, DateTimeKind.Utc); + DateTimeOffset dateTimeOffset = new DateTimeOffset(dateTime); + Guid guid = new Guid("110c91d4-5412-4dac-a960-ef0bcb8bafeb"); + var bindingData = new Dictionary + { + { "DT", dateTime }, + { "DTO", dateTimeOffset }, + { "ID", guid } + }; + string result = GetFormattedValue("DateTime eq datetime'{DT}' and DateTimeOffset eq datetime'{DTO}' and ID eq guid'{ID}'", bindingData); + Assert.Equal("DateTime eq datetime'2017-01-26T19:30:00.0000000Z' and DateTimeOffset eq datetime'2017-01-26T19:30:00.0000000Z' and ID eq guid'110c91d4-5412-4dac-a960-ef0bcb8bafeb'", result); + } + + [Theory] + [InlineData("True", true)] + [InlineData("true", true)] + [InlineData("false", true)] + [InlineData("1", true)] + [InlineData("0", true)] + [InlineData("1234567", true)] + [InlineData("-1234567", true)] + [InlineData("12345.678", true)] + [InlineData("-12345.678", true)] + [InlineData("0.0", true)] + [InlineData("2017-01-23T09:13:28", false)] + [InlineData("2017-01-23T13:40:33.9300406-08:00", false)] + [InlineData("2017-01-23", false)] + [InlineData("110C91D4-5412-4DAC-A960-EF0BCB8BAFEB", false)] // binary (base 64) + [InlineData("TWFyeSBoYWQgYSBsaXR0bGUgbGFtYiE=", false)] + [InlineData("", false)] + [InlineData("test value", false)] + [InlineData("test'value", false)] + [InlineData("test(value", false)] + [InlineData("test)value", false)] + [InlineData("BirthDate) neq 8 or day(BirthDate", false)] + public void ValidateNonStringLiteral_ReturnsExpectedResult(string value, bool expectedResult) + { + bool result = TableFilterFormatter.TryValidateNonStringLiteral(value); + Assert.Equal(expectedResult, result); + } + + private static string GetFormattedValue(string templateString, Dictionary bindingData) + { + var template = BindingTemplate.FromString(templateString); + return TableFilterFormatter.Format(template, (IReadOnlyDictionary)bindingData); + } + } +} diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/TestJobHostContextFactory.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/TestJobHostContextFactory.cs index 9450af3c4..f07b02667 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/TestJobHostContextFactory.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/TestJobHostContextFactory.cs @@ -29,7 +29,7 @@ public Task CreateAndLogHostStartedAsync(JobHost host, Cancellat }; return JobHostContextFactory.CreateAndLogHostStartedAsync( - host, StorageAccountProvider, config.Queues, typeLocator, DefaultJobActivator.Instance, nameResolver, + host, StorageAccountProvider, config.Queues, config.Blobs, typeLocator, DefaultJobActivator.Instance, nameResolver, new NullConsoleProvider(), new JobHostConfiguration(), shutdownToken, cancellationToken, new WebJobsExceptionHandler(), new FixedHostIdProvider(Guid.NewGuid().ToString("N")), null, new EmptyFunctionIndexProvider(), diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/TypeUtilityTests.cs b/test/Microsoft.Azure.WebJobs.Host.UnitTests/TypeUtilityTests.cs index a30ae0fcb..f55065adb 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/TypeUtilityTests.cs +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/TypeUtilityTests.cs @@ -2,18 +2,41 @@ // Licensed under the MIT License. See License.txt in the project root for license information. using System; +using System.Reflection; +using System.Threading.Tasks; using Xunit; namespace Microsoft.Azure.WebJobs.Host.UnitTests { public class TypeUtilityTests { + [Theory] + [InlineData("VoidMethod", false)] + [InlineData("AsyncVoidMethod", true)] + [InlineData("AsyncTaskMethod", true)] + public void IsAsync_ReturnsExpectedResult(string methodName, bool expectedResult) + { + var method = typeof(TypeUtilityTests).GetMethod(methodName, BindingFlags.Public|BindingFlags.Static); + bool result = TypeUtility.IsAsync(method); + Assert.Equal(result, expectedResult); + } + + [Theory] + [InlineData("VoidMethod", false)] + [InlineData("AsyncVoidMethod", true)] + [InlineData("AsyncTaskMethod", false)] + public void IsAsyncVoid_ReturnsExpectedResult(string methodName, bool expectedResult) + { + var method = typeof(TypeUtilityTests).GetMethod(methodName, BindingFlags.Public | BindingFlags.Static); + bool result = TypeUtility.IsAsyncVoid(method); + Assert.Equal(result, expectedResult); + } + [Theory] [InlineData(typeof(TypeUtilityTests), false)] [InlineData(typeof(string), false)] [InlineData(typeof(int), false)] [InlineData(typeof(int?), true)] - [InlineData(typeof(Nullable), true)] public void IsNullable_ReturnsExpectedResult(Type type, bool expected) { Assert.Equal(expected, TypeUtility.IsNullable(type)); @@ -24,10 +47,13 @@ public void IsNullable_ReturnsExpectedResult(Type type, bool expected) [InlineData(typeof(string), "String")] [InlineData(typeof(int), "Int32")] [InlineData(typeof(int?), "Nullable")] - [InlineData(typeof(Nullable), "Nullable")] public void GetFriendlyName(Type type, string expected) { Assert.Equal(expected, TypeUtility.GetFriendlyName(type)); } + + public static void VoidMethod() { } + public static async void AsyncVoidMethod() { await Task.FromResult(0); } + public static async Task AsyncTaskMethod() { await Task.FromResult(0); } } } diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/WebJobs.Host.UnitTests.csproj b/test/Microsoft.Azure.WebJobs.Host.UnitTests/WebJobs.Host.UnitTests.csproj index b28974c80..867baa63c 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/WebJobs.Host.UnitTests.csproj +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/WebJobs.Host.UnitTests.csproj @@ -14,6 +14,9 @@ 512 + ..\test.ruleset + true + false true @@ -24,7 +27,7 @@ prompt 4 true - 5 + default pdbonly @@ -34,7 +37,7 @@ prompt 4 true - 5 + default true @@ -55,15 +58,15 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True @@ -74,8 +77,8 @@ ..\..\packages\WindowsAzure.Storage.7.2.1\lib\net40\Microsoft.WindowsAzure.Storage.dll True - - ..\..\packages\Moq.4.5.23\lib\net45\Moq.dll + + ..\..\packages\Moq.4.5.30\lib\net45\Moq.dll True @@ -89,7 +92,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True @@ -121,6 +124,7 @@ + @@ -137,6 +141,7 @@ + @@ -216,13 +221,13 @@ - + diff --git a/test/Microsoft.Azure.WebJobs.Host.UnitTests/packages.config b/test/Microsoft.Azure.WebJobs.Host.UnitTests/packages.config index c6fbf9926..3de92605c 100644 --- a/test/Microsoft.Azure.WebJobs.Host.UnitTests/packages.config +++ b/test/Microsoft.Azure.WebJobs.Host.UnitTests/packages.config @@ -2,13 +2,17 @@ - - - + + + - + - + + + + + diff --git a/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/InstanceCountLoggerBaseTests.cs b/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/InstanceCountLoggerBaseTests.cs index af3ce1fab..e49c34b5d 100644 --- a/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/InstanceCountLoggerBaseTests.cs +++ b/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/InstanceCountLoggerBaseTests.cs @@ -186,7 +186,7 @@ public void LongInstance() } // Pure in-memory logger. - public class TestLogger : InstanceCountLoggerBase + internal class TestLogger : InstanceCountLoggerBase { // Record WriteEntry results. Map Ticks --> value at that time. public Dictionary _dict = new Dictionary(); @@ -194,6 +194,10 @@ public class TestLogger : InstanceCountLoggerBase // Events to handshake between Poll() from test thread and WriteEntry() on background thread. private readonly AutoResetEvent _eventPollReady = new AutoResetEvent(false); private readonly AutoResetEvent _eventFinishedWrite = new AutoResetEvent(false); + + // Map of "last heartbeat". HeartbeatEntity.RowKey --> heartbeat value. + private Dictionary _heartbeats = new Dictionary(); + public long _newTicks; public int _totalActive; @@ -202,6 +206,15 @@ public TestLogger() { } + // For testing conveneince, get _heartbeats in an easily-comparable form + public string GetHeartbeatSummary() + { + return string.Join(";", + from kv in _heartbeats + orderby kv.Key + select string.Format("{0}={1}", kv.Key, kv.Value)); + } + // Callback from background Poller thread. protected override Task WriteEntry(long ticks, int currentActive, int totalThisPeriod) { diff --git a/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/LoggerTest.cs b/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/LoggerTest.cs index 15c0612e9..3a314a035 100644 --- a/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/LoggerTest.cs +++ b/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/LoggerTest.cs @@ -35,6 +35,35 @@ public void Dispose() } } + // Test abandonded status + [Fact] + public void StatusTest() + { + DateTime t1 = new DateTime(1950, 12, 1); + + // Stale heart beat is abandoned + var entity = new InstanceTableEntity + { + RowKey = Guid.NewGuid().ToString(), + StartTime = t1, + FunctionInstanceHeartbeatExpiry = t1, // stale heartbeat + + }; + var item = entity.ToFunctionLogItem(); + + Assert.Equal(FunctionInstanceStatus.Abandoned, item.GetStatus()); + + // recent heartbeat means running . + entity.FunctionInstanceHeartbeatExpiry = DateTime.UtcNow.AddMinutes(2); + item = entity.ToFunctionLogItem(); + Assert.Equal(FunctionInstanceStatus.Running, item.GetStatus()); + + // endtime means complete + entity.EndTime = DateTime.UtcNow; + item = entity.ToFunctionLogItem(); + Assert.Equal(FunctionInstanceStatus.CompletedSuccess, item.GetStatus()); + } + // End-2-end test that function instance counter can write to tables [Fact] public async Task FunctionInstance() @@ -369,11 +398,10 @@ public async Task LogStart() var entries = await GetRecentAsync(reader, l1.FunctionId); Assert.Equal(1, entries.Length); - Assert.Equal(entries[0].Status, FunctionInstanceStatus.Running); + Assert.Equal(entries[0].GetStatus(), FunctionInstanceStatus.Running); Assert.Equal(entries[0].EndTime, null); l1.EndTime = l1.StartTime.Add(TimeSpan.FromSeconds(1)); - l1.Status = FunctionInstanceStatus.CompletedSuccess; await writer.AddAsync(l1); await writer.FlushAsync(); @@ -382,8 +410,8 @@ public async Task LogStart() entries = await GetRecentAsync(reader, l1.FunctionId); Assert.Equal(1, entries.Length); - Assert.Equal(entries[0].Status, FunctionInstanceStatus.CompletedSuccess); - Assert.Equal(entries[0].EndTime.Value.DateTime, l1.EndTime); + Assert.Equal(entries[0].GetStatus(), FunctionInstanceStatus.CompletedSuccess); + Assert.Equal(entries[0].EndTime.Value, l1.EndTime); } // Logs are case-insensitive, case-preserving @@ -423,7 +451,7 @@ public async Task Casing() { var entries = await GetRecentAsync(reader, l1.FunctionId); Assert.Equal(1, entries.Length); - Assert.Equal(entries[0].Status, FunctionInstanceStatus.Running); + Assert.Equal(entries[0].GetStatus(), FunctionInstanceStatus.Running); Assert.Equal(entries[0].EndTime, null); Assert.Equal(entries[0].FunctionName, FuncOriginal); // preserving. } @@ -656,17 +684,8 @@ static async Task GetRecentAsync(ILogReader reader, Func static async Task WriteAsync(ILogWriter writer, FunctionInstanceLogItem item) { - item.Status = FunctionInstanceStatus.Running; await writer.AddAsync(item); // Start - if (item.ErrorDetails == null) - { - item.Status = FunctionInstanceStatus.CompletedSuccess; - } - else - { - item.Status = FunctionInstanceStatus.CompletedFailure; - } item.EndTime = item.StartTime.AddSeconds(1); await writer.AddAsync(item); // end } diff --git a/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/WebJobs.Logging.FunctionalTests.csproj b/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/WebJobs.Logging.FunctionalTests.csproj index eee2e0889..545c017d7 100644 --- a/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/WebJobs.Logging.FunctionalTests.csproj +++ b/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/WebJobs.Logging.FunctionalTests.csproj @@ -15,6 +15,9 @@ + ..\test.ruleset + true + false true @@ -24,7 +27,7 @@ DEBUG;TRACE prompt 4 - 5 + default pdbonly @@ -33,7 +36,7 @@ TRACE prompt 4 - 5 + default true @@ -50,15 +53,15 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True @@ -76,7 +79,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True diff --git a/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/packages.config b/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/packages.config index 21e77b936..c4d817c9d 100644 --- a/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/packages.config +++ b/test/Microsoft.Azure.WebJobs.Logging.FunctionalTests/packages.config @@ -1,12 +1,16 @@  - - - + + + - + + + + + diff --git a/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/BrokeredMessageToByteArrayConverterTests.cs b/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/BrokeredMessageToByteArrayConverterTests.cs index e0dc034e5..5b66125b5 100644 --- a/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/BrokeredMessageToByteArrayConverterTests.cs +++ b/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/BrokeredMessageToByteArrayConverterTests.cs @@ -1,7 +1,9 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; using System.IO; +using System.Runtime.Serialization; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -16,10 +18,10 @@ public class BrokeredMessageToByteArrayConverterTests private const string TestString = "This is a test!"; [Theory] - [InlineData(ContentTypes.ApplicationOctetStream)] [InlineData(ContentTypes.TextPlain)] [InlineData(ContentTypes.ApplicationJson)] - public async Task ConvertAsync_ReturnsExpectedResult(string contentType) + [InlineData(ContentTypes.ApplicationOctetStream)] + public async Task ConvertAsync_ReturnsExpectedResult_WithStream(string contentType) { MemoryStream ms = new MemoryStream(); StreamWriter sw = new StreamWriter(ms); @@ -35,5 +37,80 @@ public async Task ConvertAsync_ReturnsExpectedResult(string contentType) string decoded = Encoding.UTF8.GetString(result); Assert.Equal(TestString, decoded); } + + [Theory] + [InlineData(null)] + [InlineData("some-other-contenttype")] + public async Task ConvertAsync_Throws_WithStream_UnknownContentType(string contentType) + { + MemoryStream ms = new MemoryStream(); + StreamWriter sw = new StreamWriter(ms); + sw.Write(TestString); + sw.Flush(); + ms.Seek(0, SeekOrigin.Begin); + + BrokeredMessage message = new BrokeredMessage(ms); + message.ContentType = contentType; + BrokeredMessageToByteArrayConverter converter = new BrokeredMessageToByteArrayConverter(); + + var ex = await Assert.ThrowsAnyAsync(() => converter.ConvertAsync(message, CancellationToken.None)); + + Assert.IsType(ex.InnerException); + string expectedType = contentType ?? "null"; + Assert.StartsWith(string.Format("The BrokeredMessage with ContentType '{0}' failed to deserialize to a byte[] with the message: ", expectedType), ex.Message); + } + + [Theory] + [InlineData(ContentTypes.TextPlain)] + [InlineData(ContentTypes.ApplicationJson)] + [InlineData(ContentTypes.ApplicationOctetStream)] + public async Task ConvertAsync_ReturnsExpectedResult_WithSerializedByteArray_KnownContentType(string contentType) + { + byte[] bytes = Encoding.UTF8.GetBytes(TestString); + BrokeredMessage message = new BrokeredMessage(bytes); + message.ContentType = contentType; + BrokeredMessageToByteArrayConverter converter = new BrokeredMessageToByteArrayConverter(); + + byte[] result = await converter.ConvertAsync(message, CancellationToken.None); + + // we expect to read back the DataContract-serialized string, since the ContentType tells us to read it back as-is. + string decoded = Encoding.UTF8.GetString(result); + Assert.Equal("@ base64Binary3http://schemas.microsoft.com/2003/10/Serialization/�This is a test!", decoded); + } + + [Theory] + [InlineData(null)] + [InlineData("some-other-contenttype")] + public async Task ConvertAsync_ReturnsExpectedResult_WithSerializedByteArray_UnknownContentType(string contentType) + { + byte[] bytes = Encoding.UTF8.GetBytes(TestString); + BrokeredMessage message = new BrokeredMessage(bytes); + message.ContentType = contentType; + BrokeredMessageToByteArrayConverter converter = new BrokeredMessageToByteArrayConverter(); + + byte[] result = await converter.ConvertAsync(message, CancellationToken.None); + + // we expect to read back the DataContract-serialized string, since the ContentType tells us to read it back as-is. + string decoded = Encoding.UTF8.GetString(result); + Assert.Equal(TestString, decoded); + } + + [Fact] + public async Task ConvertAsync_Throws_WithSerializedObject() + { + BrokeredMessage message = new BrokeredMessage(new TestObject { Text = TestString }); + + BrokeredMessageToByteArrayConverter converter = new BrokeredMessageToByteArrayConverter(); + var exception = await Assert.ThrowsAsync(() => converter.ConvertAsync(message, CancellationToken.None)); + + Assert.IsType(exception.InnerException); + Assert.StartsWith("The BrokeredMessage with ContentType 'null' failed to deserialize to a byte[] with the message: ", exception.Message); + } + + [Serializable] + public class TestObject + { + public string Text { get; set; } + } } } diff --git a/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/BrokeredMessageToStringConverterTests.cs b/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/BrokeredMessageToStringConverterTests.cs index 66003fa4a..ab72cdd73 100644 --- a/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/BrokeredMessageToStringConverterTests.cs +++ b/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/BrokeredMessageToStringConverterTests.cs @@ -1,7 +1,9 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. +using System; using System.IO; +using System.Runtime.Serialization; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.WebJobs.ServiceBus.Triggers; @@ -19,7 +21,9 @@ public class BrokeredMessageToStringConverterTests [InlineData(ContentTypes.TextPlain, TestString)] [InlineData(ContentTypes.ApplicationJson, TestJson)] [InlineData(ContentTypes.ApplicationOctetStream, TestString)] - public async Task ConvertAsync_ReturnsExpectedResult(string contentType, string value) + [InlineData(null, TestJson)] + [InlineData("application/xml", TestJson)] + public async Task ConvertAsync_ReturnsExpectedResult_WithStream(string contentType, string value) { MemoryStream ms = new MemoryStream(); StreamWriter sw = new StreamWriter(ms); @@ -35,5 +39,39 @@ public async Task ConvertAsync_ReturnsExpectedResult(string contentType, string Assert.Equal(value, result); } + + [Theory] + [InlineData(ContentTypes.TextPlain, TestString)] + [InlineData(ContentTypes.ApplicationJson, TestJson)] + [InlineData(ContentTypes.ApplicationOctetStream, TestString)] + [InlineData(null, TestJson)] + [InlineData("application/xml", TestJson)] + public async Task ConvertAsync_ReturnsExpectedResult_WithSerializedString(string contentType, string value) + { + BrokeredMessage message = new BrokeredMessage(value); + message.ContentType = contentType; + + BrokeredMessageToStringConverter converter = new BrokeredMessageToStringConverter(); + string result = await converter.ConvertAsync(message, CancellationToken.None); + Assert.Equal(value, result); + } + + [Fact] + public async Task ConvertAsync_Throws_WithSerializedObject() + { + BrokeredMessage message = new BrokeredMessage(new TestObject { Text = TestString }); + + BrokeredMessageToStringConverter converter = new BrokeredMessageToStringConverter(); + var exception = await Assert.ThrowsAsync(() => converter.ConvertAsync(message, CancellationToken.None)); + + Assert.IsType(exception.InnerException); + Assert.StartsWith("The BrokeredMessage with ContentType 'null' failed to deserialize to a string with the message:", exception.Message); + } + + [Serializable] + public class TestObject + { + public string Text { get; set; } + } } } diff --git a/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/Listeners/ServiceBusListenerTests.cs b/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/Listeners/ServiceBusListenerTests.cs index 7a6d5c3a7..612ac173a 100644 --- a/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/Listeners/ServiceBusListenerTests.cs +++ b/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/Listeners/ServiceBusListenerTests.cs @@ -86,7 +86,7 @@ public async Task StartAsync_CallsMessagingProviderToCreateReceiver() { await _listener.StartAsync(CancellationToken.None); }); - Assert.Equal("Unable to connect to Service Bus using HTTP connectivity mode.", ex.Message); + Assert.Equal("No such host is known", ex.Message); _mockMessagingProvider.VerifyAll(); } diff --git a/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/WebJobs.ServiceBus.UnitTests.csproj b/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/WebJobs.ServiceBus.UnitTests.csproj index 5f435675f..f23c75d97 100644 --- a/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/WebJobs.ServiceBus.UnitTests.csproj +++ b/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/WebJobs.ServiceBus.UnitTests.csproj @@ -24,7 +24,7 @@ prompt 4 true - 5 + default pdbonly @@ -34,7 +34,7 @@ prompt 4 true - 5 + default true @@ -55,19 +55,19 @@ True - ..\..\packages\Microsoft.Data.Edm.5.8.1\lib\net40\Microsoft.Data.Edm.dll + ..\..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll True - ..\..\packages\Microsoft.Data.OData.5.8.1\lib\net40\Microsoft.Data.OData.dll + ..\..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll True - ..\..\packages\Microsoft.Data.Services.Client.5.8.1\lib\net40\Microsoft.Data.Services.Client.dll + ..\..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll True - ..\..\packages\WindowsAzure.ServiceBus.3.4.1\lib\net45-full\Microsoft.ServiceBus.dll + ..\..\packages\WindowsAzure.ServiceBus.3.4.5\lib\net45-full\Microsoft.ServiceBus.dll True @@ -78,8 +78,8 @@ ..\..\packages\WindowsAzure.Storage.7.2.1\lib\net40\Microsoft.WindowsAzure.Storage.dll True - - ..\..\packages\Moq.4.5.23\lib\net45\Moq.dll + + ..\..\packages\Moq.4.5.30\lib\net45\Moq.dll True @@ -94,7 +94,7 @@ - ..\..\packages\System.Spatial.5.8.1\lib\net40\System.Spatial.dll + ..\..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll True diff --git a/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/packages.config b/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/packages.config index 65d188aed..6a5c2fa26 100644 --- a/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/packages.config +++ b/test/Microsoft.Azure.WebJobs.ServiceBus.UnitTests/packages.config @@ -2,14 +2,18 @@ - - - + + + - + - - + + + + + + diff --git a/test/test.ruleset b/test/test.ruleset index 7bb99de9e..793eefb5f 100644 --- a/test/test.ruleset +++ b/test/test.ruleset @@ -40,5 +40,6 @@ + \ No newline at end of file diff --git a/tools/NuGetProj.settings.targets b/tools/NuGetProj.settings.targets index 6ef30976e..5c131c510 100644 --- a/tools/NuGetProj.settings.targets +++ b/tools/NuGetProj.settings.targets @@ -5,7 +5,7 @@ $(MSBuildThisFileDirectory) http://www.microsoft.com/web/webpi/eula/aspnetcomponent_rtw_enu.htm 2.0.0 - -beta2 + diff --git a/tools/SkipStrongNames.xml b/tools/SkipStrongNames.xml index 986b28c6e..98236936d 100644 --- a/tools/SkipStrongNames.xml +++ b/tools/SkipStrongNames.xml @@ -7,6 +7,7 @@ +