-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add streams-based trigger, include new tests that ensure multiple fun…
…ctions instances don't duplicate events
- Loading branch information
Showing
9 changed files
with
441 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| using System; | ||
| using System.Linq; | ||
| using System.Text.Json; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Azure.WebJobs.Host.Executors; | ||
| using StackExchange.Redis; | ||
|
|
||
|
|
||
| namespace Microsoft.Azure.WebJobs.Extensions.Redis | ||
| { | ||
| /// <summary> | ||
| /// Responsible for managing connections and listening to a given Azure Redis Cache. | ||
| /// </summary> | ||
| internal sealed class RedisStreamsListener : RedisPollingListenerBase | ||
| { | ||
| internal bool deleteAfterProcess; | ||
| internal string consumerGroup; | ||
| internal StreamPosition[] positions; | ||
| internal string consumerName; | ||
|
|
||
| public RedisStreamsListener(string connectionString, string keys, TimeSpan pollingInterval, int messagesPerWorker, int batchSize, string consumerGroup, bool deleteAfterProcess, ITriggeredFunctionExecutor executor) | ||
| : base(connectionString, keys, pollingInterval, messagesPerWorker, batchSize, executor) | ||
| { | ||
| this.consumerGroup = consumerGroup; | ||
| this.deleteAfterProcess = deleteAfterProcess; | ||
| this.positions = this.keys.Select((key) => new StreamPosition(key, StreamPosition.NewMessages)).ToArray(); | ||
| this.consumerName = Guid.NewGuid().ToString(); | ||
| } | ||
|
|
||
| public override async void BeforePolling() | ||
| { | ||
| IDatabase db = multiplexer.GetDatabase(); | ||
|
|
||
| // create consumer group for each stream key | ||
| foreach (RedisKey key in keys) | ||
| { | ||
| try | ||
| { | ||
| if (!await db.StreamCreateConsumerGroupAsync(key, consumerGroup)) | ||
| { | ||
| throw new Exception($"Could not create consumer group for stream key {key}"); | ||
| } | ||
| } | ||
| catch (RedisServerException e) | ||
| { | ||
| // consumer group already exists | ||
| if (!e.Message.Contains("BUSYGROUP")) | ||
| { | ||
| throw; | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public override async Task PollAsync(CancellationToken cancellationToken) | ||
| { | ||
| IDatabase db = multiplexer.GetDatabase(); | ||
| RedisStream[] streams = await db.StreamReadGroupAsync(positions, consumerGroup, consumerName, batchSize); | ||
|
|
||
| for (int i = 0; i < streams.Length; i++) | ||
| { | ||
| if (streams[i].Entries.Length > 0) | ||
| { | ||
| foreach (StreamEntry entry in streams[i].Entries) | ||
| { | ||
| var triggerValue = new RedisMessageModel | ||
| { | ||
| Trigger = streams[i].Key, | ||
| Message = JsonSerializer.Serialize(entry.Values.ToDictionary(value => value.Name.ToString(), value => value.Value.ToString())) | ||
| }; | ||
|
|
||
| await executor.TryExecuteAsync(new TriggeredFunctionData() { TriggerValue = triggerValue }, cancellationToken); | ||
| }; | ||
|
|
||
| RedisValue[] entryIds = streams[i].Entries.Select(entry => entry.Id).ToArray(); | ||
| await db.StreamAcknowledgeAsync(streams[i].Key, consumerGroup, entryIds); | ||
|
|
||
| if (deleteAfterProcess) | ||
| { | ||
| await db.StreamDeleteAsync(streams[i].Key, entryIds); | ||
| } | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| public override void BeforeClosing() | ||
| { | ||
| IDatabase db = multiplexer.GetDatabase(); | ||
| foreach (RedisKey key in keys) | ||
| { | ||
| db.StreamDeleteConsumerAsync(key, consumerGroup, consumerName); | ||
| } | ||
| } | ||
|
|
||
| public override Task<RedisPollingMetrics> GetMetricsAsync() | ||
| { | ||
| var metrics = new RedisPollingMetrics | ||
| { | ||
| Remaining = keys.Sum((key) => multiplexer.GetDatabase().StreamLength(key)), | ||
| Timestamp = DateTime.UtcNow, | ||
| }; | ||
|
|
||
| return Task.FromResult(metrics); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| using System; | ||
| using Microsoft.Azure.WebJobs.Description; | ||
|
|
||
| namespace Microsoft.Azure.WebJobs.Extensions.Redis | ||
| { | ||
| /// <summary> | ||
| /// Streams trigger binding attributes. | ||
| /// </summary> | ||
| [Binding] | ||
| [AttributeUsage(AttributeTargets.Parameter)] | ||
| public class RedisStreamsTriggerAttribute : RedisPollingTriggerAttributeBase | ||
| { | ||
| /// <summary> | ||
| /// Name of the consumer group to use when reading the streams. | ||
| /// </summary> | ||
| public string ConsumerGroup { get; set; } = "AzureFunctionRedisExtension"; | ||
|
|
||
| /// <summary> | ||
| /// If true, the function will delete the stream entries after processing. | ||
| /// </summary> | ||
| public bool DeleteAfterProcess { get; set; } = false; | ||
|
|
||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| using System; | ||
| using System.Threading.Tasks; | ||
| using System.Collections.Generic; | ||
| using Microsoft.Azure.WebJobs.Host.Bindings; | ||
| using Microsoft.Azure.WebJobs.Host.Listeners; | ||
| using Microsoft.Azure.WebJobs.Host.Protocols; | ||
| using Microsoft.Azure.WebJobs.Host.Triggers; | ||
|
|
||
| namespace Microsoft.Azure.WebJobs.Extensions.Redis | ||
| { | ||
| /// <summary> | ||
| /// Trigger Binding, manages and binds context to listener. | ||
| /// </summary> | ||
| internal class RedisStreamsTriggerBinding : ITriggerBinding | ||
| { | ||
| private readonly string connectionString; | ||
| private readonly TimeSpan pollingInterval; | ||
| private readonly int messagesPerWorker; | ||
| private readonly string keys; | ||
| private readonly int count; | ||
| private readonly string consumerGroup; | ||
| private readonly bool deleteAfterProcess; | ||
|
|
||
| public RedisStreamsTriggerBinding(string connectionString, string keys, TimeSpan pollingInterval, int messagesPerWorker, int count, string consumerGroup, bool deleteAfterProcess) | ||
| { | ||
| this.connectionString = connectionString; | ||
| this.keys = keys; | ||
| this.pollingInterval = pollingInterval; | ||
| this.messagesPerWorker = messagesPerWorker; | ||
| this.count = count; | ||
| this.consumerGroup = consumerGroup; | ||
| this.deleteAfterProcess = deleteAfterProcess; | ||
| } | ||
|
|
||
| public Type TriggerValueType => typeof(RedisMessageModel); | ||
|
|
||
| public IReadOnlyDictionary<string, Type> BindingDataContract => new Dictionary<string, Type>(); | ||
|
|
||
| public Task<ITriggerData> BindAsync(object value, ValueBindingContext context) | ||
| { | ||
| IReadOnlyDictionary<string, object> bindingData = new Dictionary<string, object>(); | ||
| return Task.FromResult<ITriggerData>(new TriggerData(null, bindingData)); | ||
| } | ||
|
|
||
| public Task<IListener> CreateListenerAsync(ListenerFactoryContext context) | ||
| { | ||
| if (context == null) | ||
| { | ||
| throw new ArgumentNullException("context"); | ||
| } | ||
|
|
||
| return Task.FromResult<IListener>(new RedisStreamsListener(connectionString, keys, pollingInterval, messagesPerWorker, count, consumerGroup, deleteAfterProcess, context.Executor)); | ||
| } | ||
|
|
||
| public ParameterDescriptor ToParameterDescriptor() | ||
| { | ||
| return new ParameterDescriptor(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| using System; | ||
| using System.Reflection; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.Extensions.Configuration; | ||
| using Microsoft.Azure.WebJobs.Host.Triggers; | ||
|
|
||
| namespace Microsoft.Azure.WebJobs.Extensions.Redis | ||
| { | ||
| /// <summary> | ||
| /// Provides trigger binding, variables configured in local.settings.json are being retrieved here. | ||
| /// </summary> | ||
| internal class RedisStreamsTriggerBindingProvider : ITriggerBindingProvider | ||
| { | ||
| private readonly IConfiguration configuration; | ||
|
|
||
| public RedisStreamsTriggerBindingProvider(IConfiguration configuration) | ||
| { | ||
| this.configuration = configuration; | ||
| } | ||
|
|
||
| public Task<ITriggerBinding> TryCreateAsync(TriggerBindingProviderContext context) | ||
| { | ||
| if (context == null) | ||
| { | ||
| throw new ArgumentNullException("context"); | ||
| } | ||
|
|
||
| ParameterInfo parameter = context.Parameter; | ||
| RedisStreamsTriggerAttribute attribute = parameter.GetCustomAttribute<RedisStreamsTriggerAttribute>(inherit: false); | ||
|
|
||
| if (attribute == null) | ||
| { | ||
| return Task.FromResult<ITriggerBinding>(null); | ||
| } | ||
|
|
||
| string connectionString = RedisUtilities.ResolveString(configuration, attribute.ConnectionString, "ConnectionString"); | ||
| string keys = RedisUtilities.ResolveString(configuration, attribute.Keys, "Keys"); | ||
| int messagesPerWorker = attribute.MessagesPerWorker; | ||
| int batchSize = attribute.BatchSize; | ||
| TimeSpan pollingInterval = TimeSpan.FromMilliseconds(attribute.PollingIntervalInMs); | ||
| string consumerGroup = RedisUtilities.ResolveString(configuration, attribute.ConsumerGroup, "ConsumerGroup"); | ||
| bool deleteAfterProcess = attribute.DeleteAfterProcess; | ||
|
|
||
| return Task.FromResult<ITriggerBinding>(new RedisStreamsTriggerBinding(connectionString, keys, pollingInterval, messagesPerWorker, batchSize, consumerGroup, deleteAfterProcess)); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.