From 86fcc40c3e45113589aa7414243b32743908e812 Mon Sep 17 00:00:00 2001 From: Roman Yavnikov <45608740+Romazes@users.noreply.github.com> Date: Thu, 1 Aug 2024 01:02:13 +0300 Subject: [PATCH] Feature: Generic brokerage downloader wrapper (#8235) * feat: new constructor of AlgorithmNodePacket * refactor: extract JobQueue configuration * remove: not used `using` in IDataDownloader * feat: create BrokerageDataDownloader * Revert "refactor: extract JobQueue configuration" This reverts commit 5778936b712befd6e367bee3decc4a3565ca69af. * Revert "feat: new constructor of AlgorithmNodePacket" This reverts commit d7a565ff761243ae1391bd45fd10c93221f0329e. * feat: new config `data-download-brokerage` in DataDownloadProvider * refactor: initialize in BrokerageDataDownloader * remove: not used `using` in Program's DataDownloadProvider * remove: not used ref on QuantConnect.Queue proj * refactor: use default market based on SecurityType * refactor: MarketName in DataDownloadConfig struct test:feat: validate MarketName * feat: support Canonical Symbols in BrokerageDataDownloader * remove: not used command arguments * feat: init CacheProvider of IOptionChainProvider in Downloader * feat: add brokerage message event in BrokerageDataDownloader --- Common/IDataDownloader.cs | 1 - DownloaderDataProvider/DataDownloadConfig.cs | 11 +- .../Models/BrokerageDataDownloader.cs | 134 ++++++++++++++++++ DownloaderDataProvider/Program.cs | 9 +- DownloaderDataProvider/config.example.json | 5 +- .../DataDownloadConfigTests.cs | 57 ++++++++ 6 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 DownloaderDataProvider/Models/BrokerageDataDownloader.cs create mode 100644 Tests/DownloaderDataProvider/DataDownloadConfigTests.cs diff --git a/Common/IDataDownloader.cs b/Common/IDataDownloader.cs index 9858e9f18a8f..d7760951a777 100644 --- a/Common/IDataDownloader.cs +++ b/Common/IDataDownloader.cs @@ -13,7 +13,6 @@ * limitations under the License. */ -using System; using QuantConnect.Data; using System.Collections.Generic; diff --git a/DownloaderDataProvider/DataDownloadConfig.cs b/DownloaderDataProvider/DataDownloadConfig.cs index ecee122ca325..c6ea4b75748a 100644 --- a/DownloaderDataProvider/DataDownloadConfig.cs +++ b/DownloaderDataProvider/DataDownloadConfig.cs @@ -15,6 +15,8 @@ */ using System.Globalization; +using QuantConnect.Logging; +using QuantConnect.Brokerages; using QuantConnect.Configuration; using QuantConnect.DownloaderDataProvider.Launcher.Models.Constants; @@ -74,8 +76,15 @@ public DataDownloadConfig() EndDate = DateTime.ParseExact(Config.Get(DownloaderCommandArguments.CommandEndDate).ToString(), DateFormat.EightCharacter, CultureInfo.InvariantCulture); #pragma warning disable CA1308 // class Market keeps all name in lowercase - MarketName = Config.Get(DownloaderCommandArguments.CommandMarketName).ToString()?.ToLower(CultureInfo.InvariantCulture) ?? Market.USA; + MarketName = Config.Get(DownloaderCommandArguments.CommandMarketName).ToString().ToLower(CultureInfo.InvariantCulture); #pragma warning restore CA1308 + + if (string.IsNullOrEmpty(MarketName)) + { + MarketName = DefaultBrokerageModel.DefaultMarketMap[SecurityType]; + Log.Trace($"{nameof(DataDownloadConfig)}: Default market '{MarketName}' applied for SecurityType '{SecurityType}'"); + } + if (!Market.SupportedMarkets().Contains(MarketName)) { throw new ArgumentException($"The specified market '{MarketName}' is not supported. Supported markets are: {string.Join(", ", Market.SupportedMarkets())}."); diff --git a/DownloaderDataProvider/Models/BrokerageDataDownloader.cs b/DownloaderDataProvider/Models/BrokerageDataDownloader.cs new file mode 100644 index 000000000000..fcb7ca7e3cd1 --- /dev/null +++ b/DownloaderDataProvider/Models/BrokerageDataDownloader.cs @@ -0,0 +1,134 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Util; +using QuantConnect.Data; +using QuantConnect.Packets; +using QuantConnect.Interfaces; +using QuantConnect.Securities; +using QuantConnect.Configuration; + +namespace QuantConnect.DownloaderDataProvider.Launcher.Models +{ + /// + /// Class for downloading data from a brokerage. + /// + public class BrokerageDataDownloader : IDataDownloader + { + /// + /// Represents the Brokerage implementation. + /// + private IBrokerage _brokerage; + + /// + /// Provides access to exchange hours and raw data times zones in various markets + /// + private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + + /// + /// Initializes a new instance of the class. + /// + public BrokerageDataDownloader() + { + var liveNodeConfiguration = new LiveNodePacket() { Brokerage = Config.Get("data-downloader-brokerage") }; + + try + { + // import the brokerage data for the configured brokerage + var brokerageFactory = Composer.Instance.Single(factory => factory.BrokerageType.MatchesTypeName(liveNodeConfiguration.Brokerage)); + liveNodeConfiguration.BrokerageData = brokerageFactory.BrokerageData; + } + catch (InvalidOperationException error) + { + throw new InvalidOperationException($"{nameof(BrokerageDataDownloader)}.An error occurred while resolving brokerage data for a live job. Brokerage: {liveNodeConfiguration.Brokerage}.", error); + } + + _brokerage = Composer.Instance.GetExportedValueByTypeName(liveNodeConfiguration.Brokerage); + + _brokerage.Message += (object _, Brokerages.BrokerageMessageEvent e) => + { + if (e.Type == Brokerages.BrokerageMessageType.Error) + { + Logging.Log.Error(e.Message); + } + else + { + Logging.Log.Trace(e.Message); + } + }; + + ((IDataQueueHandler)_brokerage).SetJob(liveNodeConfiguration); + } + + /// + /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). + /// + /// model class for passing in parameters for historical data + /// Enumerable of base data for this symbol + public IEnumerable? Get(DataDownloaderGetParameters dataDownloaderGetParameters) + { + var symbol = dataDownloaderGetParameters.Symbol; + var resolution = dataDownloaderGetParameters.Resolution; + var startUtc = dataDownloaderGetParameters.StartUtc; + var endUtc = dataDownloaderGetParameters.EndUtc; + var tickType = dataDownloaderGetParameters.TickType; + + var dataType = LeanData.GetDataType(resolution, tickType); + var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); + var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); + + var symbols = new List { symbol }; + if (symbol.IsCanonical()) + { + symbols = GetChainSymbols(symbol, true).ToList(); + } + + return symbols + .Select(symbol => + { + var request = new Data.HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours: exchangeHours, dataTimeZone: dataTimeZone, resolution, + includeExtendedMarketHours: true, false, DataNormalizationMode.Raw, tickType); + + var history = _brokerage.GetHistory(request); + + if (history == null) + { + Logging.Log.Trace($"{nameof(BrokerageDataDownloader)}.{nameof(Get)}: Ignoring history request for unsupported symbol {symbol}"); + } + + return history; + }) + .Where(history => history != null) + .SelectMany(history => history); + } + + /// + /// Returns an IEnumerable of Future/Option contract symbols for the given root ticker + /// + /// The Symbol to get futures/options chain for + /// Include expired contracts + private IEnumerable GetChainSymbols(Symbol symbol, bool includeExpired) + { + if (_brokerage is IDataQueueUniverseProvider universeProvider) + { + return universeProvider.LookupSymbols(symbol, includeExpired); + } + else + { + throw new InvalidOperationException($"{nameof(BrokerageDataDownloader)}.{nameof(GetChainSymbols)}: The current brokerage does not support fetching canonical symbols. Please ensure your brokerage instance supports this feature."); + } + } + } +} diff --git a/DownloaderDataProvider/Program.cs b/DownloaderDataProvider/Program.cs index 9806286ea470..5556d5dcc6b9 100644 --- a/DownloaderDataProvider/Program.cs +++ b/DownloaderDataProvider/Program.cs @@ -15,13 +15,13 @@ */ using NodaTime; -using System.Timers; using QuantConnect.Util; using QuantConnect.Data; using QuantConnect.Logging; using QuantConnect.Interfaces; using QuantConnect.Securities; using QuantConnect.Configuration; +using QuantConnect.Lean.Engine.DataFeeds; using DataFeeds = QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.DownloaderDataProvider.Launcher.Models.Constants; @@ -181,6 +181,13 @@ public static void InitializeConfigurations() var mapFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("map-file-provider", "LocalDiskMapFileProvider")); var factorFileProvider = Composer.Instance.GetExportedValueByTypeName(Config.Get("factor-file-provider", "LocalDiskFactorFileProvider")); + var optionChainProvider = Composer.Instance.GetPart(); + if (optionChainProvider == null) + { + optionChainProvider = new CachingOptionChainProvider(new LiveOptionChainProvider(new ZipDataCacheProvider(dataProvider, false), mapFileProvider)); + Composer.Instance.AddPart(optionChainProvider); + } + mapFileProvider.Initialize(dataProvider); factorFileProvider.Initialize(mapFileProvider, dataProvider); } diff --git a/DownloaderDataProvider/config.example.json b/DownloaderDataProvider/config.example.json index 097fa79cbfde..9c29ad828ed2 100644 --- a/DownloaderDataProvider/config.example.json +++ b/DownloaderDataProvider/config.example.json @@ -13,5 +13,8 @@ "job-organization-id": "", // Data downloader provider - "data-downloader": "" + "data-downloader": "", + + // Specifies the name of the brokerage service used for data downloading (optional). + "data-downloader-brokerage": "" } diff --git a/Tests/DownloaderDataProvider/DataDownloadConfigTests.cs b/Tests/DownloaderDataProvider/DataDownloadConfigTests.cs new file mode 100644 index 000000000000..de0f1de07239 --- /dev/null +++ b/Tests/DownloaderDataProvider/DataDownloadConfigTests.cs @@ -0,0 +1,57 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using NUnit.Framework; +using QuantConnect.Configuration; +using QuantConnect.DownloaderDataProvider.Launcher; + +namespace QuantConnect.Tests.DownloaderDataProvider +{ + [TestFixture] + public class DataDownloadConfigTests + { + [TestCase(null, "BTCUSDT", SecurityType.Crypto, "coinbase", false)] + [TestCase(null, "BTCUSDT", SecurityType.Crypto, "coinbase", true)] + [TestCase("", "ETHUSDT", SecurityType.Crypto, "coinbase", false)] + [TestCase("", "ETHUSDT", SecurityType.Crypto, "coinbase", true)] + [TestCase(null, "AAPL", SecurityType.Equity, "usa", false)] + [TestCase(null, "AAPL", SecurityType.Equity, "usa", true)] + [TestCase("", "AAPL", SecurityType.Equity, "usa", false)] + [TestCase("", "AAPL", SecurityType.Equity, "usa", true)] + [TestCase("USA", "AAPL", SecurityType.Equity, "usa")] + [TestCase("ICE", "AAPL", SecurityType.Equity, "ice")] + public void ValidateMarketArguments(string market, string ticker, SecurityType securityType, string expectedMarket, bool skipConfigMarket = false) + { + Config.Set("data-type", "Trade"); + Config.Set("resolution", "Daily"); + Config.Set("security-type", $"{securityType}"); + Config.Set("tickers", $"{{\"{ticker}\": \"\"}}"); + Config.Set("start-date", "20240101"); + Config.Set("end-date", "20240202"); + + if (!skipConfigMarket) + { + Config.Set("market", market); + } + + var dataDownloadConfig = new DataDownloadConfig(); + + Assert.That(dataDownloadConfig.MarketName, Is.EqualTo(expectedMarket)); + + Config.Reset(); + } + } +}