From 8cb3f3618662999231bdfce58c67fd4eb0573a1e Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Wed, 30 Aug 2023 16:38:52 -0500 Subject: [PATCH 1/9] Fix bug and add unit test After debugging the logs from the strategies deployed in the cloud, it was found the configuration name used by the API to use QC + IB data feed. Therefore, only the method which returned the data handler name was changed. --- lean/commands/cloud/live/deploy.py | 2 +- .../brokerages/cloud/cloud_brokerage.py | 5 ++++- .../cloud/live/test_cloud_live_commands.py | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index be68cfcb..043ca98f 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -251,7 +251,7 @@ def deploy(project: str, ensure_options(essential_properties) essential_properties_value = {brokerage_instance.convert_variable_to_lean_key(prop) : kwargs[prop] for prop in essential_properties} brokerage_instance.update_configs(essential_properties_value) - # now required properties can be fetched as per data provider from esssential properties + # now required properties can be fetched as per data provider from essential properties required_properties = [brokerage_instance.convert_lean_key_to_variable(prop) for prop in brokerage_instance.get_required_properties([InternalInputUserInput])] ensure_options(required_properties) required_properties_value = {brokerage_instance.convert_variable_to_lean_key(prop) : kwargs[prop] for prop in required_properties} diff --git a/lean/models/brokerages/cloud/cloud_brokerage.py b/lean/models/brokerages/cloud/cloud_brokerage.py index e306238b..5f35c3df 100644 --- a/lean/models/brokerages/cloud/cloud_brokerage.py +++ b/lean/models/brokerages/cloud/cloud_brokerage.py @@ -80,5 +80,8 @@ def get_price_data_handler(self) -> str: """ # TODO: Handle this case with json conditions if self.get_name() == "Interactive Brokers": - return "InteractiveBrokersHandler" if self.get_config_value_from_name("ib-data-feed") else "QuantConnectHandler" + if self.get_config_value_from_name("ib-data-feed") == "Interactive Brokers": + return "InteractiveBrokersHandler" + elif self.get_config_value_from_name("ib-data-feed") == "QuantConnect + InteractiveBrokers": + return "quantconnecthandler+interactivebrokershandler" return "QuantConnectHandler" diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index da87f486..a9cce023 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -94,6 +94,28 @@ def test_cloud_live_deploy() -> None: mock.ANY, mock.ANY) +def test_cloud_live_deploy_with_ib_using_hybrid_datafeed() -> None: + create_fake_lean_cli_directory() + + api_client = mock.Mock() + api_client.nodes.get_all.return_value = create_qc_nodes() + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} + container.api_client = api_client + + cloud_project_manager = mock.Mock() + container.cloud_project_manager = cloud_project_manager + + cloud_runner = mock.Mock() + container.cloud_runner = cloud_runner + + result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", "Interactive Brokers", "--node", "live", + "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no", + "--ib-data-feed", "QuantConnect + InteractiveBrokers", "--ib-user-name", "test_user", + "--ib-account", "DU2366417", "--ib-password", "test_password"]) + + assert result.exit_code == 0 + assert "Data provider: quantconnecthandler+interactivebrokershandler" in result.output.split("\n") + @pytest.mark.parametrize("notice_method,configs", [("emails", "customAddress:customSubject"), ("emails", "customAddress1:customSubject1,customAddress2:customSubject2"), ("webhooks", "customAddress:header1=value1"), From e46a0152ebfceaae648e46409c1cd54ea440f0dd Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Fri, 1 Sep 2023 15:20:43 -0500 Subject: [PATCH 2/9] Modify get_price_data_handler() to rely on json --- .../brokerages/cloud/cloud_brokerage.py | 11 +++++----- .../cloud/live/test_cloud_live_commands.py | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/lean/models/brokerages/cloud/cloud_brokerage.py b/lean/models/brokerages/cloud/cloud_brokerage.py index 5f35c3df..7d82d737 100644 --- a/lean/models/brokerages/cloud/cloud_brokerage.py +++ b/lean/models/brokerages/cloud/cloud_brokerage.py @@ -79,9 +79,10 @@ def get_price_data_handler(self) -> str: :return: the value to assign to the "dataHandler" property of the live/create API endpoint """ # TODO: Handle this case with json conditions - if self.get_name() == "Interactive Brokers": - if self.get_config_value_from_name("ib-data-feed") == "Interactive Brokers": - return "InteractiveBrokersHandler" - elif self.get_config_value_from_name("ib-data-feed") == "QuantConnect + InteractiveBrokers": - return "quantconnecthandler+interactivebrokershandler" + [property_name] = [name for name in self.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)] + brokerage_name = self.get_name().replace(" ", "") + if brokerage_name == self.get_config_value_from_name(property_name): + return self.get_name().replace(" ", "") + "Handler" + elif brokerage_name in self.get_config_value_from_name(property_name): + return "quantconnecthandler+" + brokerage_name.lower() + "handler" return "QuantConnectHandler" diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index a9cce023..ebc9ca91 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -116,6 +116,27 @@ def test_cloud_live_deploy_with_ib_using_hybrid_datafeed() -> None: assert result.exit_code == 0 assert "Data provider: quantconnecthandler+interactivebrokershandler" in result.output.split("\n") +def test_cloud_live_deploy_with_tradier_using_tradier_datafeed() -> None: + create_fake_lean_cli_directory() + + api_client = mock.Mock() + api_client.nodes.get_all.return_value = create_qc_nodes() + container.api_client = api_client + + cloud_project_manager = mock.Mock() + container.cloud_project_manager = cloud_project_manager + + cloud_runner = mock.Mock() + container.cloud_runner = cloud_runner + + result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", "Tradier", "--node", "live", + "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no", + "--tradier-data-feed", "Tradier", "--tradier-account-id", "123", + "--tradier-access-token", "456", "--tradier-environment", "paper"]) + + assert result.exit_code == 0 + assert "Data provider: Tradier" in result.output.split("\n") + @pytest.mark.parametrize("notice_method,configs", [("emails", "customAddress:customSubject"), ("emails", "customAddress1:customSubject1,customAddress2:customSubject2"), ("webhooks", "customAddress:header1=value1"), From 262612a1d96d1b0e2371f9e220a68bab5fe7899c Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Fri, 1 Sep 2023 15:39:31 -0500 Subject: [PATCH 3/9] Solve bugs --- lean/models/brokerages/cloud/cloud_brokerage.py | 12 +++++++----- .../commands/cloud/live/test_cloud_live_commands.py | 4 +++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lean/models/brokerages/cloud/cloud_brokerage.py b/lean/models/brokerages/cloud/cloud_brokerage.py index 7d82d737..7daf086c 100644 --- a/lean/models/brokerages/cloud/cloud_brokerage.py +++ b/lean/models/brokerages/cloud/cloud_brokerage.py @@ -79,10 +79,12 @@ def get_price_data_handler(self) -> str: :return: the value to assign to the "dataHandler" property of the live/create API endpoint """ # TODO: Handle this case with json conditions - [property_name] = [name for name in self.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)] + property_name = [name for name in self.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)] + property_name = property_name[0] if len(property_name) != 0 else "" brokerage_name = self.get_name().replace(" ", "") - if brokerage_name == self.get_config_value_from_name(property_name): - return self.get_name().replace(" ", "") + "Handler" - elif brokerage_name in self.get_config_value_from_name(property_name): - return "quantconnecthandler+" + brokerage_name.lower() + "handler" + if property_name != "": + if brokerage_name == self.get_config_value_from_name(property_name): + return self.get_name().replace(" ", "") + "Handler" + elif brokerage_name in self.get_config_value_from_name(property_name): + return "quantconnecthandler+" + brokerage_name.lower() + "handler" return "QuantConnectHandler" diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index ebc9ca91..fc706dce 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -330,7 +330,9 @@ def test_cloud_live_deploy_with_live_holdings(brokerage: str, holdings: str) -> if brokerage == "Trading Technologies": options.extend(["--live-cash-balance", "USD:100"]) elif brokerage == "Interactive Brokers": - options.extend(["--ib-data-feed", "no"]) + options.extend(["--ib-data-feed", "QuantConnect"]) + elif brokerage == "Tradier": + options.extend(["--tradier-data-feed", "QuantConnect"]) result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", brokerage, "--live-holdings", holdings, "--node", "live", "--auto-restart", "yes", "--notify-order-events", "no", From c01259fb5195b3b3267b4c14d5e838bf64ff897e Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Wed, 6 Sep 2023 12:48:20 -0500 Subject: [PATCH 4/9] Make get_price_data_handler() more generic --- lean/models/brokerages/cloud/cloud_brokerage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lean/models/brokerages/cloud/cloud_brokerage.py b/lean/models/brokerages/cloud/cloud_brokerage.py index 7daf086c..fba89d04 100644 --- a/lean/models/brokerages/cloud/cloud_brokerage.py +++ b/lean/models/brokerages/cloud/cloud_brokerage.py @@ -83,8 +83,8 @@ def get_price_data_handler(self) -> str: property_name = property_name[0] if len(property_name) != 0 else "" brokerage_name = self.get_name().replace(" ", "") if property_name != "": - if brokerage_name == self.get_config_value_from_name(property_name): - return self.get_name().replace(" ", "") + "Handler" - elif brokerage_name in self.get_config_value_from_name(property_name): + if "QuantConnect +" in self.get_config_value_from_name(property_name): return "quantconnecthandler+" + brokerage_name.lower() + "handler" + else: + return self.get_config_value_from_name(property_name).replace(" ", "") + "Handler" return "QuantConnectHandler" From d2f63c229d250d874b26caf7e8389ed27afd258a Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Wed, 6 Sep 2023 12:52:27 -0500 Subject: [PATCH 5/9] Fix bug --- tests/commands/cloud/live/test_cloud_live_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index fc706dce..f53cc574 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -131,11 +131,11 @@ def test_cloud_live_deploy_with_tradier_using_tradier_datafeed() -> None: result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", "Tradier", "--node", "live", "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no", - "--tradier-data-feed", "Tradier", "--tradier-account-id", "123", + "--tradier-data-feed", "Tradier Brokerage", "--tradier-account-id", "123", "--tradier-access-token", "456", "--tradier-environment", "paper"]) assert result.exit_code == 0 - assert "Data provider: Tradier" in result.output.split("\n") + assert "Data provider: TradierBrokerage" in result.output.split("\n") @pytest.mark.parametrize("notice_method,configs", [("emails", "customAddress:customSubject"), ("emails", "customAddress1:customSubject1,customAddress2:customSubject2"), From da03a3628ceb6eb0c89b76e12e5d695f48ad1bb4 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Wed, 6 Sep 2023 13:40:08 -0500 Subject: [PATCH 6/9] update version of modules json file --- lean/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean/models/__init__.py b/lean/models/__init__.py index c824d28a..2eba3c18 100644 --- a/lean/models/__init__.py +++ b/lean/models/__init__.py @@ -17,7 +17,7 @@ from time import time json_modules = {} -file_name = "modules-1.11.json" +file_name = "modules-1.12.json" directory = Path(__file__).parent file_path = directory.parent / file_name From a523bac1b9887177ea507ab9f61ea38335454e08 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Wed, 6 Sep 2023 15:42:24 -0500 Subject: [PATCH 7/9] Update README.md --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a3024c2c..53059f9e 100644 --- a/README.md +++ b/README.md @@ -269,12 +269,16 @@ Options: Weekly restart UTC time (hh:mm:ss). Each week on Sunday your algorithm is restarted at this time, and will require 2FA verification. This is required by Interactive Brokers. Use this option explicitly to override the default value. - --ib-data-feed BOOLEAN Whether the Interactive Brokers price data feed must be used instead of the - QuantConnect price data feed + --ib-data-feed [QuantConnect|Interactive Brokers|QuantConnect + InteractiveBrokers] + The data feed to use. These are the available ones: Interactive Brokers price data + feed, QuantConnect price data feed or QuantConnect + InteractiveBrokers price data feed. --tradier-account-id TEXT Your Tradier account id --tradier-access-token TEXT Your Tradier access token --tradier-environment [live|paper] Whether the developer sandbox should be used + --tradier-data-feed [QuantConnect|Tradier Brokerage] + The data feed to use. These are the available ones: QuantConnect price data feed or + Tradier Brokerage data feed. --oanda-account-id TEXT Your OANDA account id --oanda-access-token TEXT Your OANDA API token --oanda-environment [Practice|Trade] From ad5f7b2eabf545d53470d269570a82ca75ab15ac Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Fri, 8 Sep 2023 11:38:38 -0500 Subject: [PATCH 8/9] Add support for interactive mode --- lean/commands/cloud/live/deploy.py | 3 ++- lean/models/brokerages/cloud/__init__.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index 043ca98f..bab5fff5 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -25,7 +25,7 @@ from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage from lean.models.configuration import InternalInputUserInput from lean.models.click_options import options_from_json, get_configs_for_options -from lean.models.brokerages.cloud import all_cloud_brokerages +from lean.models.brokerages.cloud import all_cloud_brokerages, cloud_brokerage_data_feeds from lean.commands.cloud.live.live import live from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\ _configure_initial_cash_interactively, _configure_initial_holdings_interactively @@ -308,6 +308,7 @@ def deploy(project: str, else: lean_config = container.lean_config_manager.get_lean_config() brokerage_instance = _configure_brokerage(lean_config, logger, kwargs, show_secrets=show_secrets) + _configure_data_feed(brokerage_instance, logger) live_node = _configure_live_node(logger, api_client, cloud_project) notify_order_events, notify_insights, notify_methods = _configure_notifications(logger) auto_restart = _configure_auto_restart(logger) diff --git a/lean/models/brokerages/cloud/__init__.py b/lean/models/brokerages/cloud/__init__.py index b1e641de..72867060 100644 --- a/lean/models/brokerages/cloud/__init__.py +++ b/lean/models/brokerages/cloud/__init__.py @@ -13,13 +13,28 @@ from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage from lean.models import json_modules -from typing import List +from typing import Dict, Type, List +from lean.models.brokerages.local.data_feed import DataFeed all_cloud_brokerages: List[CloudBrokerage] = [] +all_cloud_data_feeds: List[DataFeed] = [] +cloud_brokerage_data_feeds: Dict[Type[CloudBrokerage], + List[Type[DataFeed]]] = {} for json_module in json_modules: if "cloud-brokerage" in json_module["type"]: all_cloud_brokerages.append(CloudBrokerage(json_module)) + if "data-queue-handler" in json_module["type"]: + all_cloud_data_feeds.append((DataFeed(json_module))) + +for cloud_brokerage in all_cloud_brokerages: + data_feed_property_found = False + for x in cloud_brokerage.get_all_input_configs(): + if "data-feed" in x.__getattribute__("_id"): + data_feed_property_found = True + cloud_brokerage_data_feeds[cloud_brokerage] = x.__getattribute__("_choices") + if not data_feed_property_found: + cloud_brokerage_data_feeds[cloud_brokerage] = [] [PaperTradingBrokerage] = [ cloud_brokerage for cloud_brokerage in all_cloud_brokerages if cloud_brokerage._id == "QuantConnectBrokerage"] From 7ce92196af498d2ac2b4dc12c00479d0f2f2ff30 Mon Sep 17 00:00:00 2001 From: Marinovsky Date: Fri, 8 Sep 2023 12:03:10 -0500 Subject: [PATCH 9/9] Add _configure_data_feed() method --- lean/commands/cloud/live/deploy.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index bab5fff5..640680e2 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -113,6 +113,20 @@ def _configure_brokerage(lean_config: Dict[str, Any], logger: Logger, user_provi user_provided_options, hide_input=not show_secrets) +def _configure_data_feed(brokerage: CloudBrokerage, logger: Logger) -> None: + """Configures the data feed to use based on the brokerage given. + + :param brokerage: the cloud brokerage + :param logger: the logger to use + """ + if len(cloud_brokerage_data_feeds[brokerage]) != 0: + data_feed_selected = logger.prompt_list("Select a data feed", [ + Option(id=data_feed, label=data_feed) for data_feed in cloud_brokerage_data_feeds[brokerage] + ], multiple=False) + data_feed_property_name = [name for name in brokerage.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)] + data_feed_property_name = data_feed_property_name[0] if len(data_feed_property_name) != 0 else "" + brokerage.update_value_for_given_config(data_feed_property_name, data_feed_selected) + def _configure_live_node(logger: Logger, api_client: APIClient, cloud_project: QCProject) -> QCNode: """Interactively configures the live node to use.