-
-
Notifications
You must be signed in to change notification settings - Fork 3.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature: new sorting property in
SubscriptionDataSource
and `SortEn…
…umerator` (#8352) * feat: sort property in SubscriptionDataSource * refactor: read lines (with sort opportunity) in TextSubscriptionDataSourceReader * refactor: xml description of Sort property in SubscriptionDataSource * revet: TextSubscriptionDataSourceReader changes * feat: develop UnsortedTextSubscriptionDataSourceReader * remove: UnsortedTextSubscriptionDataSourceReader * feat: develop SortEnumerator feat: use SortEnumerator in SubscriptionDataReader * revert: TextSubscriptionDataSourceReader changes * feat: use sort Enumerator in LiveCustomDataSubscriptionEnumeratorFactory * test:feat: Unsorted Live Data Feed with using Object Store * test:feat: history CustomData request with unsorted data * refactor: develop TryWrapper on SortEnumerator * remove: extra spacing * refactor: Sort in SubscriptionDataSource * feat: add RegressionAlgorithms when source return data in Descending format * refactor: xml description of prop Sort in SubscriptionDataSource * refactor: SortEnumerator * test:refactor access modifier in ExampleCustomDataWithSort * test:refactor: increase performance LiveDataFeedSourcesDataFromObjectStore * feat: add xml comment of regression algorithm * test:revert: access modifier to public in internal test class * test:refactor: use while instead of timer * Revert "test:refactor: use while instead of timer" This reverts commit 4fe43da.
- Loading branch information
Showing
9 changed files
with
724 additions
and
35 deletions.
There are no files selected for viewing
211 changes: 211 additions & 0 deletions
211
Algorithm.CSharp/DescendingCustomDataObjectStoreRegressionAlgorithm.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,211 @@ | ||
/* | ||
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. | ||
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||
* you may not use this file except in compliance with the License. | ||
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
* | ||
*/ | ||
|
||
using System; | ||
using System.Linq; | ||
using QuantConnect.Data; | ||
using System.Globalization; | ||
using QuantConnect.Interfaces; | ||
using System.Collections.Generic; | ||
|
||
namespace QuantConnect.Algorithm.CSharp | ||
{ | ||
/// <summary> | ||
/// The regression algorithm showcases the utilization of a custom data source with the Sort flag set to true. | ||
/// This means that the source initially provides data in descending order, which is then organized into ascending order and returned in the <see cref="OnData(Slice)"/> function. | ||
/// </summary> | ||
public class DescendingCustomDataObjectStoreRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition | ||
{ | ||
protected virtual string CustomDataKey => "CustomData/SortCustomData"; | ||
|
||
protected readonly static string[] descendingCustomData = new string[] | ||
{ | ||
"2024-10-03 19:00:00,173.5,176.0,172.0,175.2,120195681,4882.29", | ||
"2024-10-02 18:00:00,174.0,177.0,173.0,175.8,116275729,4641.97", | ||
"2024-10-01 17:00:00,175.0,178.0,172.5,174.5,127707078,6591.27", | ||
"2024-09-30 11:00:00,174.8,176.5,172.8,175.0,127707078,6591.27", | ||
"2024-09-27 10:00:00,172.5,175.0,171.5,173.5,120195681,4882.29", | ||
"2024-09-26 09:00:00,171.0,172.5,170.0,171.8,117516350,4820.53", | ||
"2024-09-25 08:00:00,169.5,172.0,169.0,171.0,110427867,4661.55", | ||
"2024-09-24 07:00:00,170.0,171.0,168.0,169.5,127624733,4823.52", | ||
"2024-09-23 06:00:00,172.0,173.5,169.5,171.5,123586417,4303.93", | ||
"2024-09-20 05:00:00,168.0,171.0,167.5,170.5,151929179,5429.87", | ||
"2024-09-19 04:00:00,170.5,171.5,166.0,167.0,160523863,5219.24", | ||
"2024-09-18 03:00:00,173.0,174.0,169.0,172.0,145721790,5163.09", | ||
"2024-09-17 02:00:00,171.0,173.5,170.0,172.5,144794030,5405.72", | ||
"2024-09-16 01:00:00,168.0,171.0,167.0,170.0,214402430,8753.33", | ||
"2024-09-13 16:00:00,173.5,176.0,172.0,175.2,120195681,4882.29", | ||
"2024-09-12 15:00:00,174.5,177.5,173.5,176.5,171728134,7774.83", | ||
"2024-09-11 14:00:00,175.0,178.0,174.0,175.5,191516153,8349.59", | ||
"2024-09-10 13:00:00,174.5,176.0,173.0,174.0,151162819,5915.8", | ||
"2024-09-09 12:00:00,176.0,178.0,175.0,177.0,116275729,4641.97" | ||
}; | ||
|
||
private Symbol _customSymbol; | ||
|
||
private List<SortCustomData> _receivedData = new(); | ||
|
||
public override void Initialize() | ||
{ | ||
SetStartDate(2024, 09, 09); | ||
SetEndDate(2024, 10, 03); | ||
SetCash(100000); | ||
|
||
SetBenchmark(x => 0); | ||
|
||
SortCustomData.CustomDataKey = CustomDataKey; | ||
|
||
_customSymbol = AddData<SortCustomData>("SortCustomData", Resolution.Daily).Symbol; | ||
|
||
// Saving data here for demonstration and regression testing purposes. | ||
// In real scenarios, data has to be saved to the object store before the algorithm starts. | ||
ObjectStore.Save(CustomDataKey, string.Join("\n", descendingCustomData)); | ||
} | ||
|
||
public override void OnData(Slice slice) | ||
{ | ||
if (slice.ContainsKey(_customSymbol)) | ||
{ | ||
var sortCustomData = slice.Get<SortCustomData>(_customSymbol); | ||
if (sortCustomData.Open == 0 || sortCustomData.High == 0 || sortCustomData.Low == 0 || sortCustomData.Close == 0 || sortCustomData.Price == 0) | ||
{ | ||
throw new RegressionTestException("One or more custom data fields (Open, High, Low, Close, Price) are zero."); | ||
} | ||
|
||
_receivedData.Add(sortCustomData); | ||
} | ||
} | ||
|
||
public override void OnEndOfAlgorithm() | ||
{ | ||
if (_receivedData.Count == 0) | ||
{ | ||
throw new RegressionTestException("Custom data was not fetched"); | ||
} | ||
|
||
var history = History<SortCustomData>(_customSymbol, StartDate, EndDate, Resolution.Hour).ToList(); | ||
|
||
if (history.Count != _receivedData.Count) | ||
{ | ||
throw new RegressionTestException("History request returned different data than expected"); | ||
} | ||
|
||
// Iterate through the history collection, checking if the EndTime is in ascending order. | ||
for (int i = 0; i < history.Count - 1; i++) | ||
{ | ||
if (history[i].EndTime > history[i + 1].EndTime) | ||
{ | ||
throw new RegressionTestException($"Order failure: {history[i].EndTime} > {history[i + 1].EndTime} at index {i}."); | ||
} | ||
} | ||
} | ||
|
||
/// <summary> | ||
/// Final status of the algorithm | ||
/// </summary> | ||
public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; | ||
|
||
/// <summary> | ||
/// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. | ||
/// </summary> | ||
public bool CanRunLocally => true; | ||
|
||
/// <summary> | ||
/// This is used by the regression test system to indicate which languages this algorithm is written in. | ||
/// </summary> | ||
public List<Language> Languages => new() { Language.CSharp, Language.Python }; | ||
|
||
/// <summary> | ||
/// Data Points count of all TimeSlices of algorithm | ||
/// </summary> | ||
public long DataPoints => 20; | ||
|
||
/// <summary> | ||
/// Data Points count of the algorithm history | ||
/// </summary> | ||
public int AlgorithmHistoryDataPoints => 19; | ||
|
||
/// <summary> | ||
/// This is used by the regression test system to indicate what the expected statistics are from running the algorithm | ||
/// </summary> | ||
public Dictionary<string, string> ExpectedStatistics => new () | ||
{ | ||
{"Total Orders", "0"}, | ||
{"Average Win", "0%"}, | ||
{"Average Loss", "0%"}, | ||
{"Compounding Annual Return", "0%"}, | ||
{"Drawdown", "0%"}, | ||
{"Expectancy", "0"}, | ||
{"Start Equity", "100000"}, | ||
{"End Equity", "100000"}, | ||
{"Net Profit", "0%"}, | ||
{"Sharpe Ratio", "0"}, | ||
{"Sortino Ratio", "0"}, | ||
{"Probabilistic Sharpe Ratio", "0%"}, | ||
{"Loss Rate", "0%"}, | ||
{"Win Rate", "0%"}, | ||
{"Profit-Loss Ratio", "0"}, | ||
{"Alpha", "0"}, | ||
{"Beta", "0"}, | ||
{"Annual Standard Deviation", "0"}, | ||
{"Annual Variance", "0"}, | ||
{"Information Ratio", "0"}, | ||
{"Tracking Error", "0"}, | ||
{"Treynor Ratio", "0"}, | ||
{"Total Fees", "$0.00"}, | ||
{"Estimated Strategy Capacity", "$0"}, | ||
{"Lowest Capacity Asset", ""}, | ||
{"Portfolio Turnover", "0%"}, | ||
{"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"} | ||
}; | ||
} | ||
|
||
public class SortCustomData : BaseData | ||
{ | ||
public static string CustomDataKey { get; set; } | ||
|
||
public decimal Open { get; set; } | ||
public decimal High { get; set; } | ||
public decimal Low { get; set; } | ||
public decimal Close { get; set; } | ||
|
||
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) | ||
{ | ||
return new SubscriptionDataSource(CustomDataKey, SubscriptionTransportMedium.ObjectStore, FileFormat.Csv) | ||
{ | ||
// Indicate that the data from the subscription will be returned in descending order. | ||
Sort = true | ||
}; | ||
} | ||
|
||
public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) | ||
{ | ||
var csv = line.Split(","); | ||
var data = new SortCustomData() | ||
{ | ||
Symbol = config.Symbol, | ||
Time = DateTime.ParseExact(csv[0], DateFormat.DB, CultureInfo.InvariantCulture), | ||
Value = csv[4].ToDecimal(), | ||
Open = csv[1].ToDecimal(), | ||
High = csv[2].ToDecimal(), | ||
Low = csv[3].ToDecimal(), | ||
Close = csv[4].ToDecimal() | ||
}; | ||
|
||
return data; | ||
} | ||
} | ||
} |
109 changes: 109 additions & 0 deletions
109
Algorithm.Python/DescendingCustomDataObjectStoreRegressionAlgorithm.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,109 @@ | ||
# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. | ||
# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
|
||
from AlgorithmImports import * | ||
|
||
|
||
### <summary> | ||
### The regression algorithm showcases the utilization of a custom data source with the Sort flag set to true. | ||
### This means that the source initially provides data in descending order, which is then organized into ascending order and returned in the 'on_data' function. | ||
### </summary> | ||
class DescendingCustomDataObjectStoreRegressionAlgorithm(QCAlgorithm): | ||
descending_custom_data = [ | ||
"2024-10-03 19:00:00,173.5,176.0,172.0,175.2,120195681,4882.29", | ||
"2024-10-02 18:00:00,174.0,177.0,173.0,175.8,116275729,4641.97", | ||
"2024-10-01 17:00:00,175.0,178.0,172.5,174.5,127707078,6591.27", | ||
"2024-09-30 11:00:00,174.8,176.5,172.8,175.0,127707078,6591.27", | ||
"2024-09-27 10:00:00,172.5,175.0,171.5,173.5,120195681,4882.29", | ||
"2024-09-26 09:00:00,171.0,172.5,170.0,171.8,117516350,4820.53", | ||
"2024-09-25 08:00:00,169.5,172.0,169.0,171.0,110427867,4661.55", | ||
"2024-09-24 07:00:00,170.0,171.0,168.0,169.5,127624733,4823.52", | ||
"2024-09-23 06:00:00,172.0,173.5,169.5,171.5,123586417,4303.93", | ||
"2024-09-20 05:00:00,168.0,171.0,167.5,170.5,151929179,5429.87", | ||
"2024-09-19 04:00:00,170.5,171.5,166.0,167.0,160523863,5219.24", | ||
"2024-09-18 03:00:00,173.0,174.0,169.0,172.0,145721790,5163.09", | ||
"2024-09-17 02:00:00,171.0,173.5,170.0,172.5,144794030,5405.72", | ||
"2024-09-16 01:00:00,168.0,171.0,167.0,170.0,214402430,8753.33", | ||
"2024-09-13 16:00:00,173.5,176.0,172.0,175.2,120195681,4882.29", | ||
"2024-09-12 15:00:00,174.5,177.5,173.5,176.5,171728134,7774.83", | ||
"2024-09-11 14:00:00,175.0,178.0,174.0,175.5,191516153,8349.59", | ||
"2024-09-10 13:00:00,174.5,176.0,173.0,174.0,151162819,5915.8", | ||
"2024-09-09 12:00:00,176.0,178.0,175.0,177.0,116275729,4641.97" | ||
] | ||
|
||
def initialize(self): | ||
self.set_start_date(2024, 9, 9) | ||
self.set_end_date(2024, 10, 3) | ||
self.set_cash(100000) | ||
|
||
self.set_benchmark(lambda x: 0) | ||
|
||
SortCustomData.custom_data_key = self.get_custom_data_key() | ||
|
||
self.custom_symbol = self.add_data(SortCustomData, "SortCustomData", Resolution.DAILY).symbol | ||
|
||
# Saving data here for demonstration and regression testing purposes. | ||
# In real scenarios, data has to be saved to the object store before the algorithm starts. | ||
self.object_store.save(self.get_custom_data_key(), "\n".join(self.descending_custom_data)) | ||
|
||
self.received_data = [] | ||
|
||
def on_data(self, slice: Slice): | ||
if slice.contains_key(self.custom_symbol): | ||
custom_data = slice.get(SortCustomData, self.custom_symbol) | ||
if custom_data.open == 0 or custom_data.high == 0 or custom_data.low == 0 or custom_data.close == 0 or custom_data.price == 0: | ||
raise Exception("One or more custom data fields (open, high, low, close, price) are zero.") | ||
|
||
self.received_data.append(custom_data) | ||
|
||
def on_end_of_algorithm(self): | ||
if not self.received_data: | ||
raise Exception("Custom data was not fetched") | ||
|
||
# Make sure history requests work as expected | ||
history = self.history(SortCustomData, self.custom_symbol, self.start_date, self.end_date, Resolution.DAILY) | ||
|
||
if history.shape[0] != len(self.received_data): | ||
raise Exception("History request returned more or less data than expected") | ||
|
||
# Iterate through the history collection, checking if the time is in ascending order. | ||
for i in range(len(history) - 1): | ||
# [1] - time | ||
if history.index[i][1] > history.index[i + 1][1]: | ||
raise RegressionTestException( | ||
f"Order failure: {history.index[i][1]} > {history.index[i + 1][1]} at index {i}.") | ||
|
||
def get_custom_data_key(self): | ||
return "CustomData/SortCustomData" | ||
|
||
|
||
class SortCustomData(PythonData): | ||
custom_data_key = "" | ||
|
||
def get_source(self, config, date, is_live): | ||
subscription = SubscriptionDataSource(self.custom_data_key, SubscriptionTransportMedium.OBJECT_STORE, | ||
FileFormat.CSV) | ||
# Indicate that the data from the subscription will be returned in descending order. | ||
subscription.Sort = True | ||
return subscription | ||
|
||
def reader(self, config, line, date, is_live): | ||
data = line.split(',') | ||
obj_data = SortCustomData() | ||
obj_data.symbol = config.symbol | ||
obj_data.time = datetime.strptime(data[0], '%Y-%m-%d %H:%M:%S') | ||
obj_data.value = float(data[4]) | ||
obj_data["Open"] = float(data[1]) | ||
obj_data["High"] = float(data[2]) | ||
obj_data["Low"] = float(data[3]) | ||
obj_data["Close"] = float(data[4]) | ||
return obj_data |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.