Skip to content

Commit

Permalink
Feature: new sorting property in SubscriptionDataSource and `SortEn…
Browse files Browse the repository at this point in the history
…umerator` (#8352)

* feat: sort property in SubscriptionDataSource

* refactor: read lines (with sort opportunity) in TextSubscriptionDataSourceReader

* refactor: xml description of Sort property in SubscriptionDataSource

* revet: TextSubscriptionDataSourceReader changes

* feat: develop UnsortedTextSubscriptionDataSourceReader

* remove: UnsortedTextSubscriptionDataSourceReader

* feat: develop SortEnumerator
feat: use SortEnumerator in SubscriptionDataReader

* revert: TextSubscriptionDataSourceReader changes

* feat: use sort Enumerator in LiveCustomDataSubscriptionEnumeratorFactory

* test:feat: Unsorted Live Data Feed with using Object Store

* test:feat: history CustomData request with unsorted data

* refactor: develop TryWrapper on SortEnumerator

* remove: extra spacing

* refactor: Sort in SubscriptionDataSource

* feat: add RegressionAlgorithms when source return data in Descending format

* refactor: xml description of prop Sort in  SubscriptionDataSource

* refactor: SortEnumerator

* test:refactor access modifier in ExampleCustomDataWithSort

* test:refactor: increase performance LiveDataFeedSourcesDataFromObjectStore

* feat: add xml comment of regression algorithm

* test:revert: access modifier to public in internal test class

* test:refactor: use while instead of timer

* Revert "test:refactor: use while instead of timer"

This reverts commit 4fe43da.
  • Loading branch information
Romazes authored Oct 7, 2024
1 parent 0e61415 commit 6b22254
Show file tree
Hide file tree
Showing 9 changed files with 724 additions and 35 deletions.
211 changes: 211 additions & 0 deletions Algorithm.CSharp/DescendingCustomDataObjectStoreRegressionAlgorithm.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

using System;
using System.Linq;
using QuantConnect.Data;
using System.Globalization;
using QuantConnect.Interfaces;
using System.Collections.Generic;

namespace QuantConnect.Algorithm.CSharp
{
/// <summary>
/// The regression algorithm showcases the utilization of a custom data source with the Sort flag set to true.
/// This means that the source initially provides data in descending order, which is then organized into ascending order and returned in the <see cref="OnData(Slice)"/> function.
/// </summary>
public class DescendingCustomDataObjectStoreRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
{
protected virtual string CustomDataKey => "CustomData/SortCustomData";

protected readonly static string[] descendingCustomData = new string[]
{
"2024-10-03 19:00:00,173.5,176.0,172.0,175.2,120195681,4882.29",
"2024-10-02 18:00:00,174.0,177.0,173.0,175.8,116275729,4641.97",
"2024-10-01 17:00:00,175.0,178.0,172.5,174.5,127707078,6591.27",
"2024-09-30 11:00:00,174.8,176.5,172.8,175.0,127707078,6591.27",
"2024-09-27 10:00:00,172.5,175.0,171.5,173.5,120195681,4882.29",
"2024-09-26 09:00:00,171.0,172.5,170.0,171.8,117516350,4820.53",
"2024-09-25 08:00:00,169.5,172.0,169.0,171.0,110427867,4661.55",
"2024-09-24 07:00:00,170.0,171.0,168.0,169.5,127624733,4823.52",
"2024-09-23 06:00:00,172.0,173.5,169.5,171.5,123586417,4303.93",
"2024-09-20 05:00:00,168.0,171.0,167.5,170.5,151929179,5429.87",
"2024-09-19 04:00:00,170.5,171.5,166.0,167.0,160523863,5219.24",
"2024-09-18 03:00:00,173.0,174.0,169.0,172.0,145721790,5163.09",
"2024-09-17 02:00:00,171.0,173.5,170.0,172.5,144794030,5405.72",
"2024-09-16 01:00:00,168.0,171.0,167.0,170.0,214402430,8753.33",
"2024-09-13 16:00:00,173.5,176.0,172.0,175.2,120195681,4882.29",
"2024-09-12 15:00:00,174.5,177.5,173.5,176.5,171728134,7774.83",
"2024-09-11 14:00:00,175.0,178.0,174.0,175.5,191516153,8349.59",
"2024-09-10 13:00:00,174.5,176.0,173.0,174.0,151162819,5915.8",
"2024-09-09 12:00:00,176.0,178.0,175.0,177.0,116275729,4641.97"
};

private Symbol _customSymbol;

private List<SortCustomData> _receivedData = new();

public override void Initialize()
{
SetStartDate(2024, 09, 09);
SetEndDate(2024, 10, 03);
SetCash(100000);

SetBenchmark(x => 0);

SortCustomData.CustomDataKey = CustomDataKey;

_customSymbol = AddData<SortCustomData>("SortCustomData", Resolution.Daily).Symbol;

// Saving data here for demonstration and regression testing purposes.
// In real scenarios, data has to be saved to the object store before the algorithm starts.
ObjectStore.Save(CustomDataKey, string.Join("\n", descendingCustomData));
}

public override void OnData(Slice slice)
{
if (slice.ContainsKey(_customSymbol))
{
var sortCustomData = slice.Get<SortCustomData>(_customSymbol);
if (sortCustomData.Open == 0 || sortCustomData.High == 0 || sortCustomData.Low == 0 || sortCustomData.Close == 0 || sortCustomData.Price == 0)
{
throw new RegressionTestException("One or more custom data fields (Open, High, Low, Close, Price) are zero.");
}

_receivedData.Add(sortCustomData);
}
}

public override void OnEndOfAlgorithm()
{
if (_receivedData.Count == 0)
{
throw new RegressionTestException("Custom data was not fetched");
}

var history = History<SortCustomData>(_customSymbol, StartDate, EndDate, Resolution.Hour).ToList();

if (history.Count != _receivedData.Count)
{
throw new RegressionTestException("History request returned different data than expected");
}

// Iterate through the history collection, checking if the EndTime is in ascending order.
for (int i = 0; i < history.Count - 1; i++)
{
if (history[i].EndTime > history[i + 1].EndTime)
{
throw new RegressionTestException($"Order failure: {history[i].EndTime} > {history[i + 1].EndTime} at index {i}.");
}
}
}

/// <summary>
/// Final status of the algorithm
/// </summary>
public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;

/// <summary>
/// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
/// </summary>
public bool CanRunLocally => true;

/// <summary>
/// This is used by the regression test system to indicate which languages this algorithm is written in.
/// </summary>
public List<Language> Languages => new() { Language.CSharp, Language.Python };

/// <summary>
/// Data Points count of all TimeSlices of algorithm
/// </summary>
public long DataPoints => 20;

/// <summary>
/// Data Points count of the algorithm history
/// </summary>
public int AlgorithmHistoryDataPoints => 19;

/// <summary>
/// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
/// </summary>
public Dictionary<string, string> ExpectedStatistics => new ()
{
{"Total Orders", "0"},
{"Average Win", "0%"},
{"Average Loss", "0%"},
{"Compounding Annual Return", "0%"},
{"Drawdown", "0%"},
{"Expectancy", "0"},
{"Start Equity", "100000"},
{"End Equity", "100000"},
{"Net Profit", "0%"},
{"Sharpe Ratio", "0"},
{"Sortino Ratio", "0"},
{"Probabilistic Sharpe Ratio", "0%"},
{"Loss Rate", "0%"},
{"Win Rate", "0%"},
{"Profit-Loss Ratio", "0"},
{"Alpha", "0"},
{"Beta", "0"},
{"Annual Standard Deviation", "0"},
{"Annual Variance", "0"},
{"Information Ratio", "0"},
{"Tracking Error", "0"},
{"Treynor Ratio", "0"},
{"Total Fees", "$0.00"},
{"Estimated Strategy Capacity", "$0"},
{"Lowest Capacity Asset", ""},
{"Portfolio Turnover", "0%"},
{"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
};
}

public class SortCustomData : BaseData
{
public static string CustomDataKey { get; set; }

public decimal Open { get; set; }
public decimal High { get; set; }
public decimal Low { get; set; }
public decimal Close { get; set; }

public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
return new SubscriptionDataSource(CustomDataKey, SubscriptionTransportMedium.ObjectStore, FileFormat.Csv)
{
// Indicate that the data from the subscription will be returned in descending order.
Sort = true
};
}

public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
{
var csv = line.Split(",");
var data = new SortCustomData()
{
Symbol = config.Symbol,
Time = DateTime.ParseExact(csv[0], DateFormat.DB, CultureInfo.InvariantCulture),
Value = csv[4].ToDecimal(),
Open = csv[1].ToDecimal(),
High = csv[2].ToDecimal(),
Low = csv[3].ToDecimal(),
Close = csv[4].ToDecimal()
};

return data;
}
}
}
109 changes: 109 additions & 0 deletions Algorithm.Python/DescendingCustomDataObjectStoreRegressionAlgorithm.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and

from AlgorithmImports import *


### <summary>
### The regression algorithm showcases the utilization of a custom data source with the Sort flag set to true.
### This means that the source initially provides data in descending order, which is then organized into ascending order and returned in the 'on_data' function.
### </summary>
class DescendingCustomDataObjectStoreRegressionAlgorithm(QCAlgorithm):
descending_custom_data = [
"2024-10-03 19:00:00,173.5,176.0,172.0,175.2,120195681,4882.29",
"2024-10-02 18:00:00,174.0,177.0,173.0,175.8,116275729,4641.97",
"2024-10-01 17:00:00,175.0,178.0,172.5,174.5,127707078,6591.27",
"2024-09-30 11:00:00,174.8,176.5,172.8,175.0,127707078,6591.27",
"2024-09-27 10:00:00,172.5,175.0,171.5,173.5,120195681,4882.29",
"2024-09-26 09:00:00,171.0,172.5,170.0,171.8,117516350,4820.53",
"2024-09-25 08:00:00,169.5,172.0,169.0,171.0,110427867,4661.55",
"2024-09-24 07:00:00,170.0,171.0,168.0,169.5,127624733,4823.52",
"2024-09-23 06:00:00,172.0,173.5,169.5,171.5,123586417,4303.93",
"2024-09-20 05:00:00,168.0,171.0,167.5,170.5,151929179,5429.87",
"2024-09-19 04:00:00,170.5,171.5,166.0,167.0,160523863,5219.24",
"2024-09-18 03:00:00,173.0,174.0,169.0,172.0,145721790,5163.09",
"2024-09-17 02:00:00,171.0,173.5,170.0,172.5,144794030,5405.72",
"2024-09-16 01:00:00,168.0,171.0,167.0,170.0,214402430,8753.33",
"2024-09-13 16:00:00,173.5,176.0,172.0,175.2,120195681,4882.29",
"2024-09-12 15:00:00,174.5,177.5,173.5,176.5,171728134,7774.83",
"2024-09-11 14:00:00,175.0,178.0,174.0,175.5,191516153,8349.59",
"2024-09-10 13:00:00,174.5,176.0,173.0,174.0,151162819,5915.8",
"2024-09-09 12:00:00,176.0,178.0,175.0,177.0,116275729,4641.97"
]

def initialize(self):
self.set_start_date(2024, 9, 9)
self.set_end_date(2024, 10, 3)
self.set_cash(100000)

self.set_benchmark(lambda x: 0)

SortCustomData.custom_data_key = self.get_custom_data_key()

self.custom_symbol = self.add_data(SortCustomData, "SortCustomData", Resolution.DAILY).symbol

# Saving data here for demonstration and regression testing purposes.
# In real scenarios, data has to be saved to the object store before the algorithm starts.
self.object_store.save(self.get_custom_data_key(), "\n".join(self.descending_custom_data))

self.received_data = []

def on_data(self, slice: Slice):
if slice.contains_key(self.custom_symbol):
custom_data = slice.get(SortCustomData, self.custom_symbol)
if custom_data.open == 0 or custom_data.high == 0 or custom_data.low == 0 or custom_data.close == 0 or custom_data.price == 0:
raise Exception("One or more custom data fields (open, high, low, close, price) are zero.")

self.received_data.append(custom_data)

def on_end_of_algorithm(self):
if not self.received_data:
raise Exception("Custom data was not fetched")

# Make sure history requests work as expected
history = self.history(SortCustomData, self.custom_symbol, self.start_date, self.end_date, Resolution.DAILY)

if history.shape[0] != len(self.received_data):
raise Exception("History request returned more or less data than expected")

# Iterate through the history collection, checking if the time is in ascending order.
for i in range(len(history) - 1):
# [1] - time
if history.index[i][1] > history.index[i + 1][1]:
raise RegressionTestException(
f"Order failure: {history.index[i][1]} > {history.index[i + 1][1]} at index {i}.")

def get_custom_data_key(self):
return "CustomData/SortCustomData"


class SortCustomData(PythonData):
custom_data_key = ""

def get_source(self, config, date, is_live):
subscription = SubscriptionDataSource(self.custom_data_key, SubscriptionTransportMedium.OBJECT_STORE,
FileFormat.CSV)
# Indicate that the data from the subscription will be returned in descending order.
subscription.Sort = True
return subscription

def reader(self, config, line, date, is_live):
data = line.split(',')
obj_data = SortCustomData()
obj_data.symbol = config.symbol
obj_data.time = datetime.strptime(data[0], '%Y-%m-%d %H:%M:%S')
obj_data.value = float(data[4])
obj_data["Open"] = float(data[1])
obj_data["High"] = float(data[2])
obj_data["Low"] = float(data[3])
obj_data["Close"] = float(data[4])
return obj_data
6 changes: 6 additions & 0 deletions Common/Data/SubscriptionDataSource.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,12 @@ public class SubscriptionDataSource : IEquatable<SubscriptionDataSource>
{
private readonly static IReadOnlyList<KeyValuePair<string, string>> _empty = new List<KeyValuePair<string, string>>();

/// <summary>
/// Specifies whether the data source should be sorted.
/// <c>If False</c>, data will be returned in the original order, <c>else</c> it will be ordered by time.
/// </summary>
public bool Sort { get; set; }

/// <summary>
/// Identifies where to get the subscription's data from
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Python.Runtime;
using QuantConnect.Data;
using QuantConnect.Data.UniverseSelection;
using QuantConnect.Interfaces;
Expand Down Expand Up @@ -143,7 +144,8 @@ private IEnumerator<BaseData> EnumerateDataSourceReader(SubscriptionDataConfig c
{
var newLocalFrontier = localFrontier.Value;
var dataSourceReader = GetSubscriptionDataSourceReader(source, dataCacheProvider, config, localDate, baseDataInstance, dataProvider);
foreach (var datum in dataSourceReader.Read(source))
using var subscriptionEnumerator = SortEnumerator<DateTime>.TryWrapSortEnumerator(source.Sort, dataSourceReader.Read(source));
foreach (var datum in subscriptionEnumerator)
{
// always skip past all times emitted on the previous invocation of this enumerator
// this allows data at the same time from the same refresh of the source while excluding
Expand Down
Loading

0 comments on commit 6b22254

Please sign in to comment.