diff --git a/Algorithm/QCAlgorithm.Python.cs b/Algorithm/QCAlgorithm.Python.cs index 40625361a10e..428bb731e3ed 100644 --- a/Algorithm/QCAlgorithm.Python.cs +++ b/Algorithm/QCAlgorithm.Python.cs @@ -944,11 +944,10 @@ public PyObject History(PyObject tickers, int periods, Resolution? resolution = var symbols = tickers.ConvertToSymbolEnumerable().ToArray(); var dataType = Extensions.GetCustomDataTypeFromSymbols(symbols); - var df = GetDataFrame( + return GetDataFrame( History(symbols, periods, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset), flatten, dataType); - return FormatCanonicalOptionHistoryDataFrameIndex(symbols, df, flatten); } /// @@ -1019,11 +1018,10 @@ public PyObject History(PyObject tickers, DateTime start, DateTime end, Resoluti var symbols = tickers.ConvertToSymbolEnumerable().ToArray(); var dataType = Extensions.GetCustomDataTypeFromSymbols(symbols); - var df = GetDataFrame( + return GetDataFrame( History(symbols, start, end, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset), flatten, dataType); - return FormatCanonicalOptionHistoryDataFrameIndex(symbols, df, flatten); } /// @@ -1865,45 +1863,5 @@ private PyObject TryCleanupCollectionDataFrame(Type dataType, PyObject history) } return history; } - - private static bool IsCanonicalOption(Symbol symbol) - { - return symbol.SecurityType.IsOption() && symbol.IsCanonical(); - } - - /// - /// Renames the data frame index for canonical options history (basically option chains) data frames - /// - private PyObject FormatCanonicalOptionHistoryDataFrameIndex(Symbol[] symbols, PyObject df, bool flatten) - { - if (df == null) - { - return null; - } - - if (!flatten || symbols.Length == 0 || !IsCanonicalOption(symbols[0])) - { - return df; - } - - using var _ = Py.GIL(); - - if (df.GetAttr("empty").GetAndDispose()) - { - return df; - } - - using var renameArgs = new PyDict(); - using var canonicalName = "canonical".ToPython(); - renameArgs.SetItem("collection_symbol", canonicalName); - - using var kwargs = Py.kw("inplace", true); - - using var index = df.GetAttr("index"); - using var setNames = index.GetAttr("set_names"); - setNames.Invoke(new[] { renameArgs }, kwargs); - - return df; - } } } diff --git a/Common/Python/PandasConverter.DataFrameGenerator.cs b/Common/Python/PandasConverter.DataFrameGenerator.cs index 720f4e014639..a2d609a186f7 100644 --- a/Common/Python/PandasConverter.DataFrameGenerator.cs +++ b/Common/Python/PandasConverter.DataFrameGenerator.cs @@ -33,6 +33,7 @@ public partial class PandasConverter private class DataFrameGenerator { private static readonly string[] MultiBaseDataCollectionDataFrameNames = new[] { "collection_symbol", "time" }; + private static readonly string[] MultiCanonicalOptionDataFrameNames = new[] { "canonical", "time" }; private static readonly string[] SingleBaseDataCollectionDataFrameNames = new[] { "time" }; private readonly Type _dataType; @@ -45,7 +46,7 @@ private class DataFrameGenerator /// PandasData instances for each symbol. Does not hold BaseDataCollection instances. /// private Dictionary _pandasData; - private List<(Symbol, DateTime, IEnumerable)> _collections; + private List<(Symbol Symbol, DateTime Time, IEnumerable Data)> _collections; private int _maxLevels; private bool _shouldUseSymbolOnlyIndex; @@ -81,9 +82,9 @@ protected void AddData(IEnumerable slices) { foreach (var data in slice.AllData) { - if (_flatten && data is BaseDataCollection collection) + if (_flatten && IsBaseDataCollection(data.GetType())) { - AddCollection(collection.Symbol, collection.EndTime, collection); + AddCollection(data.Symbol, data.EndTime, (data as IEnumerable).Cast()); continue; } @@ -151,8 +152,7 @@ protected void AddData(IEnumerable data) where T : ISymbolProvider { var type = typeof(T); - var isBaseDataCollection = type.IsAssignableTo(typeof(BaseData)) && - type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition().IsAssignableTo(typeof(IEnumerable<>))); + var isBaseDataCollection = IsBaseDataCollection(type); if (_flatten && isBaseDataCollection) { @@ -215,13 +215,16 @@ public PyObject GenerateDataFrame(int? levels = null, bool sort = true, bool fil { return ConcatDataFrames(dataFrames, sort, dropna: true); } - else if (_collections.DistinctBy(x => x.Item1).Count() > 1) + else if (_collections.DistinctBy(x => x.Symbol).Count() > 1) { var keys = collectionsDataFrames .Select(x => new object[] { x.Item1, x.Item2 }) .Concat(pandasDataDataFrames.Select(x => new object[] { x, DateTime.MinValue })); + var names = _collections.Any(x => x.Symbol.SecurityType.IsOption() && x.Symbol.IsCanonical()) + ? MultiCanonicalOptionDataFrameNames + : MultiBaseDataCollectionDataFrameNames; - return ConcatDataFrames(dataFrames, keys, MultiBaseDataCollectionDataFrameNames, sort, dropna: true); + return ConcatDataFrames(dataFrames, keys, names, sort, dropna: true); } else { @@ -273,7 +276,7 @@ private IEnumerable GetPandasDataDataFrames(int? levels, bool filterMi yield break; } - foreach (var (symbol, time, data) in _collections.GroupBy(x => x.Item1).SelectMany(x => x)) + foreach (var (symbol, time, data) in _collections.GroupBy(x => x.Symbol).SelectMany(x => x)) { var generator = new DataFrameGenerator(_dataType, timeAsColumn: !symbolOnlyIndex, flatten: _flatten); generator.AddData(data); @@ -301,6 +304,21 @@ private void AddCollection(Symbol symbol, DateTime time, IEnumerable + /// Determines whether the type is considered a base data collection for flattening. + /// Any object that is a and implements + /// is considered a base data collection. + /// This allows detecting collections of cases like (which is a direct subclass of + /// ) and , which is a collection of + /// + private static bool IsBaseDataCollection(Type type) + { + return type.IsAssignableTo(typeof(BaseData)) && + type.GetInterfaces().Any(x => x.IsGenericType && + x.GetGenericTypeDefinition().IsAssignableTo(typeof(IEnumerable<>)) && + x.GenericTypeArguments[0].IsAssignableTo(typeof(ISymbolProvider))); + } } private class DataFrameGenerator : DataFrameGenerator diff --git a/Common/Python/PandasData.DataTypeMember.cs b/Common/Python/PandasData.DataTypeMember.cs index e991100d93fe..1f1d445edfbc 100644 --- a/Common/Python/PandasData.DataTypeMember.cs +++ b/Common/Python/PandasData.DataTypeMember.cs @@ -18,24 +18,30 @@ using System; using System.Collections.Generic; using System.Reflection; -using System.Runtime.CompilerServices; using System.Text; namespace QuantConnect.Python { public partial class PandasData { + private static DataTypeMember CreateDataTypeMember(MemberInfo member, DataTypeMember[] children = null) + { + return member switch + { + PropertyInfo property => new PropertyMember(property, children), + FieldInfo field => new FieldMember(field, children), + _ => throw new ArgumentException($"Member type {member.MemberType} is not supported") + }; + } + /// /// Represents a member of a data type, either a property or a field and it's children members in case it's a complex type. /// It contains logic to get the member name and the children names, taking into account the parent prefixes. /// - private class DataTypeMember + private abstract class DataTypeMember { private static readonly StringBuilder _stringBuilder = new StringBuilder(); - private PropertyInfo _property; - private FieldInfo _field; - private DataTypeMember _parent; private string _name; @@ -43,11 +49,9 @@ private class DataTypeMember public DataTypeMember[] Children { get; } - public bool IsNonExpandable { get; init; } + public abstract bool IsProperty { get; } - public bool IsProperty => _property != null; - - public bool IsField => _field != null; + public abstract bool IsField { get; } /// /// The prefix to be used for the children members when a class being expanded has multiple properties/fields of the same type @@ -64,14 +68,11 @@ private class DataTypeMember public bool IsTickProperty { get; } - private DataTypeMember(MemberInfo member, DataTypeMember[] children = null) + public DataTypeMember(MemberInfo member, DataTypeMember[] children = null) { Member = member; Children = children; - _property = member as PropertyInfo; - _field = member as FieldInfo; - IsTickLastPrice = member == _tickLastPriceMember || member == _openInterestLastPriceMember; IsTickProperty = IsProperty && member.DeclaringType == typeof(Tick); @@ -84,34 +85,6 @@ private DataTypeMember(MemberInfo member, DataTypeMember[] children = null) } } - public static DataTypeMember CreateWithChildren(MemberInfo member, DataTypeMember[] children) - { - return new DataTypeMember(member, children); - } - - public static DataTypeMember Create(MemberInfo member) - { - return new DataTypeMember(member); - } - - public static DataTypeMember CreateNonExpandableMember(MemberInfo member) - { - return new DataTypeMember(member) - { - IsNonExpandable = true - }; - } - - public PropertyInfo AsProperty() - { - return _property; - } - - public FieldInfo AsField() - { - return _field; - } - public void SetPrefix() { Prefix = Member.Name.ToLowerInvariant(); @@ -146,37 +119,13 @@ public IEnumerable GetMemberNames() return GetMemberNames(null); } - public object GetValue(object instance) - { - if (IsProperty) - { - return _property.GetValue(instance); - } + public abstract object GetValue(object instance); - return _field.GetValue(instance); - } + public abstract Type GetMemberType(); public override string ToString() { - return $"{GetMemberType(Member).Name} {Member.Name}"; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Type GetMemberType() - { - return GetMemberType(Member); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Type GetMemberType(MemberInfo member) - { - return member switch - { - PropertyInfo property => property.PropertyType, - FieldInfo field => field.FieldType, - // Should not happen - _ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}") - }; + return $"{GetMemberType().Name} {Member.Name}"; } private string BuildMemberName(string baseName) @@ -229,5 +178,55 @@ private string GetBaseName() return baseName.ToLowerInvariant(); } } + + private class PropertyMember : DataTypeMember + { + private PropertyInfo _property; + + public override bool IsProperty => true; + + public override bool IsField => false; + + public PropertyMember(PropertyInfo property, DataTypeMember[] children = null) + : base(property, children) + { + _property = property; + } + + public override object GetValue(object instance) + { + return _property.GetValue(instance); + } + + public override Type GetMemberType() + { + return _property.PropertyType; + } + } + + private class FieldMember : DataTypeMember + { + private FieldInfo _field; + + public override bool IsProperty => false; + + public override bool IsField => true; + + public FieldMember(FieldInfo field, DataTypeMember[] children = null) + : base(field, children) + { + _field = field; + } + + public override object GetValue(object instance) + { + return _field.GetValue(instance); + } + + public override Type GetMemberType() + { + return _field.FieldType; + } + } } } diff --git a/Common/Python/PandasData.cs b/Common/Python/PandasData.cs index 122cf925b86f..f9886df1d7cc 100644 --- a/Common/Python/PandasData.cs +++ b/Common/Python/PandasData.cs @@ -168,24 +168,17 @@ private void Add(object data, bool overrideValues) } var typeMembers = GetInstanceDataTypeMembers(data).ToList(); - var isNonExpandable = typeMembers.Count == 1 && typeMembers[0].IsNonExpandable; var endTime = default(DateTime); if (_isBaseData) { endTime = ((IBaseData)data).EndTime; - if (_timeAsColumn && !isNonExpandable) + if (_timeAsColumn) { AddToSeries("time", endTime, endTime, overrideValues); } } - if (isNonExpandable) - { - AddToSeries("instance", endTime, data, overrideValues); - return; - } - AddMembersData(data, typeMembers, endTime, overrideValues); if (data is DynamicData dynamicData) @@ -471,13 +464,6 @@ public static PyObject ToPandasDataFrame(IEnumerable pandasDatas, bo private IEnumerable GetInstanceDataTypeMembers(object data) { var type = data.GetType(); - - if (type.IsDefined(PandasNonExpandableAttribute)) - { - _series.TryAdd("instance", new Serie(withTimeIndex: !_timeAsColumn)); - return new List { DataTypeMember.CreateNonExpandableMember(type) }; - } - if (!_members.TryGetValue(type, out var members)) { HashSet columnNames; @@ -555,8 +541,8 @@ private static IEnumerable GetDataTypeMembers(Type type, string[ return members .Select(member => { - DataTypeMember dataTypeMember; - var memberType = DataTypeMember.GetMemberType(member); + var dataTypeMember = CreateDataTypeMember(member); + var memberType = dataTypeMember.GetMemberType(); // Should we unpack its properties into columns? if (memberType.IsClass @@ -567,11 +553,7 @@ private static IEnumerable GetDataTypeMembers(Type type, string[ && !memberType.IsDefined(PandasNonExpandableAttribute) && !member.IsDefined(PandasNonExpandableAttribute)))) { - dataTypeMember = DataTypeMember.CreateWithChildren(member, GetDataTypeMembers(memberType, forcedInclusionMembers).ToArray()); - } - else - { - dataTypeMember = DataTypeMember.Create(member); + dataTypeMember = CreateDataTypeMember(member, GetDataTypeMembers(memberType, forcedInclusionMembers).ToArray()); } return (memberType, dataTypeMember); @@ -678,17 +660,12 @@ private PyObject CreateIndexSourceValue(DateTime index, PyObject[] list) /// to add to the value associated with the specific key. Can be null. private void AddToSeries(string key, DateTime time, object input, bool overrideValues) { - var serie = GetSerie(key); - serie.Add(time, input, overrideValues); - } - - private Serie GetSerie(string key) - { - if (!_series.TryGetValue(key, out var value)) + if (!_series.TryGetValue(key, out var serie)) { - throw new ArgumentException($"PandasData.GetSerie(): {Messages.PandasData.KeyNotFoundInSeries(key)}"); + throw new ArgumentException($"PandasData.AddToSeries(): {Messages.PandasData.KeyNotFoundInSeries(key)}"); } - return value; + + serie.Add(time, input, overrideValues); } private class Serie