Skip to content

Commit

Permalink
Housekeeping
Browse files Browse the repository at this point in the history
  • Loading branch information
@jhonabreul
jhonabreul committed Nov 4, 2024
1 parent 83335c3 commit 0263ecf
Show file tree
Hide file tree
Showing 4 changed files with 103 additions and 151 deletions.
46 changes: 2 additions & 44 deletions Algorithm/QCAlgorithm.Python.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -944,11 +944,10 @@ public PyObject History(PyObject tickers, int periods, Resolution? resolution =
var symbols = tickers.ConvertToSymbolEnumerable().ToArray();
var dataType = Extensions.GetCustomDataTypeFromSymbols(symbols);

var df = GetDataFrame(
return GetDataFrame(
History(symbols, periods, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset),
flatten,
dataType);
return FormatCanonicalOptionHistoryDataFrameIndex(symbols, df, flatten);
}

/// <summary>
Expand Down Expand Up @@ -1019,11 +1018,10 @@ public PyObject History(PyObject tickers, DateTime start, DateTime end, Resoluti
var symbols = tickers.ConvertToSymbolEnumerable().ToArray();
var dataType = Extensions.GetCustomDataTypeFromSymbols(symbols);

var df = GetDataFrame(
return GetDataFrame(
History(symbols, start, end, resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset),
flatten,
dataType);
return FormatCanonicalOptionHistoryDataFrameIndex(symbols, df, flatten);
}

/// <summary>
Expand Down Expand Up @@ -1865,45 +1863,5 @@ private PyObject TryCleanupCollectionDataFrame(Type dataType, PyObject history)
}
return history;
}

private static bool IsCanonicalOption(Symbol symbol)
{
return symbol.SecurityType.IsOption() && symbol.IsCanonical();
}

/// <summary>
/// Renames the data frame index for canonical options history (basically option chains) data frames
/// </summary>
private PyObject FormatCanonicalOptionHistoryDataFrameIndex(Symbol[] symbols, PyObject df, bool flatten)
{
if (df == null)
{
return null;
}

if (!flatten || symbols.Length == 0 || !IsCanonicalOption(symbols[0]))
{
return df;
}

using var _ = Py.GIL();

if (df.GetAttr("empty").GetAndDispose<bool>())
{
return df;
}

using var renameArgs = new PyDict();
using var canonicalName = "canonical".ToPython();
renameArgs.SetItem("collection_symbol", canonicalName);

using var kwargs = Py.kw("inplace", true);

using var index = df.GetAttr("index");
using var setNames = index.GetAttr("set_names");
setNames.Invoke(new[] { renameArgs }, kwargs);

return df;
}
}
}
34 changes: 26 additions & 8 deletions Common/Python/PandasConverter.DataFrameGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ public partial class PandasConverter
private class DataFrameGenerator
{
private static readonly string[] MultiBaseDataCollectionDataFrameNames = new[] { "collection_symbol", "time" };
private static readonly string[] MultiCanonicalOptionDataFrameNames = new[] { "canonical", "time" };
private static readonly string[] SingleBaseDataCollectionDataFrameNames = new[] { "time" };

private readonly Type _dataType;
Expand All @@ -45,7 +46,7 @@ private class DataFrameGenerator
/// PandasData instances for each symbol. Does not hold BaseDataCollection instances.
/// </summary>
private Dictionary<Symbol, PandasData> _pandasData;
private List<(Symbol, DateTime, IEnumerable<ISymbolProvider>)> _collections;
private List<(Symbol Symbol, DateTime Time, IEnumerable<ISymbolProvider> Data)> _collections;

private int _maxLevels;
private bool _shouldUseSymbolOnlyIndex;
Expand Down Expand Up @@ -81,9 +82,9 @@ protected void AddData(IEnumerable<Slice> slices)
{
foreach (var data in slice.AllData)
{
if (_flatten && data is BaseDataCollection collection)
if (_flatten && IsBaseDataCollection(data.GetType()))
{
AddCollection(collection.Symbol, collection.EndTime, collection);
AddCollection(data.Symbol, data.EndTime, (data as IEnumerable).Cast<ISymbolProvider>());
continue;
}

Expand Down Expand Up @@ -151,8 +152,7 @@ protected void AddData<T>(IEnumerable<T> data)
where T : ISymbolProvider
{
var type = typeof(T);
var isBaseDataCollection = type.IsAssignableTo(typeof(BaseData)) &&
type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition().IsAssignableTo(typeof(IEnumerable<>)));
var isBaseDataCollection = IsBaseDataCollection(type);

if (_flatten && isBaseDataCollection)
{
Expand Down Expand Up @@ -215,13 +215,16 @@ public PyObject GenerateDataFrame(int? levels = null, bool sort = true, bool fil
{
return ConcatDataFrames(dataFrames, sort, dropna: true);
}
else if (_collections.DistinctBy(x => x.Item1).Count() > 1)
else if (_collections.DistinctBy(x => x.Symbol).Count() > 1)
{
var keys = collectionsDataFrames
.Select(x => new object[] { x.Item1, x.Item2 })
.Concat(pandasDataDataFrames.Select(x => new object[] { x, DateTime.MinValue }));
var names = _collections.Any(x => x.Symbol.SecurityType.IsOption() && x.Symbol.IsCanonical())
? MultiCanonicalOptionDataFrameNames
: MultiBaseDataCollectionDataFrameNames;

return ConcatDataFrames(dataFrames, keys, MultiBaseDataCollectionDataFrameNames, sort, dropna: true);
return ConcatDataFrames(dataFrames, keys, names, sort, dropna: true);
}
else
{
Expand Down Expand Up @@ -273,7 +276,7 @@ private IEnumerable<PyObject> GetPandasDataDataFrames(int? levels, bool filterMi
yield break;
}

foreach (var (symbol, time, data) in _collections.GroupBy(x => x.Item1).SelectMany(x => x))
foreach (var (symbol, time, data) in _collections.GroupBy(x => x.Symbol).SelectMany(x => x))
{
var generator = new DataFrameGenerator(_dataType, timeAsColumn: !symbolOnlyIndex, flatten: _flatten);
generator.AddData(data);
Expand Down Expand Up @@ -301,6 +304,21 @@ private void AddCollection(Symbol symbol, DateTime time, IEnumerable<ISymbolProv
_collections ??= new();
_collections.Add((symbol, time, data));
}

/// <summary>
/// Determines whether the type is considered a base data collection for flattening.
/// Any object that is a <see cref="BaseData"/> and implements <see cref="IEnumerable{ISymbolProvider}"/>
/// is considered a base data collection.
/// This allows detecting collections of cases like <see cref="OptionUniverse"/> (which is a direct subclass of
/// <see cref="BaseDataCollection"/>) and <see cref="OptionChain"/>, which is a collection of <see cref="OptionContract"/>
/// </summary>
private static bool IsBaseDataCollection(Type type)
{
return type.IsAssignableTo(typeof(BaseData)) &&
type.GetInterfaces().Any(x => x.IsGenericType &&
x.GetGenericTypeDefinition().IsAssignableTo(typeof(IEnumerable<>)) &&
x.GenericTypeArguments[0].IsAssignableTo(typeof(ISymbolProvider)));
}
}

private class DataFrameGenerator<T> : DataFrameGenerator
Expand Down
135 changes: 67 additions & 68 deletions Common/Python/PandasData.DataTypeMember.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,36 +18,40 @@
using System;
using System.Collections.Generic;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text;

namespace QuantConnect.Python
{
public partial class PandasData
{
private static DataTypeMember CreateDataTypeMember(MemberInfo member, DataTypeMember[] children = null)
{
return member switch
{
PropertyInfo property => new PropertyMember(property, children),
FieldInfo field => new FieldMember(field, children),
_ => throw new ArgumentException($"Member type {member.MemberType} is not supported")
};
}

/// <summary>
/// Represents a member of a data type, either a property or a field and it's children members in case it's a complex type.
/// It contains logic to get the member name and the children names, taking into account the parent prefixes.
/// </summary>
private class DataTypeMember
private abstract class DataTypeMember
{
private static readonly StringBuilder _stringBuilder = new StringBuilder();

private PropertyInfo _property;
private FieldInfo _field;

private DataTypeMember _parent;
private string _name;

public MemberInfo Member { get; }

public DataTypeMember[] Children { get; }

public bool IsNonExpandable { get; init; }
public abstract bool IsProperty { get; }

public bool IsProperty => _property != null;

public bool IsField => _field != null;
public abstract bool IsField { get; }

/// <summary>
/// The prefix to be used for the children members when a class being expanded has multiple properties/fields of the same type
Expand All @@ -64,14 +68,11 @@ private class DataTypeMember

public bool IsTickProperty { get; }

private DataTypeMember(MemberInfo member, DataTypeMember[] children = null)
public DataTypeMember(MemberInfo member, DataTypeMember[] children = null)
{
Member = member;
Children = children;

_property = member as PropertyInfo;
_field = member as FieldInfo;

IsTickLastPrice = member == _tickLastPriceMember || member == _openInterestLastPriceMember;
IsTickProperty = IsProperty && member.DeclaringType == typeof(Tick);

Expand All @@ -84,34 +85,6 @@ private DataTypeMember(MemberInfo member, DataTypeMember[] children = null)
}
}

public static DataTypeMember CreateWithChildren(MemberInfo member, DataTypeMember[] children)
{
return new DataTypeMember(member, children);
}

public static DataTypeMember Create(MemberInfo member)
{
return new DataTypeMember(member);
}

public static DataTypeMember CreateNonExpandableMember(MemberInfo member)
{
return new DataTypeMember(member)
{
IsNonExpandable = true
};
}

public PropertyInfo AsProperty()
{
return _property;
}

public FieldInfo AsField()
{
return _field;
}

public void SetPrefix()
{
Prefix = Member.Name.ToLowerInvariant();
Expand Down Expand Up @@ -146,37 +119,13 @@ public IEnumerable<string> GetMemberNames()
return GetMemberNames(null);
}

public object GetValue(object instance)
{
if (IsProperty)
{
return _property.GetValue(instance);
}
public abstract object GetValue(object instance);

return _field.GetValue(instance);
}
public abstract Type GetMemberType();

public override string ToString()
{
return $"{GetMemberType(Member).Name} {Member.Name}";
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Type GetMemberType()
{
return GetMemberType(Member);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Type GetMemberType(MemberInfo member)
{
return member switch
{
PropertyInfo property => property.PropertyType,
FieldInfo field => field.FieldType,
// Should not happen
_ => throw new InvalidOperationException($"Unexpected member type: {member.MemberType}")
};
return $"{GetMemberType().Name} {Member.Name}";
}

private string BuildMemberName(string baseName)
Expand Down Expand Up @@ -229,5 +178,55 @@ private string GetBaseName()
return baseName.ToLowerInvariant();
}
}

private class PropertyMember : DataTypeMember
{
private PropertyInfo _property;

public override bool IsProperty => true;

public override bool IsField => false;

public PropertyMember(PropertyInfo property, DataTypeMember[] children = null)
: base(property, children)
{
_property = property;
}

public override object GetValue(object instance)
{
return _property.GetValue(instance);
}

public override Type GetMemberType()
{
return _property.PropertyType;
}
}

private class FieldMember : DataTypeMember
{
private FieldInfo _field;

public override bool IsProperty => false;

public override bool IsField => true;

public FieldMember(FieldInfo field, DataTypeMember[] children = null)
: base(field, children)
{
_field = field;
}

public override object GetValue(object instance)
{
return _field.GetValue(instance);
}

public override Type GetMemberType()
{
return _field.FieldType;
}
}
}
}
Loading

0 comments on commit 0263ecf

Please sign in to comment.