From dcbc5952a6a672c2ef0c6009e46336a2decfc0d5 Mon Sep 17 00:00:00 2001 From: Armanbqt <57376624+Armanbqt@users.noreply.github.com> Date: Wed, 15 Jan 2020 17:37:50 -0800 Subject: [PATCH] Adds UserTreeReader, SystemTreeReader, and IonReaderBuilder.Build(IIonValue). (#51) This enables users to use the `IIonReader` API over the `IIonValue` representation of an Ion stream. --- IonDotnet.Tests/Internals/TreeReaderTest.cs | 249 +++++++++++++++ IonDotnet/Builders/IonReaderBuilder.cs | 7 + IonDotnet/Internals/Tree/SystemTreeReader.cs | 313 +++++++++++++++++++ IonDotnet/Internals/Tree/UserTreeReader.cs | 135 ++++++++ IonDotnet/Tree/IIonLob.cs | 1 + IonDotnet/Tree/Impl/IonLob.cs | 5 + IonDotnet/Tree/Impl/IonStruct.cs | 5 + IonDotnet/Tree/Impl/IonValue.cs | 5 + 8 files changed, 720 insertions(+) create mode 100644 IonDotnet.Tests/Internals/TreeReaderTest.cs create mode 100644 IonDotnet/Internals/Tree/SystemTreeReader.cs create mode 100644 IonDotnet/Internals/Tree/UserTreeReader.cs diff --git a/IonDotnet.Tests/Internals/TreeReaderTest.cs b/IonDotnet.Tests/Internals/TreeReaderTest.cs new file mode 100644 index 00000000..a54c3bc7 --- /dev/null +++ b/IonDotnet.Tests/Internals/TreeReaderTest.cs @@ -0,0 +1,249 @@ +using System; +using System.Linq; +using System.Numerics; +using IonDotnet.Internals.Binary; +using IonDotnet.Internals.Tree; +using IonDotnet.Tests.Common; +using IonDotnet.Tree; +using IonDotnet.Tree.Impl; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace IonDotnet.Tests.Internals +{ + [TestClass] + public class TreeReaderTest + { + + private IValueFactory _ionValueFactory = new ValueFactory(); + + [TestMethod] + public void SingleIntNumberTest() + { + var value = _ionValueFactory.NewInt(123); + var reader = new UserTreeReader(value); + + ReaderTestCommon.SingleNumber(reader, 123); + } + + [TestMethod] + public void SingleDecimalNumberTest() + { + var decimalValue = new BigDecimal(decimal.MaxValue); + var value = _ionValueFactory.NewDecimal(decimalValue); + var reader = new UserTreeReader(value); + + Assert.AreEqual(IonType.Decimal, reader.MoveNext()); + Assert.AreEqual(decimalValue, reader.DecimalValue()); + } + + + [TestMethod] + public void SingleDoubleNumberTest() + { + var value = _ionValueFactory.NewFloat(123.456); + var reader = new UserTreeReader(value); + + Assert.AreEqual(IonType.Float, reader.MoveNext()); + Assert.AreEqual(123.456, reader.DoubleValue()); + } + + [TestMethod] + public void TimestampTest() + { + var timestamp = new Timestamp(DateTime.Now); + var value = _ionValueFactory.NewTimestamp(timestamp); + var reader = new UserTreeReader(value); + + Assert.AreEqual(IonType.Timestamp, reader.MoveNext()); + Assert.AreEqual(timestamp, reader.TimestampValue()); + } + + [TestMethod] + public void BoolValueTest() + { + var value = _ionValueFactory.NewBool(true); + var reader = new UserTreeReader(value); + + ReaderTestCommon.SingleBool(reader, true); + } + + [TestMethod] + public void StringValueTest() + { + var value = _ionValueFactory.NewString("test"); + var reader = new UserTreeReader(value); + + Assert.AreEqual(IonType.String, reader.MoveNext()); + Assert.AreEqual("test", reader.StringValue()); + } + + [TestMethod] + public void NullValueTest() + { + var value = _ionValueFactory.NewNull(); + var reader = new UserTreeReader(value); + + Assert.AreEqual(IonType.Null, reader.MoveNext()); + Assert.IsTrue(reader.CurrentIsNull); + } + + [TestMethod] + public void ListOfIntsTest() + { + //Must be: [123,456,789] + var value = _ionValueFactory.NewEmptyList(); + value.Add(_ionValueFactory.NewInt(123)); + value.Add(_ionValueFactory.NewInt(456)); + value.Add(_ionValueFactory.NewInt(789)); + var reader = new UserTreeReader(value); + + ReaderTestCommon.FlatIntList(reader); + } + + [TestMethod] + public void SimpleDatagramTest() + { + //simple datagram: {yolo:true} + var value = new IonStruct{{ "yolo", _ionValueFactory.NewBool(true) }}; + var reader = new UserTreeReader(value); + + ReaderTestCommon.OneBoolInStruct(reader); + } + + [TestMethod] + public void FlatStructScalarTest() + { + //Must be a flat struct of scalar values: + //boolean:true + //str:"yes" + //integer:123456 + //longInt:int.Max*2 + //bigInt:long.Max*10 + //double:2213.1267567f + var value = new IonStruct + { + { "boolean", _ionValueFactory.NewBool(true) }, + { "str", _ionValueFactory.NewString("yes") }, + { "integer", _ionValueFactory.NewInt(123456) }, + { "longInt", _ionValueFactory.NewInt((long)int.MaxValue * 2) }, + { "bigInt", _ionValueFactory.NewInt(BigInteger.Multiply(new BigInteger(long.MaxValue), 10)) }, + { "double", _ionValueFactory.NewFloat(2213.1267567) } + }; + var reader = new UserTreeReader(value); + + ReaderTestCommon.FlatScalar(reader); + } + + [TestMethod] + public void SingleSymbolTest() + { + //{single_symbol:'something'} + var value = new IonStruct {{ "single_symbol", _ionValueFactory.NewSymbol("something")}}; + var reader = new UserTreeReader(value); + + ReaderTestCommon.SingleSymbol(reader); + } + + [TestMethod] + public void NestedAndCombinedListStructTest() + { + //Must be: + // { + // menu: { + // id: "file", + // popup: [ + // "Open", + // "Load", + // "Close" + // ], + // deep1: { + // deep2: { + // deep3: { + // deep4val: "enddeep" + // } + // } + // }, + // positions: [ + // 1234, + // 5678, + // 90 + // ] + // } + // } + + var popupList = _ionValueFactory.NewEmptyList(); + popupList.Add(_ionValueFactory.NewString("Open")); + popupList.Add(_ionValueFactory.NewString("Load")); + popupList.Add(_ionValueFactory.NewString("Close")); + + var positionList = _ionValueFactory.NewEmptyList(); + positionList.Add(_ionValueFactory.NewInt(1234)); + positionList.Add(_ionValueFactory.NewInt(5678)); + positionList.Add(_ionValueFactory.NewInt(90)); + + var deep3 = new IonStruct {{ "deep4val", _ionValueFactory.NewString("enddeep") }}; + + var deep2 = new IonStruct {{ "deep3", deep3 }}; + + var deep1 = new IonStruct {{ "deep2", deep2 }}; + + var menu = new IonStruct + { + { "id", _ionValueFactory.NewString("file") }, + { "popup", popupList }, + { "deep1", deep1 }, + { "positions", positionList } + }; + + var value = new IonStruct {{ "menu", menu }}; + var reader = new UserTreeReader(value); + + ReaderTestCommon.Combined1(reader); + } + + [TestMethod] + public void ValueWithAnnotationTest() + { + //Must be: {withannot: years::months::days::hours::minutes::seconds::18} + var intValue = _ionValueFactory.NewInt(18); + intValue.AddTypeAnnotation("years"); + intValue.AddTypeAnnotation("months"); + intValue.AddTypeAnnotation("days"); + intValue.AddTypeAnnotation("hours"); + intValue.AddTypeAnnotation("minutes"); + intValue.AddTypeAnnotation("seconds"); + var value = new IonStruct {{ "withannot", intValue }}; + var reader = new UserTreeReader(value); + + ReaderTestCommon.ReadAnnotations_SingleField(reader); + } + + [TestMethod] + public void BlobTest() + { + //Must be in a struct: + // { blobbbb: {{data}} } + var arrayOfbytes = Enumerable.Repeat(1, 100).ToArray(); + ReadOnlySpan bytes = new ReadOnlySpan(arrayOfbytes); + var blob = _ionValueFactory.NewBlob(bytes); + var value = new IonStruct {{ "blobbbb", blob }}; + var reader = new UserTreeReader(value); + + ReaderTestCommon.Struct_OneBlob(reader); + } + + [TestMethod] + public void BlobPartialReadTest() + { + var blob = new byte[30]; + for (var i = 0; i < 30; i++) + { + blob[i] = (byte)i; + } + var value = _ionValueFactory.NewBlob(blob); + var reader = new UserTreeReader(value); + + ReaderTestCommon.Blob_PartialRead(30, 7, reader); + } + } +} diff --git a/IonDotnet/Builders/IonReaderBuilder.cs b/IonDotnet/Builders/IonReaderBuilder.cs index 2187bfb7..1de8f169 100644 --- a/IonDotnet/Builders/IonReaderBuilder.cs +++ b/IonDotnet/Builders/IonReaderBuilder.cs @@ -3,6 +3,8 @@ using System.Text; using IonDotnet.Internals.Binary; using IonDotnet.Internals.Text; +using IonDotnet.Internals.Tree; +using IonDotnet.Tree; namespace IonDotnet.Builders { @@ -40,6 +42,11 @@ public static IIonReader Build(string text, ReaderOptions options = default) return new UserTextReader(text, options.Catalog); } + public static IIonReader Build(IIonValue value, ReaderOptions options = default) + { + return new UserTreeReader(value, options.Catalog); + } + public static IIonReader Build(byte[] data, ReaderOptions options = default) { return Build(new MemoryStream(data), options); diff --git a/IonDotnet/Internals/Tree/SystemTreeReader.cs b/IonDotnet/Internals/Tree/SystemTreeReader.cs new file mode 100644 index 00000000..c06b5abf --- /dev/null +++ b/IonDotnet/Internals/Tree/SystemTreeReader.cs @@ -0,0 +1,313 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Numerics; +using IonDotnet.Tree; +using IonDotnet.Tree.Impl; + +namespace IonDotnet.Internals.Tree +{ + internal class SystemTreeReader : IIonReader + { + protected readonly ISymbolTable _systemSymbols; + protected IEnumerator _iter; + protected IIonValue _parent; + protected IIonValue _next; + protected IIonValue _current; + protected bool _eof; + protected int _top; + // Holds pairs: IonValue parent (_parent), Iterator cursor (_iter) + private Object[] _stack = new Object[10]; + private int pos = 0; + + protected SystemTreeReader(IIonValue value) + { + _systemSymbols = SharedSymbolTable.GetSystem(1); + _current = null; + _eof = false; + _top = 0; + if (value.Type() == IonType.Datagram) + { + _parent = value; + _next = null; + _iter = value.GetEnumerator(); + } + else + { + _parent = (IIonValue)value.Container; + _next = value; + } + } + + public int CurrentDepth => _top/2; + + public IonType CurrentType => _current.Type(); + + public string CurrentFieldName => _current.FieldNameSymbol.Text; + + public bool CurrentIsNull => _current.IsNull; + + public bool IsInStruct => CurrentDepth > 0 && _parent.Type() == IonType.Struct; + + public BigInteger BigIntegerValue() + { + return _current.BigIntegerValue; + } + + public bool BoolValue() + { + return _current.BoolValue; + } + + public BigDecimal DecimalValue() + { + return _current.BigDecimalValue; + } + + public double DoubleValue() + { + return _current.DoubleValue; + } + + public int GetBytes(Span buffer) + { + var lobSize = GetLobByteSize(); + var bufSize = buffer.Length; + + if (lobSize < 0 || bufSize < 0) + { + return 0; + } + else if (lobSize <= bufSize) + { + _current.Bytes().CopyTo(buffer); + pos += lobSize; + return lobSize; + } + else if (lobSize > bufSize && pos <= lobSize) + { + _current.Bytes() + .Slice(pos, bufSize) + .CopyTo(buffer); + pos += bufSize; + return bufSize; + } + + throw new IonException("Problem while copying the current blob value to a buffer"); + } + + public SymbolToken GetFieldNameSymbol() + { + return _current.FieldNameSymbol; + } + + public IntegerSize GetIntegerSize() + { + return _current.IntegerSize; + } + + public int GetLobByteSize() + { + return _current.ByteSize(); + } + + public virtual ISymbolTable GetSymbolTable() => _systemSymbols; + + public IEnumerable GetTypeAnnotations() + { + return _current.GetTypeAnnotations(); + } + + public int IntValue() + { + return _current.IntValue; + } + + public long LongValue() + { + return _current.LongValue; + } + + public virtual IonType MoveNext() + { + if (_next == null && !HasNext()) + { + _current = null; + return IonType.None; + } + _current = _next; + _next = null; + + return ((IonValue)_current).Type(); + } + + public byte[] NewByteArray() + { + return _current.Bytes().ToArray(); + } + + public void StepIn() + { + if (!IsContainer()) + { + throw new IonException("current value must be a container"); + } + + Push(); + _parent = _current; + _iter = new Children(_current); + _current = null; + } + + private bool IsContainer() + { + return _current.Type() == IonType.Struct + || _current.Type() == IonType.List + || _current.Type() == IonType.Sexp + || _current.Type() == IonType.Datagram; + } + + private void Push() + { + int oldLen = _stack.Length; + if (_top + 1 >= oldLen) + { + // we're going to do a "+2" on top so we need extra space + int newLen = oldLen * 2; + Object[] temp = new Object[newLen]; + Array.Copy(_stack, 0, temp, 0, oldLen); + _stack = temp; + } + _stack[_top++] = _parent; + _stack[_top++] = _iter; + } + + private void Pop() + { + _top--; + _iter = (IEnumerator)_stack[_top]; + _stack[_top] = null; // Allow iterator to be garbage collected! + + _top--; + _parent = (IIonValue)_stack[_top]; + _stack[_top] = null; + + // We don't know if we're at the end of the container, so check again. + _eof = false; + } + + public virtual bool HasNext() + { + IonType nextType = NextHelperSystem(); + return (nextType != IonType.None); + } + + protected IonType NextHelperSystem() + { + if (_eof) return IonType.None; + if (_next != null) return _next.Type(); + + if(_iter != null && _iter.MoveNext()) + { + _next = _iter.Current; + } + + if ((_eof = (_next == null))) + { + return IonType.None; + } + return _next.Type(); + } + + public void StepOut() + { + if (_top < 1) + { + throw new IonException("Cannot stepOut any further, already at top level."); + } + Pop(); + _current = null; + } + + public string StringValue() + { + return _current.StringValue; + } + + public SymbolToken SymbolValue() + { + return _current.SymbolValue; + } + + public Timestamp TimestampValue() + { + return _current.TimestampValue; + } + + internal class Children : IEnumerator + { + bool _eof; + int _nextIdx; + IIonValue _parent; + IIonValue _curr; + + public Children(IIonValue parent) + { + _parent = parent; + _nextIdx = -1; + _curr = null; + if (_parent.IsNull) + { + // otherwise the empty contents member will cause trouble + _eof = true; + } + } + + public IIonValue Current + { + get + { + try + { + return _parent.GetElementAt(_nextIdx); + } + catch (IndexOutOfRangeException) + { + throw new InvalidOperationException(); + } + } + } + + object IEnumerator.Current => _curr; + + public void Dispose() + { + throw new NotImplementedException(); + } + + public bool MoveNext() + { + if (_eof) + { + _curr = null; + return false; + } + + if (_nextIdx >= _parent.Count - 1) + { + _eof = true; + } + else + { + _curr = _parent.GetElementAt(++_nextIdx); + } + return !_eof; + } + + public void Reset() + { + throw new NotImplementedException(); + } + } + } +} diff --git a/IonDotnet/Internals/Tree/UserTreeReader.cs b/IonDotnet/Internals/Tree/UserTreeReader.cs new file mode 100644 index 00000000..b348c2c7 --- /dev/null +++ b/IonDotnet/Internals/Tree/UserTreeReader.cs @@ -0,0 +1,135 @@ +using System; +using IonDotnet.Tree; + +namespace IonDotnet.Internals.Tree +{ + internal class UserTreeReader : SystemTreeReader + { + /// The ID of system symbol {@value #ION_1_0}, as defined by Ion 1.0. + private const int ION_1_0_SID = 2; + private readonly ICatalog _catalog; + private ISymbolTable _currentSymtab; + private int _symbolTableTop = 0; + private ISymbolTable[] _symbolTableStack = new ISymbolTable[3]; // 3 is rare, IVM followed by a local sym tab with open content + + public UserTreeReader(IIonValue value, ICatalog catalog = null) : base(value) + { + _catalog = catalog; + _currentSymtab = _systemSymbols; + } + + public override ISymbolTable GetSymbolTable() => throw new InvalidOperationException("This operation is not supported."); + + public override bool HasNext() + { + return NextHelperUser(); + } + + public override IonType MoveNext() + { + if (!NextHelperUser()) + { + _current = null; + return IonType.None; + } + _current = _next; + _next = null; + return _current.Type(); + } + + bool NextHelperUser() + { + if (_eof) return false; + if (_next != null) return true; + + ClearSystemValueStack(); + + // read values from the system + // reader and if they are system values + // process them. Return when we've + // read all the immediate system values + IonType nextType; + while (true) + { + nextType = NextHelperSystem(); + + if (_top == 0 && _parent.Type() == IonType.Datagram) + { + if (IonType.Symbol == nextType) + { + var sym = _next; + if (sym.IsNull) + { + // there are no null values we will consume here + break; + } + int sid = sym.SymbolValue.Sid; + if (sid == -1) // if sid is unknown + { + String name = sym.SymbolValue.Text; + if (name != null) + { + sid = _systemSymbols.FindSymbolId(name); + } + } + if (sid == ION_1_0_SID && _next.GetTypeAnnotations().Count == 0) + { + // $ion_1_0 is read as an IVM only if it is not annotated + ISymbolTable symbols = _systemSymbols; + _currentSymtab = symbols; + PushSymbolTable(symbols); + _next = null; + continue; + } + } + else if (IonType.Struct == nextType && _next.HasAnnotation("$ion_symbol_table")) + { + // read a local symbol table + IIonReader reader = new UserTreeReader(_next, _catalog); + ISymbolTable symtab = ReaderLocalTable.ImportReaderTable(this, _catalog, true); + _currentSymtab = symtab; + PushSymbolTable(symtab); + _next = null; + continue; + } + } + // if we get here we didn't process a system + // value, if we had we would have 'continue'd + // so this is a value the user gets + break; + } + return (nextType != IonType.None); + } + + private void ClearSystemValueStack() + { + while (_symbolTableTop > 0) + { + _symbolTableTop--; + _symbolTableStack[_symbolTableTop] = null; + } + } + private void PushSymbolTable(ISymbolTable symbols) + { + if (_symbolTableTop >= _symbolTableStack.Length) + { + int new_len = _symbolTableStack.Length * 2; + ISymbolTable[] temp = new ISymbolTable[new_len]; + Array.Copy(_symbolTableStack, 0, temp, 0, _symbolTableStack.Length); + _symbolTableStack = temp; + } + _symbolTableStack[_symbolTableTop++] = symbols; + } + private ISymbolTable PopPassedSymbolTable() + { + if (_symbolTableTop <= 0) + { + return null; + } + _symbolTableTop--; + ISymbolTable symbols = _symbolTableStack[_symbolTableTop]; + _symbolTableStack[_symbolTableTop] = null; + return symbols; + } + } +} diff --git a/IonDotnet/Tree/IIonLob.cs b/IonDotnet/Tree/IIonLob.cs index 29276a5e..d1d20b9e 100644 --- a/IonDotnet/Tree/IIonLob.cs +++ b/IonDotnet/Tree/IIonLob.cs @@ -5,5 +5,6 @@ public interface IIonLob { ReadOnlySpan Bytes(); void SetBytes(ReadOnlySpan buffer); + int ByteSize(); } } diff --git a/IonDotnet/Tree/Impl/IonLob.cs b/IonDotnet/Tree/Impl/IonLob.cs index 28311e44..9ad0e6be 100644 --- a/IonDotnet/Tree/Impl/IonLob.cs +++ b/IonDotnet/Tree/Impl/IonLob.cs @@ -50,5 +50,10 @@ public override void MakeNull() base.MakeNull(); ByteBuffer = null; } + + public override int ByteSize() + { + return ByteBuffer.Length; + } } } diff --git a/IonDotnet/Tree/Impl/IonStruct.cs b/IonDotnet/Tree/Impl/IonStruct.cs index 669cbc9f..8656a01c 100644 --- a/IonDotnet/Tree/Impl/IonStruct.cs +++ b/IonDotnet/Tree/Impl/IonStruct.cs @@ -158,6 +158,11 @@ public override IEnumerator GetEnumerator() } } + public override IIonValue GetElementAt(int index) + { + return _values.ElementAt(index); + } + public override bool Remove(IIonValue item) { ThrowIfNull(); diff --git a/IonDotnet/Tree/Impl/IonValue.cs b/IonDotnet/Tree/Impl/IonValue.cs index e63ac7bb..0dbed742 100644 --- a/IonDotnet/Tree/Impl/IonValue.cs +++ b/IonDotnet/Tree/Impl/IonValue.cs @@ -450,6 +450,11 @@ public virtual ReadOnlySpan Bytes() } public virtual void SetBytes(ReadOnlySpan buffer) + { + throw new InvalidOperationException(GetErrorMessage()); + } + + public virtual int ByteSize() { throw new InvalidOperationException(GetErrorMessage()); }