From 134e3beed42e1946c6a4b393b413b034a733c361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Poniedzia=C5=82ek?= Date: Tue, 26 Mar 2024 11:54:46 +0100 Subject: [PATCH] Toggle some booleans in `Caster` --- .../iglu.schemaddl/parquet/Caster.scala | 8 ++++--- .../iglu/schemaddl/parquet/CasterSpec.scala | 24 +++++++++++++++---- .../schemaddl/parquet/ExampleFieldValue.scala | 4 ++-- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Caster.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Caster.scala index 2f3a13aa..15f6e2a9 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Caster.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Caster.scala @@ -174,16 +174,18 @@ object Caster { .map { name => jsonObject.get(name) match { case Some(Json.Null) => CastAccumulate[A](None, true) - case None => CastAccumulate[A](None, true) + case None => CastAccumulate[A](None, false) case Some(json) => CastAccumulate(cast(caster, field, json).some, false) } } .reduce(_ |+| _) ca match { - case CastAccumulate(Some(Validated.Invalid(_)), true) if field.nullability.nullable => + case CastAccumulate(Some(Validated.Invalid(_)), _) if field.nullability.nullable => + // Shouldn't be an error? Regardless if any null/missing value observed, + // there is no valid value and there are some accessors providing invalid value NamedValue(Field.normalizeName(field), caster.nullValue).validNel - case CastAccumulate(None, true) if field.nullability.nullable => + case CastAccumulate(None, _) if field.nullability.nullable => NamedValue(Field.normalizeName(field), caster.nullValue).validNel case CastAccumulate(None, true) => WrongType(Json.Null, field.fieldType).invalidNel diff --git a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/parquet/CasterSpec.scala b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/parquet/CasterSpec.scala index b7b53e91..2f01e085 100644 --- a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/parquet/CasterSpec.scala +++ b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/parquet/CasterSpec.scala @@ -17,12 +17,13 @@ import io.circe.literal._ import cats.data.NonEmptyList import org.specs2.matcher.ValidatedMatchers._ import org.specs2.matcher.MatchResult - import com.snowplowanalytics.iglu.schemaddl.parquet.Type.Nullability.{Nullable, Required} -import com.snowplowanalytics.iglu.schemaddl.parquet.Type.DecimalPrecision.{Digits9, Digits18, Digits38} +import com.snowplowanalytics.iglu.schemaddl.parquet.Type.DecimalPrecision.{Digits18, Digits38, Digits9} import com.snowplowanalytics.iglu.schemaddl.parquet.Caster.NamedValue import com.snowplowanalytics.iglu.schemaddl.parquet.CastError._ +import java.time.Instant + class CasterSpec extends org.specs2.Specification { def is = s2""" cast transforms any primitive value $e1 cast transforms a UTC timestamp value $e2 @@ -30,6 +31,7 @@ class CasterSpec extends org.specs2.Specification { def is = s2""" cast transforms a date value $e4 cast transforms object with matching primitive fields $e5 cast transforms object with missing nullable field $e6 + cast transforms object with missing required field $e18 cast transforms array values $e7 cast transforms nullable array values $e8 cast strips away undefined properties $e9 @@ -69,13 +71,13 @@ class CasterSpec extends org.specs2.Specification { def is = s2""" def e2 = { val input = json""""2022-02-02T01:02:03.123z"""" - val expected = TimestampValue(java.sql.Timestamp.valueOf("2022-02-02 01:02:03.123")) + val expected = TimestampValue(Instant.parse("2022-02-02T01:02:03.123Z")) testCast(Type.Timestamp, input, expected) } def e3 = { val input = json""""2022-02-02T12:02:03.123+03:00"""" - val expected = TimestampValue(java.sql.Timestamp.valueOf("2022-02-02 09:02:03.123")) + val expected = TimestampValue(Instant.parse("2022-02-02T09:02:03.123Z")) testCast(Type.Timestamp, input, expected) } @@ -277,7 +279,8 @@ class CasterSpec extends org.specs2.Specification { def is = s2""" json"""{"XYZ": 42, "xyz": "invalid"}""" -> StructValue(List(NamedValue("xyz", IntValue(42)))), json"""{"xyz": null, "XYZ": "invalid"}""" -> StructValue(List(NamedValue("xyz", NullValue))), json"""{"XYZ": null, "xyz": "invalid"}""" -> StructValue(List(NamedValue("xyz", NullValue))), - json"""{"XYZ": "invalid"}""" -> StructValue(List(NamedValue("xyz", NullValue))), + json"""{"XYZ": "invalid"}""" -> StructValue(List(NamedValue("xyz", NullValue))), //Should be error? + json"""{"xyz": "invalid", "XYZ": "invalid"}""" -> StructValue(List(NamedValue("xyz", NullValue))), // This one is null now! Should be error? ) .map { case (json, expected) => testCast(inputField, json, expected) @@ -293,4 +296,15 @@ class CasterSpec extends org.specs2.Specification { def is = s2""" val expected = NonEmptyList.one(WrongType(Json.Null, Type.String)) Caster.cast(caster, Field("top", fieldType, Nullable), inputJson) must beInvalid(expected) } + + def e18 = { + val inputJson = json"""{"bar": true}""" + val inputField = Type.Struct(List( + Field("foo", Type.Integer, Required), + Field("bar", Type.Boolean, Required))) + + val expected = NonEmptyList.one(MissingInValue("foo", json"""{"bar": true}""")) + Caster.cast(caster, Field("top", inputField, Nullable), inputJson) must beInvalid(expected) + } + } diff --git a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/parquet/ExampleFieldValue.scala b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/parquet/ExampleFieldValue.scala index 3a2a770d..6d78a2e8 100644 --- a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/parquet/ExampleFieldValue.scala +++ b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/parquet/ExampleFieldValue.scala @@ -28,7 +28,7 @@ object ExampleFieldValue { case class LongValue(value: Long) extends ExampleFieldValue case class DoubleValue(value: Double) extends ExampleFieldValue case class DecimalValue(value: BigDecimal, precision: Type.DecimalPrecision) extends ExampleFieldValue - case class TimestampValue(value: java.sql.Timestamp) extends ExampleFieldValue + case class TimestampValue(value: Instant) extends ExampleFieldValue case class DateValue(value: java.sql.Date) extends ExampleFieldValue case class StructValue(values: List[Caster.NamedValue[ExampleFieldValue]]) extends ExampleFieldValue case class ArrayValue(values: List[ExampleFieldValue]) extends ExampleFieldValue @@ -44,7 +44,7 @@ object ExampleFieldValue { def decimalValue(unscaled: BigInt, details: Type.Decimal): ExampleFieldValue = DecimalValue(BigDecimal(unscaled, details.scale), details.precision) def dateValue(v: LocalDate): ExampleFieldValue = DateValue(java.sql.Date.valueOf(v)) - def timestampValue(v: Instant): ExampleFieldValue = TimestampValue(java.sql.Timestamp.from(v)) + def timestampValue(v: Instant): ExampleFieldValue = TimestampValue(v) def structValue(vs: List[Caster.NamedValue[ExampleFieldValue]]): ExampleFieldValue = StructValue(vs) def arrayValue(vs: List[ExampleFieldValue]): ExampleFieldValue = ArrayValue(vs) }