diff --git a/.github/actions/build_and_coverage/action.yml b/.github/actions/build_and_coverage/action.yml index 130b6130c..d1fb26100 100644 --- a/.github/actions/build_and_coverage/action.yml +++ b/.github/actions/build_and_coverage/action.yml @@ -65,7 +65,7 @@ runs: if: contains(env.SCOPE, 'uploadReport') uses: codecov/codecov-action@v3 with: - files: ./target/scala-2.11/scoverage-report/scoverage.xml,./target/scala-2.12/scoverage-report/scoverage.xml,./target/scala-2.13/scoverage-report/scoverage.xml + files: ./target/scala-2.12/scoverage-report/scoverage.xml,./target/scala-2.13/scoverage-report/scoverage.xml fail_ci_if_error: true verbose: false flags: 'spark-${{ env.SPARK_VERSION }}.x' diff --git a/.github/codecov.yml b/.github/codecov.yml index b6cfe0b03..70114e3a8 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -3,7 +3,7 @@ codecov: branch: main notify: wait_for_ci: true - after_n_builds: 5 + after_n_builds: 4 coverage: precision: 2 @@ -37,4 +37,4 @@ comment: layout: "reach,diff,flags,files,footer" behavior: once # once: update, if exists. Otherwise, post new. Skip if deleted. require_changes: false # if true: only post the comment if coverage changes - after_n_builds: 5 + after_n_builds: 4 diff --git a/.github/workflows/matrix_includes.json b/.github/workflows/matrix_includes.json index 4c6f406d2..2fd5821b9 100644 --- a/.github/workflows/matrix_includes.json +++ b/.github/workflows/matrix_includes.json @@ -1,44 +1,4 @@ [ - { - "spark": "2.4.1", - "scope": "test", - "isRelease": "-" - }, - { - "spark": "2.4.2", - "scope": "test", - "isRelease": "-" - }, - { - "spark": "2.4.3", - "scope": "test", - "isRelease": "-" - }, - { - "spark": "2.4.4", - "scope": "test", - "isRelease": "-" - }, - { - "spark": "2.4.5", - "scope": "test", - "isRelease": "-" - }, - { - "spark": "2.4.6", - "scope": "test", - "isRelease": "-" - }, - { - "spark": "2.4.7", - "scope": "test", - "isRelease": "-" - }, - { - "spark": "2.4", - "scope": "test-uploadReport", - "isRelease": "release" - }, { "spark": "3.0.0", "scope": "test", diff --git a/.scala-steward.conf b/.scala-steward.conf index 762d1c9b6..2cf396e4e 100644 --- a/.scala-steward.conf +++ b/.scala-steward.conf @@ -2,9 +2,6 @@ updates.pin = [ { groupId = "org.scala-lang", artifactId = "scala-library", version = "2.12." } ] -updates.ignore = [ -# Ignored because of spark 2.4 as there is no version for scala 2.11 on further versions - { groupId = "org.scoverage", artifactId = "sbt-scoverage" } -] +updates.ignore = [] commits.message = "ScalaSteward: Update ${artifactName} from ${currentVersion} to ${nextVersion}" diff --git a/README.md b/README.md index 9f284aa5e..83d322cec 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,14 @@ Type-safe columns for spark DataFrames! [![Scala Steward badge](https://img.shields.io/badge/Scala_Steward-helping-blue.svg?style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA4AAAAQCAMAAAARSr4IAAAAVFBMVEUAAACHjojlOy5NWlrKzcYRKjGFjIbp293YycuLa3pYY2LSqql4f3pCUFTgSjNodYRmcXUsPD/NTTbjRS+2jomhgnzNc223cGvZS0HaSD0XLjbaSjElhIr+AAAAAXRSTlMAQObYZgAAAHlJREFUCNdNyosOwyAIhWHAQS1Vt7a77/3fcxxdmv0xwmckutAR1nkm4ggbyEcg/wWmlGLDAA3oL50xi6fk5ffZ3E2E3QfZDCcCN2YtbEWZt+Drc6u6rlqv7Uk0LdKqqr5rk2UCRXOk0vmQKGfc94nOJyQjouF9H/wCc9gECEYfONoAAAAASUVORK5CYII=)](https://scala-steward.org) [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/hablapps/doric/main?filepath=notebooks) -| Spark | Maven Central | Codecov | -|:-----:|:------------------------------------------------------------------------------------------------------------------------------------------------------------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 2.4.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-2.4.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | -| 3.0.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.0.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | -| 3.1.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.1.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | -| 3.2.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.2.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | -| 3.3.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.3.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | -| 3.4.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.4.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | +| Spark | Maven Central | Codecov | +|:-----:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.4.x |Deprecated [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-2.4.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | +| 3.0.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.0.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | +| 3.1.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.1.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | +| 3.2.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.2.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | +| 3.3.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.3.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | +| 3.4.x | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/0.0.7) | [![Codecov](https://img.shields.io/codecov/c/github/hablapps/doric?flag=spark-3.4.x&label=codecov&logo=codecov&token=N7ZXUXZX1I)](https://codecov.io/gh/hablapps/doric) | ---- Doric offers type-safety in DataFrame column expressions at a minimum @@ -66,30 +66,23 @@ _Maven_ `Doric` depends on Spark internals, and it's been tested against the following spark versions. -| Spark | Scala | Tested | doric | -|:-----:|:-----:|:------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------:| -| 2.4.1 | 2.11 | ✅ | - | -| 2.4.2 | 2.11 | ✅ | - | -| 2.4.3 | 2.11 | ✅ | - | -| 2.4.4 | 2.11 | ✅ | - | -| 2.4.5 | 2.11 | ✅ | - | -| 2.4.6 | 2.11 | ✅ | - | -| 2.4.7 | 2.11 | ✅ | - | -| 2.4.8 | 2.11 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/0.0.7) | -| 3.0.0 | 2.12 | ✅ | - | -| 3.0.1 | 2.12 | ✅ | - | -| 3.0.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/0.0.7) | -| 3.1.0 | 2.12 | ✅ | - | -| 3.1.1 | 2.12 | ✅ | - | -| 3.1.2 | 2.12 | ✅ | - | -| 3.1.3 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/0.0.7) | -| 3.2.0 | 2.12 | ✅ | - | -| 3.2.1 | 2.12 | ✅ | - | -| 3.2.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/0.0.7) | -| 3.3.0 | 2.12 | ✅ | - | -| 3.3.1 | 2.12 | ✅ | - | -| 3.3.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/0.0.7) | -| 3.4.0 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/0.0.7) | +| Spark | Scala | Tested | doric | +|:-----:|:-----------:|:------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| 2.4.x | 2.11 | ✅ | Deprecated [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/0.0.7) | +| 3.0.0 | 2.12 | ✅ | - | +| 3.0.1 | 2.12 | ✅ | - | +| 3.0.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/0.0.7) | +| 3.1.0 | 2.12 | ✅ | - | +| 3.1.1 | 2.12 | ✅ | - | +| 3.1.2 | 2.12 | ✅ | - | +| 3.1.3 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/0.0.7) | +| 3.2.0 | 2.12 | ✅ | - | +| 3.2.1 | 2.12 | ✅ | - | +| 3.2.2 | 2.12 / 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/0.0.7) | +| 3.3.0 | 2.12 / 2.13 | ✅ | - | +| 3.3.1 | 2.12 / 2.13 | ✅ | - | +| 3.3.2 | 2.12 / 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/0.0.7) | +| 3.4.0 | 2.12 / 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/0.0.7) | diff --git a/build.sbt b/build.sbt index 1d569768d..2cd017103 100644 --- a/build.sbt +++ b/build.sbt @@ -1,11 +1,10 @@ import scala.language.postfixOps -import sbt.Compile +//import sbt.Compile val stableVersion = "0.0.7" val sparkDefaultShortVersion = "3.4" -val spark24Version = "2.4.8" val spark30Version = "3.0.3" val spark31Version = "3.1.3" val spark32Version = "3.2.3" @@ -15,12 +14,10 @@ val spark34Version = "3.4.0" val versionRegex = """^(.*)\.(.*)\.(.*)$""".r val versionRegexShort = """^(.*)\.(.*)$""".r -val scala211 = "2.11.12" val scala212 = "2.12.15" val scala213 = "2.13.8" val parserSparkVersion: String => String = { - case versionRegexShort("2", "4") => spark24Version case versionRegexShort("3", "0") => spark30Version case versionRegexShort("3", "1") => spark31Version case versionRegexShort("3", "2") => spark32Version @@ -34,7 +31,6 @@ val long2ShortVersion: String => String = { case versionRegex(a, b, _) => } val scalaVersionSelect: String => List[String] = { - case versionRegex("2", _, _) => List(scala211) case versionRegex("3", "0", _) => List(scala212) case versionRegex("3", "1", _) => List(scala212) case versionRegex("3", "2", _) => List(scala212, scala213) @@ -42,11 +38,6 @@ val scalaVersionSelect: String => List[String] = { case versionRegex("3", "4", _) => List(scala212, scala213) } -val catsVersion: String => String = { - case versionRegex("2", _, _) => "2.0.0" - case _ => "2.7.0" -} - ThisBuild / organization := "org.hablapps" ThisBuild / homepage := Some(url("https://github.com/hablapps/doric")) ThisBuild / licenses := List( @@ -128,7 +119,7 @@ lazy val core = project crossScalaVersions := scalaVersionSelect(sparkVersion.value), libraryDependencies ++= Seq( "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided", // scala-steward:off - "org.typelevel" %% "cats-core" % catsVersion(sparkVersion.value), + "org.typelevel" %% "cats-core" % "2.7.0", "com.lihaoyi" %% "sourcecode" % "0.3.0", "com.chuusai" %% "shapeless" % "2.3.10", "org.scala-lang.modules" %% "scala-collection-compat" % "2.11.0", diff --git a/core/src/main/scala_2.11_2.12/scala/doric/types/DoricArray.scala b/core/src/main/scala_2.12/scala/doric/types/DoricArray.scala similarity index 100% rename from core/src/main/scala_2.11_2.12/scala/doric/types/DoricArray.scala rename to core/src/main/scala_2.12/scala/doric/types/DoricArray.scala diff --git a/core/src/main/scala_2.11_2.12/scala/doric/types/SparkTypeLPI_II_Seq_Set_Specific.scala b/core/src/main/scala_2.12/scala/doric/types/SparkTypeLPI_II_Seq_Set_Specific.scala similarity index 100% rename from core/src/main/scala_2.11_2.12/scala/doric/types/SparkTypeLPI_II_Seq_Set_Specific.scala rename to core/src/main/scala_2.12/scala/doric/types/SparkTypeLPI_II_Seq_Set_Specific.scala diff --git a/core/src/main/spark_2.4_mount/scala/doric/syntax/All.scala b/core/src/main/spark_2.4_mount/scala/doric/syntax/All.scala deleted file mode 100644 index d666257e9..000000000 --- a/core/src/main/spark_2.4_mount/scala/doric/syntax/All.scala +++ /dev/null @@ -1,22 +0,0 @@ -package doric.syntax - -private[doric] trait All - extends ArrayColumns - with TypeMatcher - with CommonColumns - with DStructs - with LiteralConversions - with MapColumns - with NumericColumns - with DateColumns - with TimestampColumns - with BooleanColumns - with StringColumns - with ControlStructures - with AggregationColumns - with CNameOps - with BinaryColumns - with Interpolators - with BinaryColumns30_31 - with StringColumn24 - with NumericColumns2_31 diff --git a/core/src/main/spark_2.4_mount/scala/doric/syntax/StringColumn24.scala b/core/src/main/spark_2.4_mount/scala/doric/syntax/StringColumn24.scala deleted file mode 100644 index 39d39af3e..000000000 --- a/core/src/main/spark_2.4_mount/scala/doric/syntax/StringColumn24.scala +++ /dev/null @@ -1,38 +0,0 @@ -package doric -package syntax - -import cats.implicits._ - -import org.apache.spark.sql.{Column, functions => f} -import org.apache.spark.sql.catalyst.expressions.StringSplit - -trait StringColumn24 { - implicit class StringOperationsSyntax24(s: DoricColumn[String]) { - - /** - * Splits str around matches of the given pattern. - * - * @group String Type - * @param pattern - * a string representing a regular expression. The regex string should be - * a Java regular expression. - * @see org.apache.spark.sql.functions.split - * @todo scaladoc link (issue #135) - */ - def split( - pattern: StringColumn - ): ArrayColumn[String] = - (s.elem, pattern.elem) - .mapN((str, p) => new Column(StringSplit(str.expr, p.expr))) - .toDC - - /** - * Parses a JSON string and infers its schema in DDL format. - * @throws org.apache.spark.sql.AnalysisException if it is not a foldable string expression or null - * - * @group String Type - * @see [[org.apache.spark.sql.functions.schema_of_json(json:org\.apache\.spark\.sql\.Column):* org.apache.spark.sql.functions.schema_of_json]] - */ - def schemaOfJson(): StringColumn = s.elem.map(f.schema_of_json).toDC - } -} diff --git a/core/src/main/spark_2.4_mount/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala b/core/src/main/spark_2.4_mount/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala deleted file mode 100644 index b1bf4b08d..000000000 --- a/core/src/main/spark_2.4_mount/scala/org/apache/spark/sql/doric/RelationalGroupedDatasetDoricInterface.scala +++ /dev/null @@ -1,79 +0,0 @@ -package org.apache.spark.sql.doric - -import cats.implicits._ -import doric.{DoricColumn, DoricValidated} -import java.lang.reflect.Field - -import org.apache.spark.sql.{DataFrame, RelationalGroupedDataset} -import org.apache.spark.sql.RelationalGroupedDataset.GroupType - -/** - * Interface to allow doric to access to some privet sql elements - */ -trait RelationalGroupedDatasetDoricInterface { - - private val dfGetter: Field = { - val a = classOf[RelationalGroupedDataset].getDeclaredField( - "org$apache$spark$sql$RelationalGroupedDataset$$df" - ) - a.setAccessible(true) - a - } - - private def aggregationType( - groupType: GroupType, - df: DataFrame, - cols: DoricColumn[_]* - ): DoricValidated[RelationalGroupedDataset] = { - cols.toList - .traverse(_.elem) - .run(df) - .map(x => - RelationalGroupedDataset( - df.toDF(), - x.map(_.expr), - groupType - ) - ) - } - - protected def sparkGroupBy( - df: DataFrame, - cols: DoricColumn[_]* - ): DoricValidated[RelationalGroupedDataset] = - aggregationType(RelationalGroupedDataset.GroupByType, df, cols: _*) - - protected def sparkRollup( - df: DataFrame, - cols: DoricColumn[_]* - ): DoricValidated[RelationalGroupedDataset] = - aggregationType(RelationalGroupedDataset.RollupType, df, cols: _*) - - protected def sparkCube( - df: DataFrame, - cols: DoricColumn[_]* - ): DoricValidated[RelationalGroupedDataset] = - aggregationType(RelationalGroupedDataset.CubeType, df, cols: _*) - - def sparkAgg( - relationalGroupedDataset: RelationalGroupedDataset, - expr: DoricColumn[_], - exprs: DoricColumn[_]* - ): DoricValidated[DataFrame] = { - (expr +: exprs).toList - .traverse(_.elem) - .run(dfGetter.get(relationalGroupedDataset).asInstanceOf[DataFrame]) - .map(x => relationalGroupedDataset.agg(x.head, x.tail: _*)) - } - - def sparkPivot[T]( - relationalGroupedDataset: RelationalGroupedDataset, - expr: DoricColumn[T], - values: Seq[T] - ): DoricValidated[RelationalGroupedDataset] = { - expr.elem - .run(dfGetter.get(relationalGroupedDataset).asInstanceOf[DataFrame]) - .map(x => relationalGroupedDataset.pivot(x, values)) - } - -} diff --git a/core/src/main/spark_2.4_3.0_3.1/scala/doric/syntax/BinaryColumns30_31.scala b/core/src/main/spark_3.0_3.1/scala/doric/syntax/BinaryColumns30_31.scala similarity index 100% rename from core/src/main/spark_2.4_3.0_3.1/scala/doric/syntax/BinaryColumns30_31.scala rename to core/src/main/spark_3.0_3.1/scala/doric/syntax/BinaryColumns30_31.scala diff --git a/core/src/main/spark_2.4_3.0_3.1/scala/doric/syntax/NumericColumns2_31.scala b/core/src/main/spark_3.0_3.1/scala/doric/syntax/NumericColumns2_31.scala similarity index 100% rename from core/src/main/spark_2.4_3.0_3.1/scala/doric/syntax/NumericColumns2_31.scala rename to core/src/main/spark_3.0_3.1/scala/doric/syntax/NumericColumns2_31.scala diff --git a/core/src/main/spark_2.4_3.0_3.1/scala/doric/types/SparkTypeLPI_I_Specific.scala b/core/src/main/spark_3.0_3.1/scala/doric/types/SparkTypeLPI_I_Specific.scala similarity index 100% rename from core/src/main/spark_2.4_3.0_3.1/scala/doric/types/SparkTypeLPI_I_Specific.scala rename to core/src/main/spark_3.0_3.1/scala/doric/types/SparkTypeLPI_I_Specific.scala diff --git a/core/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties new file mode 100644 index 000000000..4a52f5736 --- /dev/null +++ b/core/src/test/resources/log4j.properties @@ -0,0 +1,7 @@ +log4j.rootLogger=ERROR, console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +logrj.appender.console.Target=System.out +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + diff --git a/core/src/test/resources/log4j2.properties b/core/src/test/resources/log4j2.properties new file mode 100644 index 000000000..da16ce0b3 --- /dev/null +++ b/core/src/test/resources/log4j2.properties @@ -0,0 +1,17 @@ +status = warn + +# Name of the configuration +name = ConsoleLogConfigDemo + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Root logger level +rootLogger.level = error + +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = consoleLogger + diff --git a/core/src/test/scala/doric/syntax/StringColumnsSpec.scala b/core/src/test/scala/doric/syntax/StringColumnsSpec.scala index 24ddd8022..89685723f 100644 --- a/core/src/test/scala/doric/syntax/StringColumnsSpec.scala +++ b/core/src/test/scala/doric/syntax/StringColumnsSpec.scala @@ -1,13 +1,14 @@ package doric package syntax -import org.apache.spark.sql.types.{DataType, StructType} -import org.apache.spark.sql.{Row, functions => f} +import scala.jdk.CollectionConverters._ import java.sql.Timestamp -import java.time.format.DateTimeFormatter import java.time.{Instant, LocalDate, ZoneId} -import scala.jdk.CollectionConverters._ +import java.time.format.DateTimeFormatter + +import org.apache.spark.sql.{Row, functions => f} +import org.apache.spark.sql.types.{DataType, StructType} class StringColumnsSpec extends DoricTestElements { @@ -183,9 +184,10 @@ class StringColumnsSpec extends DoricTestElements { } describe("locate doric function") { - import org.apache.spark.sql.functions.{locate => sparkLocate} import spark.implicits._ + import org.apache.spark.sql.functions.{locate => sparkLocate} + val df = List("hello world", "abcde hello hello", "other words", null) .toDF("col1") @@ -319,16 +321,15 @@ class StringColumnsSpec extends DoricTestElements { ) } - if (spark.version >= "2.4.6") { - it("should raise an error if group > regex group result") { - intercept[java.lang.IllegalArgumentException] { - df.withColumn( - "res", - colString("col1").regexpExtract("(\\d+)-(\\d+)".lit, 4.lit) - ).collect() - } + it("should raise an error if group > regex group result") { + intercept[java.lang.IllegalArgumentException] { + df.withColumn( + "res", + colString("col1").regexpExtract("(\\d+)-(\\d+)".lit, 4.lit) + ).collect() } } + } describe("regexpReplace doric function") { diff --git a/core/src/test/scala/doric/types/DeserializeSparkTypeSpec.scala b/core/src/test/scala/doric/types/DeserializeSparkTypeSpec.scala index 3643bcfc6..f98b4abca 100644 --- a/core/src/test/scala/doric/types/DeserializeSparkTypeSpec.scala +++ b/core/src/test/scala/doric/types/DeserializeSparkTypeSpec.scala @@ -1,13 +1,11 @@ package doric package types -import Equalities._ +import doric.Equalities._ -import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.ScalaReflection import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.buildConf import org.apache.spark.sql.types.{Decimal, StructType} class DeserializeSparkTypeSpec @@ -72,20 +70,19 @@ class DeserializeSparkTypeSpec } } - if (spark.version > "2.4.8") - SQLConf.withExistingConf( - spark.sessionState.conf - .copy(DoricTestElements.JAVA8APIENABLED -> true) - ) { - if (spark.version >= "3.2") { - deserializeSparkType[java.sql.Date]( - java.sql.Date.valueOf("2022-12-31") - ) - deserializeSparkType[java.sql.Timestamp](new java.sql.Timestamp(0)) - } - deserializeSparkType[java.time.LocalDate](java.time.LocalDate.now()) - deserializeSparkType[java.time.Instant](java.time.Instant.now()) + SQLConf.withExistingConf( + spark.sessionState.conf + .copy(DoricTestElements.JAVA8APIENABLED -> true) + ) { + if (spark.version >= "3.2") { + deserializeSparkType[java.sql.Date]( + java.sql.Date.valueOf("2022-12-31") + ) + deserializeSparkType[java.sql.Timestamp](new java.sql.Timestamp(0)) } + deserializeSparkType[java.time.LocalDate](java.time.LocalDate.now()) + deserializeSparkType[java.time.Instant](java.time.Instant.now()) + } } } diff --git a/core/src/test/scala/doric/types/PrimitiveTypesSpec.scala b/core/src/test/scala/doric/types/PrimitiveTypesSpec.scala index 583e440bc..2c17d5b23 100644 --- a/core/src/test/scala/doric/types/PrimitiveTypesSpec.scala +++ b/core/src/test/scala/doric/types/PrimitiveTypesSpec.scala @@ -1,7 +1,6 @@ package doric package types -import org.apache.spark.sql.Row import org.apache.spark.sql.types.Decimal import org.apache.spark.unsafe.types.CalendarInterval @@ -90,20 +89,16 @@ class PrimitiveTypesSpec testDataType[java.sql.Date] testDataType[java.sql.Timestamp] - if (spark.version > "2.4.8") { - testDataType[java.time.LocalDate] - testDataType[java.time.Instant] - testDataType[CalendarInterval] - } + testDataType[java.time.LocalDate] + testDataType[java.time.Instant] + testDataType[CalendarInterval] testLitDataType[java.sql.Date](java.sql.Date.valueOf("2022-12-31")) testLitDataType[java.sql.Timestamp](new java.sql.Timestamp(0)) + testLitDataType[java.time.LocalDate](java.time.LocalDate.now()) + testLitDataType[java.time.Instant](java.time.Instant.now()) + // TBD: testLitDataType[CalendarInterval](CalendarInterval.fromString("1971-01-01")) - if (spark.version > "2.4.8") { - testLitDataType[java.time.LocalDate](java.time.LocalDate.now()) - testLitDataType[java.time.Instant](java.time.Instant.now()) - // TBD: testLitDataType[CalendarInterval](CalendarInterval.fromString("1971-01-01")) - } } } diff --git a/core/src/test/scala/doric/types/SerializeSparkTypesSpec.scala b/core/src/test/scala/doric/types/SerializeSparkTypesSpec.scala index 6651038aa..b63d19dd8 100644 --- a/core/src/test/scala/doric/types/SerializeSparkTypesSpec.scala +++ b/core/src/test/scala/doric/types/SerializeSparkTypesSpec.scala @@ -8,9 +8,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.ScalaReflection import org.apache.spark.sql.catalyst.expressions.{GenericRow, GenericRowWithSchema} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.buildConf -import org.apache.spark.sql.types.{Decimal, StructType} -import org.apache.spark.unsafe.types.CalendarInterval +import org.apache.spark.sql.types.StructType class SerializeSparkTypeSpec extends DoricTestElements @@ -66,16 +64,15 @@ class SerializeSparkTypeSpec serializeSparkType[java.time.Instant](java.time.Instant.now()) } - if (spark.version > "2.4.8") - SQLConf.withExistingConf( - spark.sessionState.conf - .copy(DoricTestElements.JAVA8APIENABLED -> true) - ) { - serializeSparkType[java.time.LocalDate](java.time.LocalDate.now()) - serializeSparkType[java.time.Instant](java.time.Instant.now()) - serializeSparkType[java.sql.Date](java.sql.Date.valueOf("2022-12-31")) - serializeSparkType[java.sql.Timestamp](new java.sql.Timestamp(0)) - } + SQLConf.withExistingConf( + spark.sessionState.conf + .copy(DoricTestElements.JAVA8APIENABLED -> true) + ) { + serializeSparkType[java.time.LocalDate](java.time.LocalDate.now()) + serializeSparkType[java.time.Instant](java.time.Instant.now()) + serializeSparkType[java.sql.Date](java.sql.Date.valueOf("2022-12-31")) + serializeSparkType[java.sql.Timestamp](new java.sql.Timestamp(0)) + } // TBD: CalendarIntervalType } diff --git a/docs/docs/docs/api.md b/docs/docs/docs/api.md index f4fa0f682..438ecda82 100644 --- a/docs/docs/docs/api.md +++ b/docs/docs/docs/api.md @@ -8,14 +8,14 @@ permalink: docs/api/ Here you can see the whole doric API for every supported spark version: -| Spark | Scala | API | doric | -|:-----:|:-----:|:---------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------:| -| 2.4.8 | 2.11 | [doric_2-4_2.11](spark-2.4/scala-2.11/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/@STABLE_VERSION@) | -| 3.0.2 | 2.12 | [doric_3-0_2.12](spark-3.0/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/@STABLE_VERSION@) | -| 3.1.3 | 2.12 | [doric_3-1_2.12](spark-3.1/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | -| 3.2.3 | 2.12 | [doric_3-2_2.12](spark-3.2/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | -| 3.2.3 | 2.13 | [doric_3-2_2.13](spark-3.2/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/@STABLE_VERSION@) | -| 3.3.2 | 2.12 | [doric_3-3_2.12](spark-3.3/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | -| 3.3.2 | 2.13 | [doric_3-3_2.13](spark-3.3/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | -| 3.4.0 | 2.12 | [doric_3-3_2.12](spark-3.4/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/@STABLE_VERSION@) | -| 3.4.0 | 2.13 | [doric_3-3_2.13](spark-3.4/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.13/@STABLE_VERSION@) | +| Spark | Scala | API | doric | +|:-----:|:-----:|:--------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| 2.4.8 | 2.11 | Deprecated [doric_2-4_2.11](spark-2.4/scala-2.11/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/0.0.7) | +| 3.0.2 | 2.12 | [doric_3-0_2.12](spark-3.0/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/@STABLE_VERSION@) | +| 3.1.3 | 2.12 | [doric_3-1_2.12](spark-3.1/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | +| 3.2.3 | 2.12 | [doric_3-2_2.12](spark-3.2/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | +| 3.2.3 | 2.13 | [doric_3-2_2.13](spark-3.2/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/@STABLE_VERSION@) | +| 3.3.2 | 2.12 | [doric_3-3_2.12](spark-3.3/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | +| 3.3.2 | 2.13 | [doric_3-3_2.13](spark-3.3/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | +| 3.4.0 | 2.12 | [doric_3-3_2.12](spark-3.4/scala-2.12/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/@STABLE_VERSION@) | +| 3.4.0 | 2.13 | [doric_3-3_2.13](spark-3.4/scala-2.13/) | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.13/@STABLE_VERSION@) | diff --git a/docs/docs/docs/quickstart.md b/docs/docs/docs/quickstart.md index 6f06f1a0d..669af08ce 100644 --- a/docs/docs/docs/quickstart.md +++ b/docs/docs/docs/quickstart.md @@ -31,38 +31,31 @@ Doric is committed to use the most modern APIs first. * The latest experimental version of doric is @VERSION@. * Doric is compatible with the following Spark versions: -| Spark | Scala | Tested | doric | -|:-----:|:-----:|:------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------:| -| 2.4.1 | 2.11 | ✅ | - | -| 2.4.2 | 2.11 | ✅ | - | -| 2.4.3 | 2.11 | ✅ | - | -| 2.4.4 | 2.11 | ✅ | - | -| 2.4.5 | 2.11 | ✅ | - | -| 2.4.6 | 2.11 | ✅ | - | -| 2.4.7 | 2.11 | ✅ | - | -| 2.4.8 | 2.11 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/@STABLE_VERSION@) | -| 3.0.0 | 2.12 | ✅ | - | -| 3.0.1 | 2.12 | ✅ | - | -| 3.0.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/@STABLE_VERSION@) | -| 3.1.0 | 2.12 | ✅ | - | -| 3.1.1 | 2.12 | ✅ | - | -| 3.1.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | -| 3.2.0 | 2.12 | ✅ | - | -| 3.2.0 | 2.13 | ✅ | - | -| 3.2.1 | 2.12 | ✅ | - | -| 3.2.1 | 2.13 | ✅ | - | -| 3.2.2 | 2.12 | ✅ | - | -| 3.2.2 | 2.13 | ✅ | - | -| 3.2.3 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | -| 3.2.3 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/@STABLE_VERSION@) | -| 3.3.0 | 2.12 | ✅ | - | -| 3.3.0 | 2.13 | ✅ | - | -| 3.3.1 | 2.12 | ✅ | - | -| 3.3.1 | 2.13 | ✅ | - | -| 3.3.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | -| 3.3.2 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | -| 3.4.0 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/@STABLE_VERSION@) | -| 3.4.0 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.13/@STABLE_VERSION@) | +| Spark | Scala | Tested | doric | +|:------:|:-----:|:------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| 2.4.x (Deprecated) | 2.11 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_2-4_2.11)](https://mvnrepository.com/artifact/org.hablapps/doric_2-4_2.11/0.0.7) | +| 3.0.0 | 2.12 | ✅ | - | +| 3.0.1 | 2.12 | ✅ | - | +| 3.0.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-0_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-0_2.12/@STABLE_VERSION@) | +| 3.1.0 | 2.12 | ✅ | - | +| 3.1.1 | 2.12 | ✅ | - | +| 3.1.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-1_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-1_2.12/@STABLE_VERSION@) | +| 3.2.0 | 2.12 | ✅ | - | +| 3.2.0 | 2.13 | ✅ | - | +| 3.2.1 | 2.12 | ✅ | - | +| 3.2.1 | 2.13 | ✅ | - | +| 3.2.2 | 2.12 | ✅ | - | +| 3.2.2 | 2.13 | ✅ | - | +| 3.2.3 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.12/@STABLE_VERSION@) | +| 3.2.3 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-2_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-2_2.13/@STABLE_VERSION@) | +| 3.3.0 | 2.12 | ✅ | - | +| 3.3.0 | 2.13 | ✅ | - | +| 3.3.1 | 2.12 | ✅ | - | +| 3.3.1 | 2.13 | ✅ | - | +| 3.3.2 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.12/@STABLE_VERSION@) | +| 3.3.2 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-3_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-3_2.13/@STABLE_VERSION@) | +| 3.4.0 | 2.12 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.12)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.12/@STABLE_VERSION@) | +| 3.4.0 | 2.13 | ✅ | [![Maven Central](https://img.shields.io/maven-central/v/org.hablapps/doric_3-4_2.13)](https://mvnrepository.com/artifact/org.hablapps/doric_3-4_2.13/@STABLE_VERSION@) | __Import statements__ diff --git a/project/plugins.sbt b/project/plugins.sbt index 2e6dc0cbd..771a8e8b0 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,5 +3,5 @@ logLevel := Level.Warn addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.3.7") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.3") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.8") addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2")