Doc/link to spark functions

core/src/main/scala/doric/syntax/BinaryColumns.scala

@@ -19,6 +19,7 @@ private[syntax] trait BinaryColumns {
  * as a 32 character hex string.
  *
  * @group Binary Type
+ * @see [[org.apache.spark.sql.functions.md5]]
  */
  def md5: StringColumn = column.elem.map(f.md5).toDC
 
@@ -27,6 +28,7 @@ private[syntax] trait BinaryColumns {
  * as a 40 character hex string.
  *
  * @group Binary Type
+ * @see [[org.apache.spark.sql.functions.sha1]]
  */
  def sha1: StringColumn = column.elem.map(f.sha1).toDC
 
@@ -36,6 +38,7 @@ private[syntax] trait BinaryColumns {
  *
  * @throws java.lang.IllegalArgumentException if numBits is not in the permitted values
  * @group Binary Type
+ * @see [[org.apache.spark.sql.functions.sha2]]
  */
  def sha2(numBits: Int): StringColumn =
  column.elem.map(x => f.sha2(x, numBits)).toDC
@@ -45,6 +48,7 @@ private[syntax] trait BinaryColumns {
  * returns the value as a long column.
  *
  * @group Binary Type
+ * @see [[org.apache.spark.sql.functions.crc32]]
  */
  def crc32: LongColumn = column.elem.map(f.crc32).toDC
 
@@ -53,6 +57,7 @@ private[syntax] trait BinaryColumns {
  * This is the reverse of unbase64.
  *
  * @group Binary Type
+ * @see [[org.apache.spark.sql.functions.base64]]
  */
  def base64: StringColumn = column.elem.map(f.base64).toDC
 
@@ -62,6 +67,7 @@ private[syntax] trait BinaryColumns {
  * If either argument is null, the result will also be null.
  *
  * @group Binary Type
+ * @see [[org.apache.spark.sql.functions.decode]]
  */
  def decode(charset: StringColumn): StringColumn =
  (column.elem, charset.elem)

core/src/main/scala/doric/syntax/BooleanColumns.scala

@@ -67,6 +67,7 @@ private[syntax] trait BooleanColumns {
  *
  * @throws java.lang.RuntimeException if the condition is false
  * @group Boolean Type
+ * @see [[org.apache.spark.sql.functions.assert_true(c:org\.apache\.spark\.sql\.Column):* org.apache.spark.sql.functions.assert_true]]
  */
  def assertTrue: NullColumn = column.elem.map(f.assert_true).toDC
 
@@ -75,6 +76,7 @@ private[syntax] trait BooleanColumns {
  *
  * @throws java.lang.RuntimeException if the condition is false
  * @group Boolean Type
+ * @see [[org.apache.spark.sql.functions.assert_true(c:org\.apache\.spark\.sql\.Column,e:* org.apache.spark.sql.functions.assert_true]]
  */
  def assertTrue(msg: StringColumn): NullColumn =
  (column.elem, msg.elem).mapN(f.assert_true).toDC

core/src/main/scala/doric/syntax/CommonColumns.scala

@@ -31,6 +31,7 @@ private[syntax] trait CommonColumns extends ColGetters[NamedDoricColumn] {
  * the DoricColumns to coalesce
  * @return
  * the first column that is not null, or null if all inputs are null.
+ * @see [[org.apache.spark.sql.functions.coalesce]]
  */
  def coalesce[T](cols: DoricColumn[T]*): DoricColumn[T] =
  cols.map(_.elem).toList.sequence.map(f.coalesce(_: _*)).toDC
@@ -39,6 +40,7 @@ private[syntax] trait CommonColumns extends ColGetters[NamedDoricColumn] {
  * Calculates the hash code of given columns, and returns the result as an integer column.
  *
  * @group All Types
+ * @see [[org.apache.spark.sql.functions.hash]]
  */
  def hash(cols: DoricColumn[_]*): IntegerColumn =
  cols.map(_.elem).toList.sequence.map(f.hash(_: _*)).toDC
@@ -48,6 +50,7 @@ private[syntax] trait CommonColumns extends ColGetters[NamedDoricColumn] {
  * variant of the xxHash algorithm, and returns the result as a long column.
  *
  * @group All Types
+ * @see [[org.apache.spark.sql.functions.xxhash64]]
  */
  def xxhash64(cols: DoricColumn[_]*): LongColumn =
  cols.map(_.elem).toList.sequence.map(f.xxhash64(_: _*)).toDC
@@ -181,6 +184,7 @@ private[syntax] trait CommonColumns extends ColGetters[NamedDoricColumn] {
  * literals to compare to
  * @return
  * Boolean DoricColumn with the comparation logic.
+ * @see [[org.apache.spark.sql.Column.isin]]
  */
  def isIn(elems: T*): BooleanColumn = column.elem.map(_.isin(elems: _*)).toDC
 
@@ -189,6 +193,7 @@ private[syntax] trait CommonColumns extends ColGetters[NamedDoricColumn] {
  * @group All Types
  * @return
  * Boolean DoricColumn
+ * @see [[org.apache.spark.sql.Column.isNull]]
  */
  def isNull: BooleanColumn = column.elem.map(_.isNull).toDC
 
@@ -197,6 +202,7 @@ private[syntax] trait CommonColumns extends ColGetters[NamedDoricColumn] {
  * @group All Types
  * @return
  * Boolean DoricColumn
+ * @see [[org.apache.spark.sql.Column.isNotNull]]
  */
  def isNotNull: BooleanColumn = column.elem.map(_.isNotNull).toDC
 

core/src/main/scala/doric/syntax/DateColumns.scala

@@ -15,6 +15,7 @@ private[syntax] trait DateColumns {
  * All calls of current_date within the same query return the same value.
  *
  * @group Date Type
+ * @see [[org.apache.spark.sql.functions.current_date]]
  */
  def currentDate(): DateColumn = f.current_date().asDoric[Date]
 
@@ -32,6 +33,7 @@ private[syntax] trait DateColumns {
  * Date column after adding months
  * @note
  * Timestamp columns will be truncated to Date column
+ * @see [[org.apache.spark.sql.functions.add_months(startDate:org\.apache\.spark\.sql\.Column,numMonths:org\.apache\.spark\.sql\.Column):* org.apache.spark.sql.functions.add_months]]
  */
  def addMonths(nMonths: IntegerColumn): DateColumn =
  (column.elem, nMonths.elem).mapN(f.add_months).toDC
@@ -44,6 +46,7 @@ private[syntax] trait DateColumns {
  * @note
  * Timestamp columns will be truncated to Date column
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.date_add(start:org\.apache\.spark\.sql\.Column,days:org\.apache\.spark\.sql\.Column):* org.apache.spark.sql.functions.date_add]]
  */
  def addDays(days: IntegerColumn): DateColumn =
  (column.elem, days.elem).mapN(f.date_add).toDC
@@ -59,6 +62,7 @@ private[syntax] trait DateColumns {
  * Use specialized functions like 'year' whenever possible as they benefit from a
  * specialized implementation.
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.date_format]]
  */
  def format(format: StringColumn): StringColumn =
  (column.elem, format.elem)
@@ -75,6 +79,7 @@ private[syntax] trait DateColumns {
  * @note
  * Timestamp columns will be truncated to Date column
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.date_sub(start:org\.apache\.spark\.sql\.Column,days:org\.apache\.spark\.sql\.Column):* org.apache.spark.sql.functions.date_sub]]
  */
  def subDays(days: IntegerColumn): DateColumn =
  (column.elem, days.elem).mapN(f.date_sub).toDC
@@ -85,6 +90,7 @@ private[syntax] trait DateColumns {
  * @param dateCol
  * A Date or Timestamp column
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.datediff]]
  */
  def diff(dateCol: DoricColumn[T]): IntegerColumn =
  (column.elem, dateCol.elem)
@@ -95,6 +101,7 @@ private[syntax] trait DateColumns {
  * Extracts the day of the month as an integer from a given date.
  *
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.dayofmonth]]
  */
  def dayOfMonth: IntegerColumn = column.elem.map(f.dayofmonth).toDC
 
@@ -103,20 +110,23 @@ private[syntax] trait DateColumns {
  * Ranges from 1 for a Sunday through to 7 for a Saturday
  *
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.dayofweek]]
  */
  def dayOfWeek: IntegerColumn = column.elem.map(f.dayofweek).toDC
 
  /**
  * Extracts the day of the year as an integer from a given date.
  *
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.dayofyear]]
  */
  def dayOfYear: IntegerColumn = column.elem.map(f.dayofyear).toDC
 
  /**
  * Sets the moment to the last day of the same month.
  *
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.last_day]]
  */
  def endOfMonth: DateColumn = lastDayOfMonth
 
@@ -126,13 +136,15 @@ private[syntax] trait DateColumns {
  * month in July 2015.
  *
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.last_day]]
  */
  def lastDayOfMonth: DateColumn = column.elem.map(f.last_day).toDC
 
  /**
  * Extracts the month as an integer from a given date.
  *
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.month]]
  */
  def month: IntegerColumn = column.elem.map(f.month).toDC
 
@@ -143,7 +155,7 @@ private[syntax] trait DateColumns {
  * of their respective months. Otherwise, the difference is calculated assuming 31 days per month.
  *
  * For example:
- * {{{
+ * @example {{{
  * Date("2017-11-14").monthsBetween(Date("2017-07-14")) // returns 4.0
  * Date("2017-01-01").monthsBetween(Date("2017-01-10")) // returns 0.29032258
  * Timestamp("2017-06-01 00:00:00").monthsBetween(Timestamp("2017-06-16 12:00:00")) // returns -0.5
@@ -152,6 +164,7 @@ private[syntax] trait DateColumns {
  * @param dateCol
  * Date or Timestamp column
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.months_between(end:org\.apache\.spark\.sql\.Column,start:org\.apache\.spark\.sql\.Column):* org.apache.spark.sql.functions.months_between]]
  */
  def monthsBetween(dateCol: DoricColumn[T]): DoubleColumn =
  (column.elem, dateCol.elem).mapN(f.months_between).toDC
@@ -165,6 +178,7 @@ private[syntax] trait DateColumns {
  * If `roundOff` is set to true, the result is rounded off to 8 digits;
  * it is not rounded otherwise.
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.months_between(end:org\.apache\.spark\.sql\.Column,start:org\.apache\.spark\.sql\.Column,roundOff:* org.apache.spark.sql.functions.months_between]]
  */
  def monthsBetween(
  dateCol: DoricColumn[T],
@@ -180,14 +194,15 @@ private[syntax] trait DateColumns {
  * Returns the first date which is later than the value of the `date` column that is on the
  * specified day of the week.
  *
- * For example, `Date("2015-07-27").nextDay("Sunday")` returns Date("2015-08-02") because
- * that is the first Sunday after 2015-07-27.
+ * @example For example, `Date("2015-07-27").nextDay("Sunday")` returns Date("2015-08-02")
+ * because that is the first Sunday after 2015-07-27.
  *
  * @param dayOfWeek
  * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"
  * @note
  * Timestamp columns will be truncated to Date column
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.next_day]]
  */
  def nextDay(dayOfWeek: StringColumn): DateColumn =
  (column.elem, dayOfWeek.elem)
@@ -200,28 +215,30 @@ private[syntax] trait DateColumns {
  * Extracts the quarter as an integer from a given date.
  *
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.quarter]]
  */
  def quarter: IntegerColumn = column.elem.map(f.quarter).toDC
 
  /**
  * Returns date truncated to the unit specified by the format.
  *
- * For example, `Timestamp("2018-11-19 12:01:19").trunc("year")` returns Date("2018-01-01")
+ * @example For example, `Timestamp("2018-11-19 12:01:19").trunc("year")` returns Date("2018-01-01")
  *
  * @param format
- * if date:
- * * 'year', 'yyyy', 'yy' to truncate by year,
- * * 'month', 'mon', 'mm' to truncate by month
- * Other options are: 'week', 'quarter'
- * if timestamp:
- * * 'year', 'yyyy', 'yy' to truncate by year,
- * * 'month', 'mon', 'mm' to truncate by month,
- * * 'day', 'dd' to truncate by day,
- * Other options are:
- * * 'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter'
+ * - if <b>date</b>:
+ * - 'year', 'yyyy', 'yy' to truncate by year,
+ * - 'month', 'mon', 'mm' to truncate by month
+ * - __Other options are__: 'week', 'quarter'
+ * - if <b>timestamp</b>:
+ * - 'year', 'yyyy', 'yy' to truncate by year,
+ * - 'month', 'mon', 'mm' to truncate by month,
+ * - 'day', 'dd' to truncate by day,
+ * - __Other options are__: 'microsecond', 'millisecond', 'second', 'minute', 'hour', 'week', 'quarter'
  * @note
  * Timestamp columns will be truncated to Date column
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.trunc]]
+ * @see [[org.apache.spark.sql.functions.date_trunc]]
  */
  def truncate(format: StringColumn): DoricColumn[T] =
  (column.elem, format.elem)
@@ -243,6 +260,7 @@ private[syntax] trait DateColumns {
  * A long
  *
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.unix_timestamp(s:org\.apache\.spark\.sql\.Column):* org.apache.spark.sql.functions.unix_timestamp]]
  */
  def unixTimestamp: LongColumn = column.elem.map(f.unix_timestamp).toDC
 
@@ -253,27 +271,31 @@ private[syntax] trait DateColumns {
  * as defined by ISO 8601
  *
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.weekofyear]]
  */
  def weekOfYear: IntegerColumn = column.elem.map(f.weekofyear).toDC
 
  /**
  * Extracts the year as an integer from a given date.
  *
  * @group Date & Timestamp Type
+ * @see [[org.apache.spark.sql.functions.year]]
  */
  def year: IntegerColumn = column.elem.map(f.year).toDC
 
  /**
  * Transform date to timestamp
  *
  * @group Date Type
+ * @see [[org.apache.spark.sql.functions.to_timestamp(s:org\.apache\.spark\.sql\.Column):* org.apache.spark.sql.functions.to_timestamp]]
  */
  def toTimestamp: TimestampColumn = column.elem.map(f.to_timestamp).toDC
 
  /**
  * Transform date to Instant
  *
  * @group Date Type
+ * @see [[org.apache.spark.sql.functions.to_timestamp(s:org\.apache\.spark\.sql\.Column):* org.apache.spark.sql.functions.to_timestamp]]
  */
  def toInstant: InstantColumn = column.elem.map(f.to_timestamp).toDC
  }