UT adjust test SPARK-26677: negated null-safe equality comparison (#1…

…1522) * adjust test case Signed-off-by: fejiang <[email protected]> * format adjust Signed-off-by: fejiang <[email protected]> * modify setting Signed-off-by: fejiang <[email protected]> * changing the methods Signed-off-by: fejiang <[email protected]> * debug the name Signed-off-by: fejiang <[email protected]> --------- Signed-off-by: fejiang <[email protected]>
NVIDIA · Oct 24, 2024 · db15a61 · db15a61
1 parent 5ed0a12
commit db15a61
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 4 deletions.
diff --git a/.../src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetQuerySuite.scala b/.../src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetQuerySuite.scala
@@ -19,9 +19,33 @@
 spark-rapids-shim-json-lines ***/
 package org.apache.spark.sql.rapids.suites
 
+import com.nvidia.spark.rapids.GpuFilterExec
+
+import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.execution.datasources.parquet.ParquetQuerySuite
 import org.apache.spark.sql.rapids.utils.RapidsSQLTestsBaseTrait
 
-class RapidsParquetQuerySuite
-  extends ParquetQuerySuite
-  with RapidsSQLTestsBaseTrait {}
+class RapidsParquetQuerySuite extends ParquetQuerySuite with RapidsSQLTestsBaseTrait {
+  import testImplicits._
+
+  test("SPARK-26677: negated null-safe equality comparison should not filter " +
+    "matched row groupsn Rapids") {
+    withAllParquetReaders {
+      withTempPath { path =>
+        // Repeated values for dictionary encoding.
+        Seq(Some("A"), Some("A"), None).toDF.repartition(1)
+          .write.parquet(path.getAbsolutePath)
+        val df = spark.read.parquet(path.getAbsolutePath)
+        checkAnswer(stripSparkFilterRapids(df.where("NOT (value <=> 'A')")), df)
+      }
+    }
+  }
+
+  def stripSparkFilterRapids(df: DataFrame): DataFrame = {
+    val schema = df.schema
+    val withoutFilters = df.queryExecution.executedPlan.transform {
+      case GpuFilterExec(_, child) => child
+    }
+    spark.internalCreateDataFrame(withoutFilters.execute(), schema)
+  }
+}
diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala
@@ -84,7 +84,7 @@ class RapidsTestSettings extends BackendTestSettings {
     .exclude("struct with unannotated array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/11475"))
     .exclude("unannotated array of struct with unannotated array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/11476"))
   enableSuite[RapidsParquetQuerySuite]
-    .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/11403"))
+    .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups", ADJUST_UT("fetches the CPU version of Execution Plan instead of the GPU version."))
     .exclude("SPARK-34212 Parquet should read decimals correctly", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/11433"))
   enableSuite[RapidsParquetRebaseDatetimeSuite]
     .exclude("SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/11599"))