diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetQuerySuite.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetQuerySuite.scala index 6dfc44d4bfa..4ce44ddfd73 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetQuerySuite.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/suites/RapidsParquetQuerySuite.scala @@ -19,9 +19,33 @@ spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.suites +import com.nvidia.spark.rapids.GpuFilterExec + +import org.apache.spark.sql.DataFrame import org.apache.spark.sql.execution.datasources.parquet.ParquetQuerySuite import org.apache.spark.sql.rapids.utils.RapidsSQLTestsBaseTrait -class RapidsParquetQuerySuite - extends ParquetQuerySuite - with RapidsSQLTestsBaseTrait {} +class RapidsParquetQuerySuite extends ParquetQuerySuite with RapidsSQLTestsBaseTrait { + import testImplicits._ + + test("SPARK-26677: negated null-safe equality comparison should not filter " + + "matched row groupsn Rapids") { + withAllParquetReaders { + withTempPath { path => + // Repeated values for dictionary encoding. + Seq(Some("A"), Some("A"), None).toDF.repartition(1) + .write.parquet(path.getAbsolutePath) + val df = spark.read.parquet(path.getAbsolutePath) + checkAnswer(stripSparkFilterRapids(df.where("NOT (value <=> 'A')")), df) + } + } + } + + def stripSparkFilterRapids(df: DataFrame): DataFrame = { + val schema = df.schema + val withoutFilters = df.queryExecution.executedPlan.transform { + case GpuFilterExec(_, child) => child + } + spark.internalCreateDataFrame(withoutFilters.execute(), schema) + } +} diff --git a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala index 881d867366c..3ef7feccddc 100644 --- a/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala +++ b/tests/src/test/spark330/scala/org/apache/spark/sql/rapids/utils/RapidsTestSettings.scala @@ -84,7 +84,7 @@ class RapidsTestSettings extends BackendTestSettings { .exclude("struct with unannotated array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/11475")) .exclude("unannotated array of struct with unannotated array", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/11476")) enableSuite[RapidsParquetQuerySuite] - .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/11403")) + .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups", ADJUST_UT("fetches the CPU version of Execution Plan instead of the GPU version.")) .exclude("SPARK-34212 Parquet should read decimals correctly", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/11433")) enableSuite[RapidsParquetRebaseDatetimeSuite] .exclude("SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps", KNOWN_ISSUE("https://github.com/NVIDIA/spark-rapids/issues/11599"))