Skip to content

Commit

Permalink
Fix a json test for non utc time zone
Browse files Browse the repository at this point in the history
Signed-off-by: Robert (Bobby) Evans <[email protected]>
  • Loading branch information
revans2 committed Sep 18, 2024
1 parent 2589976 commit 394d3ae
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 12 deletions.
3 changes: 0 additions & 3 deletions integration_tests/src/main/python/json_matrix_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@

from asserts import *
from data_gen import *
from conftest import is_not_utc
from datetime import timezone
from conftest import is_databricks_runtime
from marks import approximate_float, allow_non_gpu, ignore_order, datagen_overrides
from spark_session import *

Expand Down
16 changes: 7 additions & 9 deletions integration_tests/src/main/python/json_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@

TEXT_INPUT_EXEC='FileSourceScanExec'

# allow non gpu when time zone is non-UTC because of https://github.com/NVIDIA/spark-rapids/issues/9653'
non_utc_file_source_scan_allow = ['FileSourceScanExec'] if is_not_utc() else []

non_utc_project_allow = ['ProjectExec'] if is_not_utc() else []


json_supported_gens = [
# Spark does not escape '\r' or '\n' even though it uses it to mark end of record
# This would require multiLine reads to work correctly, so we avoid these chars
Expand Down Expand Up @@ -378,7 +384,7 @@ def test_basic_json_read(std_input_path, filename, schema, read_func, allow_non_
'false'
])
@pytest.mark.parametrize('ansi_enabled', ["true", "false"])
@allow_non_gpu(TEXT_INPUT_EXEC, *not_utc_allow_for_test_json_scan)
@allow_non_gpu(TEXT_INPUT_EXEC, *non_utc_project_allow)
@pytest.mark.parametrize('date_format', [None, 'yyyy-MM-dd'])
def test_basic_from_json(std_input_path, filename, schema, allow_non_numeric_numbers, \
allow_numeric_leading_zeros, ansi_enabled, date_format):
Expand Down Expand Up @@ -582,11 +588,6 @@ def test_json_read_invalid_dates(std_input_path, filename, schema, read_func, an
else:
assert_gpu_and_cpu_are_equal_collect(f, conf=updated_conf)

# allow non gpu when time zone is non-UTC because of https://github.com/NVIDIA/spark-rapids/issues/9653'
non_utc_file_source_scan_allow = ['FileSourceScanExec'] if is_not_utc() else []

non_utc_project_allow = ['ProjectExec'] if is_not_utc() else []

@approximate_float
@pytest.mark.parametrize('filename', [
'timestamps.json',
Expand Down Expand Up @@ -818,9 +819,6 @@ def test_from_json_struct_date_fallback_non_default_format(date_gen, date_format
'ProjectExec',
conf=conf)

# allow non gpu when time zone is non-UTC because of https://github.com/NVIDIA/spark-rapids/issues/9653'
non_utc_project_allow = ['ProjectExec'] if is_not_utc() else []

@pytest.mark.parametrize('timestamp_gen', [
# "yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]"
"\"" + optional_whitespace_regex + "[1-8]{1}[0-9]{3}-[0-3]{1,2}-[0-3]{1,2}T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}(\\.[0-9]{1,6})?Z?" + optional_whitespace_regex + "\"",
Expand Down

0 comments on commit 394d3ae

Please sign in to comment.