Add parenthesis to check filter

soda/core/soda/execution/metric/numeric_query_metric.py

@@ -323,8 +323,8 @@ def create_failed_rows_sample_query(self) -> SampleQuery | None:
  where_clauses.append(invalid_condition)
 
  if self.filter:
- where_clauses.append(self.filter)
- passing_where_clauses.append(self.filter)
+ where_clauses.append(f"({self.filter})")
+ passing_where_clauses.append(f"({self.filter})")
 
  where_sql = " AND ".join(where_clauses)
  passing_where_sql = " AND ".join(passing_where_clauses)

soda/core/soda/execution/query/duplicates_query.py

@@ -23,7 +23,7 @@ def __init__(self, partition: "Partition", metric: "Metric"):
  values_filter_clauses.append(resolved_partition_filter)
 
  if metric.filter:
- values_filter_clauses.append(metric.filter)
+ values_filter_clauses.append(f"({metric.filter})")
 
  values_filter = " \n AND ".join(values_filter_clauses)
 

soda/core/tests/data_source/test_duplicates.py

@@ -69,4 +69,4 @@ def test_duplicates_with_filter(data_source_fixture: DataSourceFixture):
  scan.execute()
 
  scan.assert_all_checks_pass()
- scan.assert_log("AND country = 'NL'")
+ scan.assert_log("AND (country = 'NL')")

soda/core/tests/data_source/test_samples.py

@@ -479,3 +479,81 @@ def test_sample_with_multiple_value_condition(data_source_fixture: DataSourceFix
 
  failed_ids = [sample[0] for sample in scan._configuration.sampler.samples[0].rows]
  assert sorted(failed_ids) == sorted(["ID5", "ID7"])
+
+
+@pytest.mark.parametrize(
+ "check,sample_count,samples",
+ [
+ pytest.param(
+ """
+ - missing_count(pct) = 1:
+ missing values: [No value, N/A, error]
+ filter: cst_size IS NOT NULL OR cst_size_txt IS NOT NULL
+ """,
+ 1,
+ ["ID7"],
+ ),
+ pytest.param(
+ """
+ - missing_percent(pct) < 20:
+ missing values: [No value, N/A, error]
+ filter: cst_size IS NOT NULL or cst_size_txt IS NOT NULL
+ """,
+ 1,
+ ["ID7"],
+ ),
+ pytest.param(
+ """
+ - invalid_count(cat) < 3:
+ valid values: ['HIGH']
+ filter: country = 'BE' OR country = 'NL'
+ """,
+ 2,
+ ["ID3", "ID4"],
+ ),
+ pytest.param(
+ """
+ - invalid_percent(cat) < 30:
+ valid values: ['HIGH']
+ filter: country = 'BE' OR country = 'NL'
+ """,
+ 2,
+ ["ID3", "ID4"],
+ ),
+ pytest.param(
+ """
+ - duplicate_count(cat) = 1:
+ filter: country = 'BE' OR country = 'NL'
+ """,
+ 3,
+ ["ID1", "ID2", None],
+ ),
+ pytest.param(
+ """
+ - duplicate_percent(cat) < 20:
+ filter: country = 'BE' OR country = 'NL'
+ """,
+ 3,
+ ["ID1", "ID2", None],
+ ),
+ ],
+)
+def test_missing_with_check_and_dataset_filter(data_source_fixture: DataSourceFixture, check, sample_count, samples):
+ table_name = data_source_fixture.ensure_test_table(customers_test_table)
+ scan = data_source_fixture.create_test_scan()
+ mock_soda_cloud = scan.enable_mock_soda_cloud()
+ scan.enable_mock_sampler()
+ scan.add_sodacl_yaml_str(
+ f"""
+ filter {table_name} [not_null_id]:
+ where: id IS NOT NULL
+
+ checks for {table_name} [not_null_id]:
+ {check}
+ """
+ )
+ scan.execute()
+ scan.assert_all_checks_pass()
+ assert mock_soda_cloud.find_failed_rows_line_count(0) == sample_count
+ failed_ids = [sample[0] for sample in scan._configuration.sampler.samples[0].rows]
+ assert set(failed_ids) == set(samples)