From 6ad7e13b2803fafc4bfe538f9c011a4e5cace0f4 Mon Sep 17 00:00:00 2001 From: Aleksandra Niezgoda Date: Mon, 21 Aug 2023 10:00:03 +0200 Subject: [PATCH 1/6] add string sensors --- ...ntSensorParametersSpecIntegrationTest.java | 121 ++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 121 ++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 127 ++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 121 ++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 128 +++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 127 ++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 128 +++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 121 ++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 121 ++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 121 ++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 121 ++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 155 ++++++++++++++++++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 22 +++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 21 +++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 39 +++++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 27 +++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 33 ++++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 39 +++++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 33 ++++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 23 +++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 26 +++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 23 +++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 24 +++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 42 +++++ 36 files changed, 1960 insertions(+) create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringInvalidEmailCountSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringInvalidIp4AddressCountSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchDateRegexPercentSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchNameRegexPercentSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchRegexPercentSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringNotMatchDateRegexCountSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringNotMatchRegexCountSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValidCountryCodePercentSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValueInSetPercentSensorParametersSpecIntegrationTest.java create mode 100644 home/sensors/column/strings/string_invalid_email_count/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_invalid_email_count/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_invalid_ip4_address_count/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_invalid_ip4_address_count/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_match_date_regex_percent/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_match_date_regex_percent/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_match_name_regex_percent/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_match_name_regex_percent/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_match_regex_percent/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_match_regex_percent/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_not_match_date_regex_count/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_not_match_date_regex_count/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_not_match_regex_count/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_not_match_regex_count/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_surrounded_by_whitespace_count/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_surrounded_by_whitespace_count/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_surrounded_by_whitespace_percent/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_surrounded_by_whitespace_percent/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_valid_country_code_percent/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_valid_country_code_percent/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_valid_currency_code_percent/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_valid_currency_code_percent/oracle.sql.jinja2 create mode 100644 home/sensors/column/strings/string_value_in_set_percent/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/strings/string_value_in_set_percent/oracle.sql.jinja2 diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringInvalidEmailCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringInvalidEmailCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..2ec65942ec --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringInvalidEmailCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringInvalidEmailCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringInvalidEmailCountSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringInvalidEmailCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringInvalidEmailCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringInvalidEmailCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.string_test_data, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringInvalidEmailCountSensorParametersSpec(); + this.checkSpec = new ColumnStringInvalidEmailCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "email", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(22d, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(22d, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(22d, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(4d, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(22d, resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringInvalidIp4AddressCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringInvalidIp4AddressCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..18f79774f9 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringInvalidIp4AddressCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringInvalidIp4AddressCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringInvalidIp4AddressCountSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringInvalidIp4AddressCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringInvalidIp4AddressCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringInvalidIp4AddressCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.ip4_test, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringInvalidIp4AddressCountSensorParametersSpec(); + this.checkSpec = new ColumnStringInvalidIp4AddressCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "ip4", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(5, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "ip4", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(5, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "ip4", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(5, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "ip4", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(6, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(1, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "ip4", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(6, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(1, (double) resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchDateRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchDateRegexPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..5fc27c6f3c --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchDateRegexPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,127 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringMatchDateRegexPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringMatchDateRegexPercentSensorParametersSpec; +import com.dqops.sensors.column.strings.StringsBuiltInDateFormats; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringMatchDateRegexPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringMatchDateRegexPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringMatchDateRegexPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringMatchDateRegexPercentSensorParametersSpec(); + this.checkSpec = new ColumnStringMatchDateRegexPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + this.sut.setDateFormats(StringsBuiltInDateFormats.ISO8601); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "date_iso", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(100.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + this.sut.setDateFormats(StringsBuiltInDateFormats.ISO8601); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "date_iso", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(100.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + this.sut.setDateFormats(StringsBuiltInDateFormats.ISO8601); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "date_iso", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(100.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setDateFormats(StringsBuiltInDateFormats.ISO8601); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "date_iso", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(100.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setDateFormats(StringsBuiltInDateFormats.ISO8601); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "date_iso", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(100.0, (float) resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchNameRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchNameRegexPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..d5255ede21 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchNameRegexPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringMatchNameRegexPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringMatchNameRegexPercentSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringMatchNameRegexPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringMatchNameRegexPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringMatchNameRegexPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.full_name_test, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringMatchNameRegexPercentSensorParametersSpec(); + this.checkSpec = new ColumnStringMatchNameRegexPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "full_name", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(70.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "full_name", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(70.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "full_name", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(70.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "full_name", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(6, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "full_name", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(6, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, (float) resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchRegexPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchRegexPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..e81e4f11ca --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringMatchRegexPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,128 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringMatchRegexPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringMatchRegexPercentSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringMatchRegexPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringMatchRegexPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringMatchRegexPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.string_test_data, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringMatchRegexPercentSensorParametersSpec(); + this.checkSpec = new ColumnStringMatchRegexPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "email", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(40.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(40.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(40.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(83.333, (float) resultTable.column(0).get(0), 3); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(40.0, (float) resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringNotMatchDateRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringNotMatchDateRegexCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..12957b51be --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringNotMatchDateRegexCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,127 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringNotMatchDateRegexCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringNotMatchDateRegexCountSensorParametersSpec; +import com.dqops.sensors.column.strings.StringsBuiltInDateFormats; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringNotMatchDateRegexCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringNotMatchDateRegexCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringNotMatchDateRegexCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringNotMatchDateRegexCountSensorParametersSpec(); + this.checkSpec = new ColumnStringNotMatchDateRegexCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + this.sut.setDateFormats(StringsBuiltInDateFormats.ISO8601); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "date_iso", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + this.sut.setDateFormats(StringsBuiltInDateFormats.ISO8601); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "date_iso", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + this.sut.setDateFormats(StringsBuiltInDateFormats.ISO8601); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "date_iso", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setDateFormats(StringsBuiltInDateFormats.ISO8601); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "date_iso", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setDateFormats(StringsBuiltInDateFormats.ISO8601); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "date_iso", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0, (float) resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringNotMatchRegexCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringNotMatchRegexCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..767ec0a0b9 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringNotMatchRegexCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,128 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringNotMatchRegexCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringNotMatchRegexCountSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringNotMatchRegexCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringNotMatchRegexCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringNotMatchRegexCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.string_test_data, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringNotMatchRegexCountSensorParametersSpec(); + this.checkSpec = new ColumnStringNotMatchRegexCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "email", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(18, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(18, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(18, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(1, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setRegex("^[A-Za-z_]+[A-Za-z0-9._]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,4}$"); + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "email", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(18, (float) resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..c59fab3f87 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringSurroundedByWhitespaceCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringSurroundedByWhitespaceCountSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringSurroundedByWhitespaceCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringSurroundedByWhitespaceCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringSurroundedByWhitespaceCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.string_test_data, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringSurroundedByWhitespaceCountSensorParametersSpec(); + this.checkSpec = new ColumnStringSurroundedByWhitespaceCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0, (double) resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..47adb093e4 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringSurroundedByWhitespacePercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringSurroundedByWhitespacePercentSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringSurroundedByWhitespacePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringSurroundedByWhitespacePercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringSurroundedByWhitespacePercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.string_test_data, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringSurroundedByWhitespacePercentSensorParametersSpec(); + this.checkSpec = new ColumnStringSurroundedByWhitespacePercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "surrounded_by_whitespace", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, (float) resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValidCountryCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValidCountryCodePercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..9fbe54f755 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValidCountryCodePercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringValidCountryCodePercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringValidCountryCodePercentSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringValidCountryCodePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringValidCountryCodePercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringValidCountryCodePercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.nulls_and_uniqueness, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringValidCountryCodePercentSensorParametersSpec(); + this.checkSpec = new ColumnStringValidCountryCodePercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "valid_country_code", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(72.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "valid_country_code", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(72.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "valid_country_code", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(72.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "valid_country_code", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(100.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "valid_country_code", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(72.0, (float) resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..5053b93dcb --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValidCurrencyCodePercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringValidCurrencyCodePercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringValidCurrencyCodePercentSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnStringsStringValidCurrencyCodePercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringValidCurrencyCodePercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringValidCurrencyCodePercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.nulls_and_uniqueness, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringValidCurrencyCodePercentSensorParametersSpec(); + this.checkSpec = new ColumnStringValidCurrencyCodePercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "valid_currency_code", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(64.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "valid_currency_code", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(64.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "valid_currency_code", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(64.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "valid_currency_code", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(100.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "valid_currency_code", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(64.0, (float) resultTable.column(0).get(0)); + } +} diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValueInSetPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValueInSetPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..324c17941a --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/strings/OracleColumnStringsStringValueInSetPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,155 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.strings; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.strings.ColumnStringValueInSetPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.strings.ColumnStringsStringValueInSetPercentSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + +import java.util.ArrayList; +import java.util.List; + +@SpringBootTest +public class OracleColumnStringsStringValueInSetPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnStringsStringValueInSetPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnStringValueInSetPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.test_data_values_in_set, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(sampleTableMetadata); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnStringsStringValueInSetPercentSensorParametersSpec(); + this.checkSpec = new ColumnStringValueInSetPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + List values = new ArrayList<>(); + this.sut.setExpectedValues(values); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "strings_with_numbers", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + List values = new ArrayList<>(); + values.add("e55e"); + values.add("a111a"); + values.add("d44d"); + values.add("c33c"); + values.add("b22b"); + this.sut.setExpectedValues(values); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "strings_with_numbers", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(100.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + List values = new ArrayList<>(); + values.add("e55e"); + values.add("a111a"); + values.add("d44d"); + values.add("c33c"); + values.add("b22b"); + this.sut.setExpectedValues(values); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "strings_with_numbers", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(100.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + List values = new ArrayList<>(); + values.add("e55e"); + values.add("a111a"); + this.sut.setExpectedValues(values); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "strings_with_numbers", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(25, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(50.0, (float) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + List values = new ArrayList<>(); + values.add("e55e"); + values.add("a111a"); + values.add("d44d"); + values.add("c33c"); + values.add("b22b"); + this.sut.setExpectedValues(values); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "strings_with_numbers", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(100.0, (float) resultTable.column(0).get(0)); + } +} diff --git a/home/sensors/column/strings/string_invalid_email_count/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_invalid_email_count/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_invalid_email_count/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_invalid_email_count/oracle.sql.jinja2 b/home/sensors/column/strings/string_invalid_email_count/oracle.sql.jinja2 new file mode 100644 index 0000000000..04cb065c92 --- /dev/null +++ b/home/sensors/column/strings/string_invalid_email_count/oracle.sql.jinja2 @@ -0,0 +1,22 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} +SELECT + SUM( + CASE + WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[A-Za-z]+[A-Za-z0-9.]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4}$') + THEN 0 + ELSE 1 + END + ) AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} +FROM( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table +{{- lib.render_group_by() -}} +{{- lib.render_order_by() -}} + + diff --git a/home/sensors/column/strings/string_invalid_ip4_address_count/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_invalid_ip4_address_count/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_invalid_ip4_address_count/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_invalid_ip4_address_count/oracle.sql.jinja2 b/home/sensors/column/strings/string_invalid_ip4_address_count/oracle.sql.jinja2 new file mode 100644 index 0000000000..c892d3666c --- /dev/null +++ b/home/sensors/column/strings/string_invalid_ip4_address_count/oracle.sql.jinja2 @@ -0,0 +1,21 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} +SELECT + SUM( + CASE + WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])[.]){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[0-9][0-9]|[0-9])$') + THEN 0 + ELSE 1 + END + ) AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} + FROM ( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }} + ) analyzed_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} \ No newline at end of file diff --git a/home/sensors/column/strings/string_match_date_regex_percent/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_match_date_regex_percent/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_match_date_regex_percent/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_match_date_regex_percent/oracle.sql.jinja2 b/home/sensors/column/strings/string_match_date_regex_percent/oracle.sql.jinja2 new file mode 100644 index 0000000000..4843f08aae --- /dev/null +++ b/home/sensors/column/strings/string_match_date_regex_percent/oracle.sql.jinja2 @@ -0,0 +1,39 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} + +{% macro render_date_formats(date_formats) %} + {%- if date_formats == 'YYYY-MM-DD'-%} + 'YYYY-MM-DD' + {%- elif date_formats == 'MM/DD/YYYY' -%} + 'MM/DD/YYYY' + {%- elif date_formats == 'DD/MM/YYYY' -%} + 'DD/MM/YYYY' + {%- elif date_formats == 'YYYY/MM/DD'-%} + 'YYYY/MM/DD' + {%- elif date_formats == 'Month D, YYYY'-%} + 'MON DD, YYYY' + {%- endif -%} +{% endmacro -%} + +SELECT + CASE + WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN NULL + ELSE 100.0 * SUM( + CASE + WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{render_date_formats(parameters.date_formats)}}) IS NOT NULL + THEN 1 + ELSE 0 + END + ) / COUNT(*) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} + FROM ( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }} + ) analyzed_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} \ No newline at end of file diff --git a/home/sensors/column/strings/string_match_name_regex_percent/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_match_name_regex_percent/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_match_name_regex_percent/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_match_name_regex_percent/oracle.sql.jinja2 b/home/sensors/column/strings/string_match_name_regex_percent/oracle.sql.jinja2 new file mode 100644 index 0000000000..ab3aa70446 --- /dev/null +++ b/home/sensors/column/strings/string_match_name_regex_percent/oracle.sql.jinja2 @@ -0,0 +1,27 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} +SELECT + CASE + WHEN COUNT(*) = 0 THEN 100.0 + ELSE 100.0 * SUM( + CASE + WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, '^[[:alpha:] .''-]{2,}(\s.[[:alpha:] .''-]{2,})+$') + THEN 1 + ELSE 0 + END + ) / COUNT(*) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} +FROM ( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }} +) analyzed_table +{{- lib.render_group_by() -}} +{{- lib.render_order_by() -}} + + + diff --git a/home/sensors/column/strings/string_match_regex_percent/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_match_regex_percent/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_match_regex_percent/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_match_regex_percent/oracle.sql.jinja2 b/home/sensors/column/strings/string_match_regex_percent/oracle.sql.jinja2 new file mode 100644 index 0000000000..c7acc035ea --- /dev/null +++ b/home/sensors/column/strings/string_match_regex_percent/oracle.sql.jinja2 @@ -0,0 +1,33 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} + +{%- macro make_text_constant(string) -%} + {{ '\'' }}{{ string | replace('\'', '\'\'') }}{{ '\'' }} +{%- endmacro -%} + +{%- macro render_regex(regex) -%} + {{ make_text_constant(regex) }} +{%- endmacro -%} + +SELECT + CASE + WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN NULL + ELSE 100.0 * SUM( + CASE + WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ render_regex(parameters.regex) }}) + THEN 1 + ELSE 0 + END + ) / COUNT(*) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} + FROM ( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }} + ) analyzed_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} \ No newline at end of file diff --git a/home/sensors/column/strings/string_not_match_date_regex_count/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_not_match_date_regex_count/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_not_match_date_regex_count/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_not_match_date_regex_count/oracle.sql.jinja2 b/home/sensors/column/strings/string_not_match_date_regex_count/oracle.sql.jinja2 new file mode 100644 index 0000000000..29fcd77d41 --- /dev/null +++ b/home/sensors/column/strings/string_not_match_date_regex_count/oracle.sql.jinja2 @@ -0,0 +1,39 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} + +{% macro render_date_formats(date_formats) %} + {%- if date_formats == 'YYYY-MM-DD'-%} + 'YYYY-MM-DD' + {%- elif date_formats == 'MM/DD/YYYY' -%} + 'MM/DD/YYYY' + {%- elif date_formats == 'DD/MM/YYYY' -%} + 'DD/MM/YYYY' + {%- elif date_formats == 'YYYY/MM/DD'-%} + 'YYYY/MM/DD' + {%- elif date_formats == 'Month D, YYYY'-%} + 'MON DD, YYYY' + {%- endif -%} +{% endmacro -%} + +SELECT + CASE + WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN NULL + ELSE SUM( + CASE + WHEN TO_DATE({{ lib.render_target_column('analyzed_table') }}, {{render_date_formats(parameters.date_formats)}}) IS NULL + THEN 1 + ELSE 0 + END + ) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} + FROM ( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }} + ) analyzed_table + {{- lib.render_group_by() -}} + {{- lib.render_order_by() -}} \ No newline at end of file diff --git a/home/sensors/column/strings/string_not_match_regex_count/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_not_match_regex_count/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_not_match_regex_count/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_not_match_regex_count/oracle.sql.jinja2 b/home/sensors/column/strings/string_not_match_regex_count/oracle.sql.jinja2 new file mode 100644 index 0000000000..5d9293ad24 --- /dev/null +++ b/home/sensors/column/strings/string_not_match_regex_count/oracle.sql.jinja2 @@ -0,0 +1,33 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} + +{%- macro make_text_constant(string) -%} + {{ '\'' }}{{ string | replace('\'', '\'\'') }}{{ '\'' }} +{%- endmacro %} + +{%- macro render_regex(regex) -%} + {{ make_text_constant(regex) }} +{%- endmacro -%} + +SELECT + CASE + WHEN COUNT({{ lib.render_target_column('analyzed_table') }}) = 0 THEN NULL + ELSE SUM( + CASE + WHEN REGEXP_LIKE({{ lib.render_target_column('analyzed_table') }}, {{ render_regex(parameters.regex) }}) + THEN 0 + ELSE 1 + END + ) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} +FROM ( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }} +) analyzed_table +{{- lib.render_group_by() -}} +{{- lib.render_order_by() -}} \ No newline at end of file diff --git a/home/sensors/column/strings/string_surrounded_by_whitespace_count/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_surrounded_by_whitespace_count/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_surrounded_by_whitespace_count/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_surrounded_by_whitespace_count/oracle.sql.jinja2 b/home/sensors/column/strings/string_surrounded_by_whitespace_count/oracle.sql.jinja2 new file mode 100644 index 0000000000..537e007dd9 --- /dev/null +++ b/home/sensors/column/strings/string_surrounded_by_whitespace_count/oracle.sql.jinja2 @@ -0,0 +1,23 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} + +SELECT + SUM( + CASE + WHEN ({{ lib.render_target_column('analyzed_table')}}) IS NOT NULL + AND TRIM({{ lib.render_target_column('analyzed_table') }}) <> '' + AND ({{ lib.render_target_column('analyzed_table') }}) <> TRIM({{ lib.render_target_column('analyzed_table') }}) + THEN 1 + ELSE 0 + END + ) AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} +FROM( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table +{{- lib.render_group_by() -}} +{{- lib.render_order_by() -}} \ No newline at end of file diff --git a/home/sensors/column/strings/string_surrounded_by_whitespace_percent/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_surrounded_by_whitespace_percent/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_surrounded_by_whitespace_percent/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_surrounded_by_whitespace_percent/oracle.sql.jinja2 b/home/sensors/column/strings/string_surrounded_by_whitespace_percent/oracle.sql.jinja2 new file mode 100644 index 0000000000..5901c1f326 --- /dev/null +++ b/home/sensors/column/strings/string_surrounded_by_whitespace_percent/oracle.sql.jinja2 @@ -0,0 +1,26 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} + +SELECT + CASE + WHEN COUNT(*) = 0 THEN 100.0 + ELSE 100.0 * SUM( + CASE + WHEN ({{ lib.render_target_column('analyzed_table')}}) IS NOT NULL + AND TRIM({{ lib.render_target_column('analyzed_table') }}) <> '' + AND ({{ lib.render_target_column('analyzed_table') }}) <> TRIM({{ lib.render_target_column('analyzed_table') }}) + THEN 1 + ELSE 0 + END + ) / COUNT(*) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} +FROM( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table +{{- lib.render_group_by() -}} +{{- lib.render_order_by() -}} \ No newline at end of file diff --git a/home/sensors/column/strings/string_valid_country_code_percent/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_valid_country_code_percent/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_valid_country_code_percent/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_valid_country_code_percent/oracle.sql.jinja2 b/home/sensors/column/strings/string_valid_country_code_percent/oracle.sql.jinja2 new file mode 100644 index 0000000000..10365acbd9 --- /dev/null +++ b/home/sensors/column/strings/string_valid_country_code_percent/oracle.sql.jinja2 @@ -0,0 +1,23 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} +SELECT + CASE + WHEN COUNT(*) = 0 THEN 100.0 + ELSE 100.0 * SUM( + CASE + WHEN UPPER({{ lib.render_target_column('analyzed_table')}}) IN ('AF', 'AL', 'DZ', 'AS', 'AD', 'AO', 'AI', 'AQ', 'AG', 'AR', 'AM', 'AW', 'AU', 'AT', 'AZ', 'BS', 'BH', 'BD', 'BB', 'BY', 'BE', 'BZ', 'BJ', 'BM', 'BT', 'BO', 'BA', 'BW', 'BR', 'IO', 'VG', 'BN', 'BG', 'BF', 'BI', 'KH', 'CM', 'CA', 'CV', 'KY', 'CF', 'TD', 'CL', 'CN', 'CX', 'CC', 'CO', 'KM', 'CK', 'CR', 'HR', 'CU', 'CW', 'CY', 'CZ', 'CD', 'DK', 'DJ', 'DM', 'DO', 'TL', 'EC', 'EG', 'SV', 'GQ', 'ER', 'EE', 'ET', 'FK', 'FO', 'FJ', 'FI', 'FR', 'PF', 'GA', 'GM', 'GE', 'DE', 'GH', 'GI', 'GR', 'GL', 'GD', 'GU', 'GT', 'GG', 'GN', 'GW', 'GY', 'HT', 'HN', 'HK', 'HU', 'IS', 'IN', 'ID', 'IR', 'IQ', 'IE', 'IM', 'IL', 'IT', 'CI', 'JM', 'JP', 'JE', 'JO', 'KZ', 'KE', 'KI', 'XK', 'KW', 'KG', 'LA', 'LV', 'LB', 'LS', 'LR', 'LY', 'LI', 'LT', 'LU', 'MO', 'MK', 'MG', 'MW', 'MY', 'MV', 'ML', 'MT', 'MH', 'MR', 'MU', 'YT', 'MX', 'FM', 'MD', 'MC', 'MN', 'ME', 'MS', 'MA', 'MZ', 'MM', 'NA', 'NR', 'NP', 'NL', 'AN', 'NC', 'NZ', 'NI', 'NE', 'NG', 'NU', 'KP', 'MP', 'NO', 'OM', 'PK', 'PW', 'PS', 'PA', 'PG', 'PY', 'PE', 'PH', 'PN', 'PL', 'PT', 'PR', 'QA', 'CG', 'RE', 'RO', 'RU', 'RW', 'BL', 'SH', 'KN', 'LC', 'MF', 'PM', 'VC', 'WS', 'SM', 'ST', 'SA', 'SN', 'RS', 'SC', 'SL', 'SG', 'SX', 'SK', 'SI', 'SB', 'SO', 'ZA', 'KR', 'SS', 'ES', 'LK', 'SD', 'SR', 'SJ', 'SZ', 'SE', 'CH', 'SY', 'TW', 'TJ', 'TZ', 'TH', 'TG', 'TK', 'TO', 'TT', 'TN', 'TR', 'TM', 'TC', 'TV', 'VI', 'UG', 'UA', 'AE', 'GB', 'US', 'UY', 'UZ', 'VU', 'VA', 'VE', 'VN', 'WF', 'EH', 'YE', 'ZM', 'ZW') + THEN 1 + ELSE 0 + END + ) / COUNT(*) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} +FROM( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table +{{- lib.render_group_by() -}} +{{- lib.render_order_by() -}} \ No newline at end of file diff --git a/home/sensors/column/strings/string_valid_currency_code_percent/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_valid_currency_code_percent/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_valid_currency_code_percent/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_valid_currency_code_percent/oracle.sql.jinja2 b/home/sensors/column/strings/string_valid_currency_code_percent/oracle.sql.jinja2 new file mode 100644 index 0000000000..28bdef54ad --- /dev/null +++ b/home/sensors/column/strings/string_valid_currency_code_percent/oracle.sql.jinja2 @@ -0,0 +1,24 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} + +SELECT + CASE + WHEN COUNT(*) = 0 THEN 100.0 + ELSE 100.0 * SUM( + CASE + WHEN UPPER({{ lib.render_target_column('analyzed_table') }}) IN ('ALL', 'AFN', 'ARS', 'AWG', 'AUD', 'AZN', 'BSD', 'BBD', 'BYN', 'BZD', 'BMD', 'BOB', 'BAM', 'BWP', 'BGN', 'BRL', 'BND', 'KHR', 'CAD', 'KYD', 'CLP', 'CNY', 'COP', 'CRC', 'HRK', 'CUP', 'CZK', 'DKK', 'DOP', 'XCD', 'EGP', 'SVC', 'EUR', 'FKP', 'FJD', 'GHS', 'GIP', 'GTQ', 'GGP', 'GYD', 'HNL', 'HKD', 'HUF', 'ISK', 'INR', 'IDR', 'IRR', 'IMP', 'ILS', 'JMD', 'JPY', 'JEP', 'KZT', 'KPW', 'KRW', 'KGS', 'LAK', 'LBP', 'LRD', 'MKD', 'MYR', 'MUR', 'MXN', 'MNT', 'MZN', 'NAD', 'NPR', 'ANG', 'NZD', 'NIO', 'NGN', 'NOK', 'OMR', 'PKR', 'PAB', 'PYG', 'PEN', 'PHP', 'PLN', 'QAR', 'RON', 'RUB', 'SHP', 'SAR', 'RSD', 'SCR', 'SGD', 'SBD', 'SOS', 'ZAR', 'LKR', 'SEK', 'CHF', 'SRD', 'SYP', 'TWD', 'THB', 'TTD', 'TRY', 'TVD', 'UAH', 'AED', 'GBP', 'USD', 'UYU', 'UZS', 'VEF', 'VND', 'YER', 'ZWD', 'LEK', '؋', '$', 'Ƒ', '₼', 'BR', 'BZ$', '$B', 'KM', 'P', 'ЛВ', 'R$', '៛', '¥', '₡', 'KN', '₱', 'KČ', 'KR', 'RD$', '£', '€', '¢', 'Q', 'L', 'FT', '₹', 'RP', '﷼', '₪', 'J$', '₩', '₭', 'ДЕН', 'RM', '₨', '₮', 'د.إ', 'MT', 'C$', '₦', 'B/.', 'GS', 'S/.', 'ZŁ', 'LEI', 'ДИН.', 'S', 'R', 'NT$', '฿', 'TT$', '₺', '₴', '$U', 'BS', '₫', 'Z$') + THEN 1 + ELSE 0 + END + ) / COUNT(*) + END AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} +FROM( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table +{{- lib.render_group_by() -}} +{{- lib.render_order_by() -}} \ No newline at end of file diff --git a/home/sensors/column/strings/string_value_in_set_percent/oracle.dqoprovidersensor.yaml b/home/sensors/column/strings/string_value_in_set_percent/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..f7e46a0e7b --- /dev/null +++ b/home/sensors/column/strings/string_value_in_set_percent/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true \ No newline at end of file diff --git a/home/sensors/column/strings/string_value_in_set_percent/oracle.sql.jinja2 b/home/sensors/column/strings/string_value_in_set_percent/oracle.sql.jinja2 new file mode 100644 index 0000000000..ee66dcaa79 --- /dev/null +++ b/home/sensors/column/strings/string_value_in_set_percent/oracle.sql.jinja2 @@ -0,0 +1,42 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} + +{%- macro extract_in_list(values_list) -%} + {%- for i in values_list -%} + {%- if not loop.last -%} + {{lib.make_text_constant(i)}}{{", "}} + {%- else -%} + {{lib.make_text_constant(i)}} + {%- endif -%} + {%- endfor -%} +{% endmacro -%} + +{%- macro actual_value() -%} + {%- if 'expected_values' not in parameters or parameters.expected_values|length == 0 -%} + 0.0 + {%- else -%} + CASE + WHEN COUNT(*) = 0 THEN 100.0 + ELSE 100.0 * SUM( + CASE + WHEN {{ lib.render_target_column('analyzed_table') }} IN ({{ extract_in_list(parameters.expected_values) }}) + THEN 1 + ELSE 0 + END + ) / COUNT(*) + END + {%- endif -%} +{% endmacro -%} + +SELECT + {{ actual_value() }} AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} +FROM( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }}) analyzed_table +{{- lib.render_group_by() -}} +{{- lib.render_order_by() -}} \ No newline at end of file From 1e48909006c1fd985d84cee06312db34aaedec8b Mon Sep 17 00:00:00 2001 From: Aleksandra Niezgoda Date: Mon, 21 Aug 2023 13:45:25 +0200 Subject: [PATCH 2/6] add oracle integrity sensors --- ...ntSensorParametersSpecIntegrationTest.java | 139 ++++++++++++++++++ ...ntSensorParametersSpecIntegrationTest.java | 139 ++++++++++++++++++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 32 ++++ .../oracle.dqoprovidersensor.yaml | 8 + .../oracle.sql.jinja2 | 36 +++++ 6 files changed, 362 insertions(+) create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java create mode 100644 dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java create mode 100644 home/sensors/column/integrity/foreign_key_match_percent/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/integrity/foreign_key_match_percent/oracle.sql.jinja2 create mode 100644 home/sensors/column/integrity/foreign_key_not_match_count/oracle.dqoprovidersensor.yaml create mode 100644 home/sensors/column/integrity/foreign_key_not_match_count/oracle.sql.jinja2 diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..2d6edf0b94 --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest.java @@ -0,0 +1,139 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.integrity; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.integrity.ColumnIntegrityForeignKeyMatchPercentCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.integrity.ColumnIntegrityForeignKeyMatchPercentSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnIntegrityForeignKeyMatchPercentSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnIntegrityForeignKeyMatchPercentSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnIntegrityForeignKeyMatchPercentCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + private SampleTableMetadata sampleTableMetadataForeign; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.value_match_right_table, ProviderType.oracle); + this.sampleTableMetadataForeign = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.value_match_left_table, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(this.sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(this.sampleTableMetadataForeign); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnIntegrityForeignKeyMatchPercentSensorParametersSpec(); + this.checkSpec = new ColumnIntegrityForeignKeyMatchPercentCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + this.sut.setForeignTable(this.sampleTableMetadataForeign.getTableData().getHashedTableName()); + this.sut.setForeignColumn("primary_key"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "foreign_key", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(75.0, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + this.sut.setForeignTable(this.sampleTableMetadataForeign.getTableData().getHashedTableName()); + this.sut.setForeignColumn("primary_key"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "foreign_key", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(75.0, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + this.sut.setForeignTable(this.sampleTableMetadataForeign.getTableData().getHashedTableName()); + this.sut.setForeignColumn("primary_key"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "foreign_key", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(75.0, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setForeignTable(this.sampleTableMetadataForeign.getTableData().getHashedTableName()); + this.sut.setForeignColumn("primary_key"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "foreign_key", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(6, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setForeignTable(this.sampleTableMetadataForeign.getTableData().getHashedTableName()); + this.sut.setForeignColumn("primary_key"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "foreign_key", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(6, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(0.0, resultTable.column(0).get(0)); + } +} \ No newline at end of file diff --git a/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java new file mode 100644 index 0000000000..f35d7b0aaf --- /dev/null +++ b/dqops/src/integration-test/java/com/dqops/oracle/sensors/column/integrity/OracleColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest.java @@ -0,0 +1,139 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dqops.oracle.sensors.column.integrity; + +import com.dqops.checks.CheckTimeScale; +import com.dqops.checks.column.checkspecs.integrity.ColumnIntegrityForeignKeyNotMatchCountCheckSpec; +import com.dqops.connectors.ProviderType; +import com.dqops.execution.sensors.DataQualitySensorRunnerObjectMother; +import com.dqops.execution.sensors.SensorExecutionResult; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.execution.sensors.SensorExecutionRunParametersObjectMother; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContext; +import com.dqops.metadata.storage.localfiles.userhome.UserHomeContextObjectMother; +import com.dqops.oracle.BaseOracleIntegrationTest; +import com.dqops.sampledata.IntegrationTestSampleDataObjectMother; +import com.dqops.sampledata.SampleCsvFileNames; +import com.dqops.sampledata.SampleTableMetadata; +import com.dqops.sampledata.SampleTableMetadataObjectMother; +import com.dqops.sensors.column.integrity.ColumnIntegrityForeignKeyNotMatchCountSensorParametersSpec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import tech.tablesaw.api.Table; + + +@SpringBootTest +public class OracleColumnIntegrityForeignKeyNotMatchCountSensorParametersSpecIntegrationTest extends BaseOracleIntegrationTest { + private ColumnIntegrityForeignKeyNotMatchCountSensorParametersSpec sut; + private UserHomeContext userHomeContext; + private ColumnIntegrityForeignKeyNotMatchCountCheckSpec checkSpec; + private SampleTableMetadata sampleTableMetadata; + private SampleTableMetadata sampleTableMetadataForeign; + + @BeforeEach + void setUp() { + this.sampleTableMetadata = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.value_match_right_table, ProviderType.oracle); + this.sampleTableMetadataForeign = SampleTableMetadataObjectMother.createSampleTableMetadataForCsvFile(SampleCsvFileNames.value_match_left_table, ProviderType.oracle); + IntegrationTestSampleDataObjectMother.ensureTableExists(this.sampleTableMetadata); + IntegrationTestSampleDataObjectMother.ensureTableExists(this.sampleTableMetadataForeign); + this.userHomeContext = UserHomeContextObjectMother.createInMemoryFileHomeContextForSampleTable(sampleTableMetadata); + this.sut = new ColumnIntegrityForeignKeyNotMatchCountSensorParametersSpec(); + this.checkSpec = new ColumnIntegrityForeignKeyNotMatchCountCheckSpec(); + this.checkSpec.setParameters(this.sut); + } + + @Test + void runSensor_whenSensorExecutedProfiling_thenReturnsValues() { + this.sut.setForeignTable(this.sampleTableMetadataForeign.getTableData().getHashedTableName()); + this.sut.setForeignColumn("primary_key"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForProfilingCheck( + sampleTableMetadata, "foreign_key", this.checkSpec); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(5, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringDaily_thenReturnsValues() { + this.sut.setForeignTable(this.sampleTableMetadataForeign.getTableData().getHashedTableName()); + this.sut.setForeignColumn("primary_key"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "foreign_key", this.checkSpec, CheckTimeScale.daily); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(5, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedRecurringMonthly_thenReturnsValues() { + this.sut.setForeignTable(this.sampleTableMetadataForeign.getTableData().getHashedTableName()); + this.sut.setForeignColumn("primary_key"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForRecurringCheck( + sampleTableMetadata, "foreign_key", this.checkSpec, CheckTimeScale.monthly); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(1, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(5, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedDaily_thenReturnsValues() { + this.sut.setForeignTable(this.sampleTableMetadataForeign.getTableData().getHashedTableName()); + this.sut.setForeignColumn("primary_key"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "foreign_key", this.checkSpec, CheckTimeScale.daily,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(6, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(1, (double) resultTable.column(0).get(0)); + } + + @Test + void runSensor_whenSensorExecutedPartitionedMonthly_thenReturnsValues() { + this.sut.setForeignTable(this.sampleTableMetadataForeign.getTableData().getHashedTableName()); + this.sut.setForeignColumn("primary_key"); + + SensorExecutionRunParameters runParameters = SensorExecutionRunParametersObjectMother.createForTableColumnForPartitionedCheck( + sampleTableMetadata, "foreign_key", this.checkSpec, CheckTimeScale.monthly,"date"); + + SensorExecutionResult sensorResult = DataQualitySensorRunnerObjectMother.executeSensor(this.userHomeContext, runParameters); + + Table resultTable = sensorResult.getResultTable(); + Assertions.assertEquals(6, resultTable.rowCount()); + Assertions.assertEquals("actual_value", resultTable.column(0).name()); + Assertions.assertEquals(1, (double) resultTable.column(0).get(0)); + } +} \ No newline at end of file diff --git a/home/sensors/column/integrity/foreign_key_match_percent/oracle.dqoprovidersensor.yaml b/home/sensors/column/integrity/foreign_key_match_percent/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..50693e9bb5 --- /dev/null +++ b/home/sensors/column/integrity/foreign_key_match_percent/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true diff --git a/home/sensors/column/integrity/foreign_key_match_percent/oracle.sql.jinja2 b/home/sensors/column/integrity/foreign_key_match_percent/oracle.sql.jinja2 new file mode 100644 index 0000000000..aedc46bb46 --- /dev/null +++ b/home/sensors/column/integrity/foreign_key_match_percent/oracle.sql.jinja2 @@ -0,0 +1,32 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} + +{%- macro render_foreign_table(foreign_table) -%} +{%- if foreign_table.find(".") < 0 -%} + {{- lib.quote_identifier(foreign_table) -}} +{%- else -%} + {{ foreign_table }} +{%- endif -%} +{%- endmacro -%} + +SELECT + 100.0 * SUM( + CASE + WHEN foreign_table.{{ lib.quote_identifier(parameters.foreign_column) }} IS NULL AND {{ lib.render_target_column('analyzed_table')}} IS NOT NULL + THEN 0 + ELSE 1 + END + ) / COUNT(*) AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} +FROM ( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }} +) analyzed_table +LEFT OUTER JOIN {{ render_foreign_table(parameters.foreign_table) }} foreign_table +ON {{ lib.render_target_column('analyzed_table')}} = foreign_table.{{ lib.quote_identifier(parameters.foreign_column) }} +{{- lib.render_group_by() -}} +{{- lib.render_order_by() -}} \ No newline at end of file diff --git a/home/sensors/column/integrity/foreign_key_not_match_count/oracle.dqoprovidersensor.yaml b/home/sensors/column/integrity/foreign_key_not_match_count/oracle.dqoprovidersensor.yaml new file mode 100644 index 0000000000..50693e9bb5 --- /dev/null +++ b/home/sensors/column/integrity/foreign_key_not_match_count/oracle.dqoprovidersensor.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://cloud.dqo.ai/dqo-yaml-schema/ProviderSensorYaml-schema.json +apiVersion: dqo/v1 +kind: provider_sensor +spec: + type: sql_template + java_class_name: com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner + supports_grouping: true + supports_partitioned_checks: true diff --git a/home/sensors/column/integrity/foreign_key_not_match_count/oracle.sql.jinja2 b/home/sensors/column/integrity/foreign_key_not_match_count/oracle.sql.jinja2 new file mode 100644 index 0000000000..43320f6d50 --- /dev/null +++ b/home/sensors/column/integrity/foreign_key_not_match_count/oracle.sql.jinja2 @@ -0,0 +1,36 @@ +{% import '/dialects/oracle.sql.jinja2' as lib with context -%} + +{%- macro render_target_column(table_alias_prefix = '') -%} + {{ table_alias_prefix }}.{{ lib.quote_identifier(column_name) }} +{%- endmacro %} + +{%- macro render_foreign_table(foreign_table) -%} +{%- if foreign_table.find(".") < 0 -%} + {{- lib.quote_identifier(foreign_table) -}} +{%- else -%} + {{ foreign_table }} +{%- endif -%} +{%- endmacro -%} + +SELECT + SUM( + CASE + WHEN foreign_table.{{ lib.quote_identifier(parameters.foreign_column) }} IS NULL AND {{ render_target_column('analyzed_table')}} IS NOT NULL + THEN 1 + ELSE 0 + END + ) AS actual_value + {{- lib.render_data_grouping_projections_reference('analyzed_table') }} + {{- lib.render_time_dimension_projection_reference('analyzed_table') }} +FROM ( + SELECT + original_table.* + {{- lib.render_data_grouping_projections('original_table') }} + {{- lib.render_time_dimension_projection('original_table') }} + FROM {{ lib.render_target_table() }} original_table + {{- lib.render_where_clause(table_alias_prefix='original_table') }} +) analyzed_table +LEFT OUTER JOIN {{ render_foreign_table(parameters.foreign_table) }} foreign_table +ON {{ lib.render_target_column('analyzed_table')}} = foreign_table.{{ lib.quote_identifier(parameters.foreign_column) }} +{{- lib.render_group_by() -}} +{{- lib.render_order_by() -}} \ No newline at end of file From 9b314ea3c768a711a3906a9b0dd9659bf929d862 Mon Sep 17 00:00:00 2001 From: Piotr Czarnas Date: Mon, 21 Aug 2023 16:37:02 +0200 Subject: [PATCH 3/6] Table availability check uses a rule that can count recent failures. --- .../TableAvailabilityCheckSpec.java | 2 +- .../models/CheckResultsOverviewDataModel.java | 8 +- .../ruleeval/RuleEvaluationServiceImpl.java | 17 +++ .../execution/rules/RuleExecutionResult.java | 17 +++ .../AbstractGroupedSensorExecutor.java | 34 +++-- .../sensors/runners/AbstractSensorRunner.java | 87 +----------- .../runners/GenericSensorResultsFactory.java | 133 ++++++++++++++++++ .../JinjaSqlTemplateSensorRunner.java | 4 +- .../ColumnColumnExistsSensorRunner.java | 16 ++- .../ColumnColumnTypeHashSensorRunner.java | 22 ++- ...TableAvailabilitySensorParametersSpec.java | 2 +- .../TableAvailabilitySensorRunner.java | 23 ++- .../schema/TableColumnCountSensorRunner.java | 16 ++- ...ableColumnListOrderedHashSensorRunner.java | 16 ++- ...leColumnListUnorderedHashSensorRunner.java | 16 ++- .../TableColumnTypesHashSensorRunner.java | 16 ++- .../MaxFailuresRuleParametersSpecTests.java | 50 +++---- ...sorDocumentationModelFactoryImplTests.java | 2 +- home/rules/comparison/max_failures.py | 14 +- .../comparison/value_changed.dqorule.yaml | 2 +- home/rules/comparison/value_changed.py | 10 +- 21 files changed, 332 insertions(+), 175 deletions(-) create mode 100644 dqops/src/main/java/com/dqops/execution/sensors/runners/GenericSensorResultsFactory.java diff --git a/dqops/src/main/java/com/dqops/checks/table/checkspecs/availability/TableAvailabilityCheckSpec.java b/dqops/src/main/java/com/dqops/checks/table/checkspecs/availability/TableAvailabilityCheckSpec.java index 2282612c1a..b65058d2ba 100644 --- a/dqops/src/main/java/com/dqops/checks/table/checkspecs/availability/TableAvailabilityCheckSpec.java +++ b/dqops/src/main/java/com/dqops/checks/table/checkspecs/availability/TableAvailabilityCheckSpec.java @@ -34,7 +34,7 @@ import java.util.Objects; /** - * Table level check that verifies that a query can be executed on a table and that the server does not return errors, that the table exists, and that there are accesses to it. + * Table level check that verifies that a query can be executed on a table and that the server does not return errors, that the table exists, and that the table is accessible (queryable). */ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) diff --git a/dqops/src/main/java/com/dqops/data/checkresults/services/models/CheckResultsOverviewDataModel.java b/dqops/src/main/java/com/dqops/data/checkresults/services/models/CheckResultsOverviewDataModel.java index fd99073203..f29fe39798 100644 --- a/dqops/src/main/java/com/dqops/data/checkresults/services/models/CheckResultsOverviewDataModel.java +++ b/dqops/src/main/java/com/dqops/data/checkresults/services/models/CheckResultsOverviewDataModel.java @@ -105,8 +105,8 @@ public void appendResult(LocalDateTime timePeriod, } if (this.timePeriodsUtc.get(this.timePeriodsUtc.size() - 1).equals(timePeriodUtc)) { - if (severity > this.statuses.get(this.statuses.size() - 1).getSeverity()) { - // another result with a higher severity, replacing the current one, we found a bigger issue + if (severity != 4 && severity > this.statuses.get(this.statuses.size() - 1).getSeverity()) { + // another result with a higher severity, replacing the current one, we found a bigger issue, but we are ignoring errors this.executedAtTimestamps.set(this.statuses.size() - 1, executedAt); this.timePeriodDisplayTexts.set(this.statuses.size() - 1, makeTimePeriodDisplayText(timePeriod, timePeriodUtc, checkTimeScale)); this.statuses.set(this.statuses.size() - 1, CheckResultStatus.fromSeverity(severity)); @@ -139,6 +139,10 @@ public void appendResult(LocalDateTime timePeriod, protected String makeTimePeriodDisplayText(LocalDateTime timePeriod, Instant timePeriodUtc, CheckTimeScale checkTimeScale) { Instant timePeriodForcedUtc = timePeriod.toInstant(ZoneOffset.UTC); long timeOffsetSeconds = timePeriodForcedUtc.getEpochSecond() - timePeriodUtc.getEpochSecond(); + if (timeOffsetSeconds > 18L * 3600L || timeOffsetSeconds < -18L * 3600L) { + timeOffsetSeconds = timeOffsetSeconds % (24L * 3600L); + } + ZoneOffset zoneOffset = ZoneOffset.ofTotalSeconds((int) timeOffsetSeconds); if (checkTimeScale != null) { diff --git a/dqops/src/main/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImpl.java b/dqops/src/main/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImpl.java index 4a3caabf87..fd84c1a06e 100644 --- a/dqops/src/main/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImpl.java +++ b/dqops/src/main/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImpl.java @@ -194,6 +194,7 @@ public RuleEvaluationResult evaluateRules(ExecutionContext executionContext, AbstractRuleParametersSpec errorRule = checkSpec.getError(); AbstractRuleParametersSpec warningRule = checkSpec.getWarning(); Double expectedValue = null; + Double newActualValue = null; if (fatalRule != null) { RuleExecutionRunParameters ruleRunParametersFatal = new RuleExecutionRunParameters(actualValue, expectedValueFromSensor, @@ -208,6 +209,10 @@ public RuleEvaluationResult evaluateRules(ExecutionContext executionContext, expectedValue = ruleExecutionResultFatal.getExpectedValue(); } + if (ruleExecutionResultFatal.getNewActualValue() != null) { + newActualValue = ruleExecutionResultFatal.getNewActualValue(); + } + if (ruleExecutionResultFatal.getLowerBound() != null) { result.getFatalLowerBoundColumn().set(targetRowIndex, ruleExecutionResultFatal.getLowerBound()); } @@ -229,6 +234,10 @@ public RuleEvaluationResult evaluateRules(ExecutionContext executionContext, expectedValue = ruleExecutionResultError.getExpectedValue(); } + if (newActualValue == null && ruleExecutionResultError.getNewActualValue() != null) { + newActualValue = ruleExecutionResultError.getNewActualValue(); + } + if (ruleExecutionResultError.getLowerBound() != null) { result.getErrorLowerBoundColumn().set(targetRowIndex, ruleExecutionResultError.getLowerBound()); } @@ -250,6 +259,10 @@ public RuleEvaluationResult evaluateRules(ExecutionContext executionContext, expectedValue = ruleExecutionResultWarning.getExpectedValue(); } + if (newActualValue == null && ruleExecutionResultWarning.getNewActualValue() != null) { + newActualValue = ruleExecutionResultWarning.getNewActualValue(); + } + if (ruleExecutionResultWarning.getLowerBound() != null) { result.getWarningLowerBoundColumn().set(targetRowIndex, ruleExecutionResultWarning.getLowerBound()); } @@ -285,6 +298,10 @@ public RuleEvaluationResult evaluateRules(ExecutionContext executionContext, result.getSeverityColumn().set(targetRowIndex, highestSeverity); result.getExpectedValueColumn().set(targetRowIndex, expectedValue); + + if (newActualValue != null) { + result.getActualValueColumn().set(targetRowIndex, newActualValue); + } } } diff --git a/dqops/src/main/java/com/dqops/execution/rules/RuleExecutionResult.java b/dqops/src/main/java/com/dqops/execution/rules/RuleExecutionResult.java index bc75719b96..f7e7ebbe06 100644 --- a/dqops/src/main/java/com/dqops/execution/rules/RuleExecutionResult.java +++ b/dqops/src/main/java/com/dqops/execution/rules/RuleExecutionResult.java @@ -25,6 +25,7 @@ @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) public class RuleExecutionResult { private boolean passed; + private Double newActualValue; private Double expectedValue; private Double lowerBound; private Double upperBound; @@ -45,6 +46,22 @@ public void setPassed(boolean passed) { this.passed = passed; } + /** + * Returns a new (updated) actual value that was recalculated be the rule and should be stored in the check result instead of the value returned from the sensor. + * @return New updated actual value to store. + */ + public Double getNewActualValue() { + return newActualValue; + } + + /** + * Stores a new updated actual value that should be stored in the check result. + * @param newActualValue New updated check result. + */ + public void setNewActualValue(Double newActualValue) { + this.newActualValue = newActualValue; + } + /** * Returns the expected value that the rule considered as a valid value. * @return Expected value. diff --git a/dqops/src/main/java/com/dqops/execution/sensors/grouping/AbstractGroupedSensorExecutor.java b/dqops/src/main/java/com/dqops/execution/sensors/grouping/AbstractGroupedSensorExecutor.java index c6584401ce..0d927938d3 100644 --- a/dqops/src/main/java/com/dqops/execution/sensors/grouping/AbstractGroupedSensorExecutor.java +++ b/dqops/src/main/java/com/dqops/execution/sensors/grouping/AbstractGroupedSensorExecutor.java @@ -21,12 +21,15 @@ import com.dqops.execution.ExecutionContext; import com.dqops.execution.sensors.SensorPrepareResult; import com.dqops.execution.sensors.progress.SensorExecutionProgressListener; +import com.dqops.metadata.timeseries.TimePeriodGradient; import com.dqops.services.timezone.DefaultTimeZoneProvider; +import com.dqops.utils.datetime.LocalDateTimeTruncateUtility; import tech.tablesaw.api.*; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.ZoneOffset; /** * Implementation of the sensor executor. It is used for grouping similar sensors that can @@ -61,11 +64,14 @@ public abstract GroupedSensorExecutionResult executeGroupedSensor(ExecutionConte /** * Creates a local date time of NOW, using the default configured time zone. + * @param timePeriodGradient Time period gradient to truncate the results. * @return The local date time of NOW, based on the configured default time zone. */ - protected LocalDateTime getNowAtDefaultTimeZone() { + protected LocalDateTime getNowAtDefaultTimeZone(TimePeriodGradient timePeriodGradient) { ZoneId defaultTimeZoneId = this.defaultTimeZoneProvider.getDefaultTimeZoneId(); - return Instant.now().atZone(defaultTimeZoneId).toLocalDateTime(); + LocalDateTime localDateTime = Instant.now().atZone(defaultTimeZoneId).toLocalDateTime(); + LocalDateTime truncatedDateTime = LocalDateTimeTruncateUtility.truncateTimePeriod(localDateTime, timePeriodGradient); + return truncatedDateTime; } /** @@ -92,8 +98,12 @@ public Table createResultTableWithResult(Integer actualValue, PreparedSensorsGro row.setInt(sensorPrepareResult.getActualValueAlias(), actualValue); } - row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, getNowAtDefaultTimeZone()); - row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, Instant.now()); + SensorPrepareResult firstPrepareResult = preparedSensorsGroup.getPreparedSensors().get(0); + TimePeriodGradient timePeriodGradient = firstPrepareResult.getSensorRunParameters().getTimePeriodGradient(); + + LocalDateTime truncatedTimePeriod = getNowAtDefaultTimeZone(timePeriodGradient); + row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, truncatedTimePeriod); + row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, truncatedTimePeriod.toInstant(ZoneOffset.UTC)); return dummyResultTable; } @@ -122,8 +132,12 @@ public Table createResultTableWithResult(Long actualValue, PreparedSensorsGroup row.setLong(sensorPrepareResult.getActualValueAlias(), actualValue); } - row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, getNowAtDefaultTimeZone()); - row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, Instant.now()); + SensorPrepareResult firstPrepareResult = preparedSensorsGroup.getPreparedSensors().get(0); + TimePeriodGradient timePeriodGradient = firstPrepareResult.getSensorRunParameters().getTimePeriodGradient(); + + LocalDateTime truncatedTimePeriod = getNowAtDefaultTimeZone(timePeriodGradient); + row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, truncatedTimePeriod); + row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, truncatedTimePeriod.toInstant(ZoneOffset.UTC)); return dummyResultTable; } @@ -152,8 +166,12 @@ public Table createResultTableWithResult(Double actualValue, PreparedSensorsGrou row.setDouble(sensorPrepareResult.getActualValueAlias(), actualValue); } - row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, getNowAtDefaultTimeZone()); - row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, Instant.now()); + SensorPrepareResult firstPrepareResult = preparedSensorsGroup.getPreparedSensors().get(0); + TimePeriodGradient timePeriodGradient = firstPrepareResult.getSensorRunParameters().getTimePeriodGradient(); + + LocalDateTime truncatedTimePeriod = getNowAtDefaultTimeZone(timePeriodGradient); + row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, truncatedTimePeriod); + row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, truncatedTimePeriod.toInstant(ZoneOffset.UTC)); return dummyResultTable; } diff --git a/dqops/src/main/java/com/dqops/execution/sensors/runners/AbstractSensorRunner.java b/dqops/src/main/java/com/dqops/execution/sensors/runners/AbstractSensorRunner.java index 00b4e0741a..32d2f36d3f 100644 --- a/dqops/src/main/java/com/dqops/execution/sensors/runners/AbstractSensorRunner.java +++ b/dqops/src/main/java/com/dqops/execution/sensors/runners/AbstractSensorRunner.java @@ -36,7 +36,7 @@ * Base abstract class for sensor runners. Executes a sensor given a target data quality check. */ public abstract class AbstractSensorRunner { - private DefaultTimeZoneProvider defaultTimeZoneProvider; + protected DefaultTimeZoneProvider defaultTimeZoneProvider; /** * Dependency injection constructor. @@ -88,89 +88,4 @@ public abstract SensorExecutionResult extractSensorResults(ExecutionContext exec SensorPrepareResult sensorPrepareResult, SensorExecutionProgressListener progressListener, JobCancellationToken jobCancellationToken); - - /** - * Creates a local date time of NOW, using the default configured time zone. - * @return The local date time of NOW, based on the configured default time zone. - */ - protected LocalDateTime getNowAtDefaultTimeZone() { - ZoneId defaultTimeZoneId = this.defaultTimeZoneProvider.getDefaultTimeZoneId(); - return Instant.now().atZone(defaultTimeZoneId).toLocalDateTime(); - } - - /** - * Creates a result table given a single result to be stored. - * @param actualValue Sensor execution actual value. - * @return Dummy result table. - */ - public Table createResultTableWithResult(Integer actualValue) { - Table dummyResultTable = Table.create("dummy_results", - IntColumn.create(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME), - DateTimeColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME), - InstantColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME)); - Row row = dummyResultTable.appendRow(); - row.setInt(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME, actualValue); - row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, getNowAtDefaultTimeZone()); - row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, Instant.now()); - - return dummyResultTable; - } - - /** - * Creates a result table given a single result to be stored. - * @param actualValue Sensor execution actual value. - * @return Dummy result table. - */ - public Table createResultTableWithResult(Long actualValue) { - Table dummyResultTable = Table.create("dummy_results", - LongColumn.create(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME), - DateTimeColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME), - InstantColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME)); - Row row = dummyResultTable.appendRow(); - row.setLong(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME, actualValue); - row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, getNowAtDefaultTimeZone()); - row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, Instant.now()); - - return dummyResultTable; - } - - /** - * Creates a result table given a single result to be stored. - * @param actualValue Sensor execution actual value. - * @return Dummy result table. - */ - public Table createResultTableWithResult(Double actualValue) { - Table dummyResultTable = Table.create("dummy_results", - DoubleColumn.create(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME), - DateTimeColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME), - InstantColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME)); - Row row = dummyResultTable.appendRow(); - row.setDouble(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME, actualValue); - row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, getNowAtDefaultTimeZone()); - row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, Instant.now()); - - return dummyResultTable; - } - - /** - * Creates a one row dummy result table, used when a dummy sensor run is requested (without running the query on the data source). - * @param sensorRunParameters Sensor execution run parameters. - * @return Dummy result table. - */ - public Table createDummyResultTable(SensorExecutionRunParameters sensorRunParameters) { - Table dummyResultTable = Table.create("dummy_results", - DoubleColumn.create(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME), - DateTimeColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME), - InstantColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME)); - Row row = dummyResultTable.appendRow(); - row.setDouble(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME, 10.0); - - TimeSeriesConfigurationSpec effectiveTimeSeries = sensorRunParameters.getTimeSeries(); - if (effectiveTimeSeries != null && effectiveTimeSeries.getMode() != null) { - row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, getNowAtDefaultTimeZone()); - row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, Instant.now()); - } - - return dummyResultTable; - } } diff --git a/dqops/src/main/java/com/dqops/execution/sensors/runners/GenericSensorResultsFactory.java b/dqops/src/main/java/com/dqops/execution/sensors/runners/GenericSensorResultsFactory.java new file mode 100644 index 0000000000..787f445ae5 --- /dev/null +++ b/dqops/src/main/java/com/dqops/execution/sensors/runners/GenericSensorResultsFactory.java @@ -0,0 +1,133 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dqops.execution.sensors.runners; + +import com.dqops.data.readouts.factory.SensorReadoutsColumnNames; +import com.dqops.execution.sensors.SensorExecutionRunParameters; +import com.dqops.metadata.timeseries.TimePeriodGradient; +import com.dqops.metadata.timeseries.TimeSeriesConfigurationSpec; +import com.dqops.utils.datetime.LocalDateTimeTruncateUtility; +import tech.tablesaw.api.*; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; + +/** + * Helper class that creates fake generic sensor results. + */ +public class GenericSensorResultsFactory { + /** + * Creates a local date time of NOW, using the default configured time zone. + * @return The local date time of NOW, based on the configured default time zone. + */ + private static LocalDateTime getNowAtDefaultTimeZone(ZoneId defaultTimeZoneId, TimePeriodGradient timePeriodGradient) { + LocalDateTime localDateTime = Instant.now().atZone(defaultTimeZoneId).toLocalDateTime(); + LocalDateTime truncatedDateTime = LocalDateTimeTruncateUtility.truncateTimePeriod(localDateTime, timePeriodGradient); + return truncatedDateTime; + } + + /** + * Creates a result table given a single result to be stored. + * @param actualValue Sensor execution actual value. + * @param defaultTimeZoneId Default time zone id. + * @param timePeriodGradient Time period gradient used to truncate the time. + * @return Dummy result table. + */ + public static Table createResultTableWithResult(Integer actualValue, + ZoneId defaultTimeZoneId, + TimePeriodGradient timePeriodGradient) { + Table dummyResultTable = Table.create("dummy_results", + IntColumn.create(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME), + DateTimeColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME), + InstantColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME)); + Row row = dummyResultTable.appendRow(); + row.setInt(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME, actualValue); + LocalDateTime nowAtDefaultTimeZone = getNowAtDefaultTimeZone(defaultTimeZoneId, timePeriodGradient); + row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, nowAtDefaultTimeZone); + row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, nowAtDefaultTimeZone.toInstant(ZoneOffset.UTC)); + + return dummyResultTable; + } + + /** + * Creates a result table given a single result to be stored. + * @param actualValue Sensor execution actual value. + * @return Dummy result table. + */ + public static Table createResultTableWithResult(Long actualValue, + ZoneId defaultTimeZoneId, + TimePeriodGradient timePeriodGradient) { + Table dummyResultTable = Table.create("dummy_results", + LongColumn.create(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME), + DateTimeColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME), + InstantColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME)); + Row row = dummyResultTable.appendRow(); + row.setLong(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME, actualValue); + LocalDateTime nowAtDefaultTimeZone = getNowAtDefaultTimeZone(defaultTimeZoneId, timePeriodGradient); + row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, nowAtDefaultTimeZone); + row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, nowAtDefaultTimeZone.toInstant(ZoneOffset.UTC)); + + return dummyResultTable; + } + + /** + * Creates a result table given a single result to be stored. + * @param actualValue Sensor execution actual value. + * @return Dummy result table. + */ + public static Table createResultTableWithResult(Double actualValue, + ZoneId defaultTimeZoneId, + TimePeriodGradient timePeriodGradient) { + Table dummyResultTable = Table.create("dummy_results", + DoubleColumn.create(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME), + DateTimeColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME), + InstantColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME)); + Row row = dummyResultTable.appendRow(); + row.setDouble(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME, actualValue); + LocalDateTime nowAtDefaultTimeZone = getNowAtDefaultTimeZone(defaultTimeZoneId, timePeriodGradient); + row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, nowAtDefaultTimeZone); + row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, nowAtDefaultTimeZone.toInstant(ZoneOffset.UTC)); + + return dummyResultTable; + } + + /** + * Creates a one row dummy result table, used when a dummy sensor run is requested (without running the query on the data source). + * @param sensorRunParameters Sensor execution run parameters. + * @return Dummy result table. + */ + public static Table createDummyResultTable(SensorExecutionRunParameters sensorRunParameters, + ZoneId defaultTimeZoneId) { + Table dummyResultTable = Table.create("dummy_results", + DoubleColumn.create(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME), + DateTimeColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME), + InstantColumn.create(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME)); + Row row = dummyResultTable.appendRow(); + row.setDouble(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME, 10.0); + + TimeSeriesConfigurationSpec effectiveTimeSeries = sensorRunParameters.getTimeSeries(); + if (effectiveTimeSeries != null && effectiveTimeSeries.getMode() != null) { + LocalDateTime nowAtDefaultTimeZone = getNowAtDefaultTimeZone(defaultTimeZoneId, effectiveTimeSeries.getTimeGradient()); + row.setDateTime(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME, nowAtDefaultTimeZone); + row.setInstant(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME, nowAtDefaultTimeZone.toInstant(ZoneOffset.UTC)); + } + + return dummyResultTable; + } +} diff --git a/dqops/src/main/java/com/dqops/execution/sqltemplates/rendering/JinjaSqlTemplateSensorRunner.java b/dqops/src/main/java/com/dqops/execution/sqltemplates/rendering/JinjaSqlTemplateSensorRunner.java index 8794311c91..18f4f37c06 100644 --- a/dqops/src/main/java/com/dqops/execution/sqltemplates/rendering/JinjaSqlTemplateSensorRunner.java +++ b/dqops/src/main/java/com/dqops/execution/sqltemplates/rendering/JinjaSqlTemplateSensorRunner.java @@ -31,6 +31,7 @@ import com.dqops.execution.sensors.progress.ExecutingSqlOnConnectionEvent; import com.dqops.execution.sensors.progress.SensorExecutionProgressListener; import com.dqops.execution.sensors.runners.AbstractSensorRunner; +import com.dqops.execution.sensors.runners.GenericSensorResultsFactory; import com.dqops.metadata.definitions.sensors.ProviderSensorDefinitionSpec; import com.dqops.metadata.sources.ConnectionSpec; import com.dqops.services.timezone.DefaultTimeZoneProvider; @@ -145,7 +146,8 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, } } - Table dummyResultTable = createDummyResultTable(sensorRunParameters); + Table dummyResultTable = GenericSensorResultsFactory.createDummyResultTable(sensorRunParameters, + this.defaultTimeZoneProvider.getDefaultTimeZoneId()); return new SensorExecutionResult(sensorRunParameters, dummyResultTable); } catch (Throwable exception) { diff --git a/dqops/src/main/java/com/dqops/sensors/column/schema/ColumnColumnExistsSensorRunner.java b/dqops/src/main/java/com/dqops/sensors/column/schema/ColumnColumnExistsSensorRunner.java index 954a98310f..fd133e9f77 100644 --- a/dqops/src/main/java/com/dqops/sensors/column/schema/ColumnColumnExistsSensorRunner.java +++ b/dqops/src/main/java/com/dqops/sensors/column/schema/ColumnColumnExistsSensorRunner.java @@ -29,6 +29,7 @@ import com.dqops.execution.sensors.progress.ExecutingSqlOnConnectionEvent; import com.dqops.execution.sensors.progress.SensorExecutionProgressListener; import com.dqops.execution.sensors.runners.AbstractSensorRunner; +import com.dqops.execution.sensors.runners.GenericSensorResultsFactory; import com.dqops.metadata.sources.*; import com.dqops.services.timezone.DefaultTimeZoneProvider; import lombok.extern.slf4j.Slf4j; @@ -110,13 +111,15 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont if (introspectedTableSpec == null) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } ColumnSpec introspectedColumnSpec = introspectedTableSpec.getColumns().get(sensorRunParameters.getColumn().getColumnName()); - Table table = createResultTableWithResult(introspectedColumnSpec != null ? 1.0 : 0.0); + Table table = GenericSensorResultsFactory.createResultTableWithResult(introspectedColumnSpec != null ? 1.0 : 0.0, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } catch (Throwable exception) { @@ -162,19 +165,22 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, if (retrievedTableSpecList.size() == 0) { // table not found - Table table = createResultTableWithResult(0.0); + Table table = GenericSensorResultsFactory.createResultTableWithResult(0.0, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } TableSpec introspectedTableSpec = retrievedTableSpecList.get(0); ColumnSpec introspectedColumnSpec = introspectedTableSpec.getColumns().get(sensorRunParameters.getColumn().getColumnName()); - Table table = createResultTableWithResult(introspectedColumnSpec != null ? 1.0 : 0.0); + Table table = GenericSensorResultsFactory.createResultTableWithResult(introspectedColumnSpec != null ? 1.0 : 0.0, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } } - Table dummyResultTable = createDummyResultTable(sensorRunParameters); + Table dummyResultTable = GenericSensorResultsFactory.createDummyResultTable(sensorRunParameters, + this.defaultTimeZoneProvider.getDefaultTimeZoneId()); return new SensorExecutionResult(sensorRunParameters, dummyResultTable); } catch (Throwable exception) { diff --git a/dqops/src/main/java/com/dqops/sensors/column/schema/ColumnColumnTypeHashSensorRunner.java b/dqops/src/main/java/com/dqops/sensors/column/schema/ColumnColumnTypeHashSensorRunner.java index 24b311b139..bd126f78a2 100644 --- a/dqops/src/main/java/com/dqops/sensors/column/schema/ColumnColumnTypeHashSensorRunner.java +++ b/dqops/src/main/java/com/dqops/sensors/column/schema/ColumnColumnTypeHashSensorRunner.java @@ -29,6 +29,7 @@ import com.dqops.execution.sensors.progress.ExecutingSqlOnConnectionEvent; import com.dqops.execution.sensors.progress.SensorExecutionProgressListener; import com.dqops.execution.sensors.runners.AbstractSensorRunner; +import com.dqops.execution.sensors.runners.GenericSensorResultsFactory; import com.dqops.metadata.sources.*; import com.dqops.services.timezone.DefaultTimeZoneProvider; import com.google.common.hash.HashCode; @@ -117,7 +118,8 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont if (introspectedTableSpec == null) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } @@ -125,7 +127,8 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont if (introspectedColumnSpec == null || introspectedColumnSpec.getTypeSnapshot() == null) { // column not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } @@ -133,7 +136,8 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont long typeSnapshotHash = typeSnapshot.hashCode64(); long hashFitInDoubleExponent = typeSnapshotHash & ((1L << 52) - 1L); // because we are storing the results of data quality checks in a IEEE 754 double-precision floating-point value and we need exact match, we need to return only as many bits as the fraction part (52 bits) can fit in a Double value, without any unwanted truncations - Table table = createResultTableWithResult(hashFitInDoubleExponent); + Table table = GenericSensorResultsFactory.createResultTableWithResult(hashFitInDoubleExponent, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } catch (Throwable exception) { @@ -179,7 +183,8 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, if (retrievedTableSpecList.size() == 0) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } @@ -188,7 +193,8 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, if (introspectedColumnSpec == null || introspectedColumnSpec.getTypeSnapshot() == null) { // column not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } @@ -196,12 +202,14 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, long typeSnapshotHash = typeSnapshot.hashCode64(); long hashFitInDoubleExponent = typeSnapshotHash & ((1L << 52) - 1L); // because we are storing the results of data quality checks in a IEEE 754 double-precision floating-point value and we need exact match, we need to return only as many bits as the fraction part (52 bits) can fit in a Double value, without any unwanted truncations - Table table = createResultTableWithResult(hashFitInDoubleExponent); + Table table = GenericSensorResultsFactory.createResultTableWithResult(hashFitInDoubleExponent, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } } - Table dummyResultTable = createDummyResultTable(sensorRunParameters); + Table dummyResultTable = GenericSensorResultsFactory.createDummyResultTable(sensorRunParameters, + this.defaultTimeZoneProvider.getDefaultTimeZoneId()); return new SensorExecutionResult(sensorRunParameters, dummyResultTable); } catch (Throwable exception) { diff --git a/dqops/src/main/java/com/dqops/sensors/table/availability/TableAvailabilitySensorParametersSpec.java b/dqops/src/main/java/com/dqops/sensors/table/availability/TableAvailabilitySensorParametersSpec.java index 53db33669a..63e0e49496 100644 --- a/dqops/src/main/java/com/dqops/sensors/table/availability/TableAvailabilitySensorParametersSpec.java +++ b/dqops/src/main/java/com/dqops/sensors/table/availability/TableAvailabilitySensorParametersSpec.java @@ -26,7 +26,7 @@ import lombok.EqualsAndHashCode; /** - * Table availability sensor that executes a row count query. + * Table availability sensor that executes a row count query. This sensor returns 0.0 when no failure was detected or 1.0 when a failure was detected. */ @JsonInclude(JsonInclude.Include.NON_NULL) @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) diff --git a/dqops/src/main/java/com/dqops/sensors/table/availability/TableAvailabilitySensorRunner.java b/dqops/src/main/java/com/dqops/sensors/table/availability/TableAvailabilitySensorRunner.java index a14efc0c31..492e1f6a4c 100644 --- a/dqops/src/main/java/com/dqops/sensors/table/availability/TableAvailabilitySensorRunner.java +++ b/dqops/src/main/java/com/dqops/sensors/table/availability/TableAvailabilitySensorRunner.java @@ -24,6 +24,7 @@ import com.dqops.execution.sensors.grouping.GroupedSensorExecutionResult; import com.dqops.execution.sensors.progress.SensorExecutionProgressListener; import com.dqops.execution.sensors.runners.AbstractSensorRunner; +import com.dqops.execution.sensors.runners.GenericSensorResultsFactory; import com.dqops.execution.sqltemplates.rendering.JinjaSqlTemplateSensorRunner; import com.dqops.services.timezone.DefaultTimeZoneProvider; import lombok.extern.slf4j.Slf4j; @@ -71,7 +72,9 @@ public SensorPrepareResult prepareSensor(ExecutionContext executionContext, SensorExecutionRunParameters sensorRunParameters, SensorDefinitionFindResult sensorDefinition, SensorExecutionProgressListener progressListener) { - return this.jinjaSqlTemplateSensorRunner.prepareSensor(executionContext, sensorRunParameters, sensorDefinition, progressListener); + SensorPrepareResult sensorPrepareResult = this.jinjaSqlTemplateSensorRunner.prepareSensor(executionContext, sensorRunParameters, sensorDefinition, progressListener); + sensorPrepareResult.setSensorRunner(this); + return sensorPrepareResult; } /** @@ -97,17 +100,20 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont if (sensorExecutionResult.isSuccess()) { try { if (sensorExecutionResult.getResultTable().column(0).get(0) == null) { - Table resultTableWithResult = createResultTableWithResult(1.0); + Table resultTableWithResult = GenericSensorResultsFactory.createResultTableWithResult(0.0, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorRunParameters.getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, resultTableWithResult); } } catch (Exception exception) { - Table resultTable = createResultTableWithResult(0.0); + Table resultTable = GenericSensorResultsFactory.createResultTableWithResult(1.0, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorRunParameters.getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, resultTable); } return sensorExecutionResult; } - Table resultTable = createResultTableWithResult(0.0); + Table resultTable = GenericSensorResultsFactory.createResultTableWithResult(1.0, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorRunParameters.getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, resultTable); } @@ -134,17 +140,20 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, if (sensorExecutionResult.isSuccess()) { try { if (sensorExecutionResult.getResultTable().column(0).get(0) == null) { - Table resultTableWithResult = createResultTableWithResult(1.0); + Table resultTableWithResult = GenericSensorResultsFactory.createResultTableWithResult(0.0, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorRunParameters.getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, resultTableWithResult); } } catch (Exception exception) { - Table resultTable = createResultTableWithResult(0.0); + Table resultTable = GenericSensorResultsFactory.createResultTableWithResult(1.0, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorRunParameters.getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, resultTable); } return sensorExecutionResult; } - Table resultTable = createResultTableWithResult(0.0); + Table resultTable = GenericSensorResultsFactory.createResultTableWithResult(1.0, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorRunParameters.getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, resultTable); } } diff --git a/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnCountSensorRunner.java b/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnCountSensorRunner.java index cef0c5fb92..12c792b6de 100644 --- a/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnCountSensorRunner.java +++ b/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnCountSensorRunner.java @@ -29,6 +29,7 @@ import com.dqops.execution.sensors.progress.ExecutingSqlOnConnectionEvent; import com.dqops.execution.sensors.progress.SensorExecutionProgressListener; import com.dqops.execution.sensors.runners.AbstractSensorRunner; +import com.dqops.execution.sensors.runners.GenericSensorResultsFactory; import com.dqops.metadata.sources.ConnectionSpec; import com.dqops.metadata.sources.PhysicalTableName; import com.dqops.metadata.sources.TableSpec; @@ -116,13 +117,15 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont if (introspectedTableSpec == null) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } int columnCount = introspectedTableSpec.getColumns().size(); - Table table = createResultTableWithResult(columnCount); + Table table = GenericSensorResultsFactory.createResultTableWithResult(columnCount, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } catch (Throwable exception) { @@ -168,19 +171,22 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, if (retrievedTableSpecList.size() == 0) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } TableSpec introspectedTableSpec = retrievedTableSpecList.get(0); int columnCount = introspectedTableSpec.getColumns().size(); - Table table = createResultTableWithResult(columnCount); + Table table = GenericSensorResultsFactory.createResultTableWithResult(columnCount, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } } - Table dummyResultTable = createDummyResultTable(sensorRunParameters); + Table dummyResultTable = GenericSensorResultsFactory.createDummyResultTable(sensorRunParameters, + this.defaultTimeZoneProvider.getDefaultTimeZoneId()); return new SensorExecutionResult(sensorRunParameters, dummyResultTable); } catch (Throwable exception) { diff --git a/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnListOrderedHashSensorRunner.java b/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnListOrderedHashSensorRunner.java index 663298dd25..66e1f5a1b3 100644 --- a/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnListOrderedHashSensorRunner.java +++ b/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnListOrderedHashSensorRunner.java @@ -29,6 +29,7 @@ import com.dqops.execution.sensors.progress.ExecutingSqlOnConnectionEvent; import com.dqops.execution.sensors.progress.SensorExecutionProgressListener; import com.dqops.execution.sensors.runners.AbstractSensorRunner; +import com.dqops.execution.sensors.runners.GenericSensorResultsFactory; import com.dqops.metadata.sources.ConnectionSpec; import com.dqops.metadata.sources.PhysicalTableName; import com.dqops.metadata.sources.TableSpec; @@ -117,7 +118,8 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont if (introspectedTableSpec == null) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } @@ -127,7 +129,8 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont long fullHash = Math.abs(Hashing.combineOrdered(elementHashes).asLong()); long hashFitInDoubleExponent = fullHash & ((1L << 52) - 1L); // because we are storing the results of data quality checks in a IEEE 754 double-precision floating-point value and we need exact match, we need to return only as many bits as the fraction part (52 bits) can fit in a Double value, without any unwanted truncations - Table table = createResultTableWithResult(hashFitInDoubleExponent); + Table table = GenericSensorResultsFactory.createResultTableWithResult(hashFitInDoubleExponent, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } catch (Throwable exception) { @@ -173,7 +176,8 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, if (retrievedTableSpecList.size() == 0) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } @@ -184,12 +188,14 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, long fullHash = Math.abs(Hashing.combineOrdered(elementHashes).asLong()); long hashFitInDoubleExponent = fullHash & ((1L << 52) - 1L); // because we are storing the results of data quality checks in a IEEE 754 double-precision floating-point value and we need exact match, we need to return only as many bits as the fraction part (52 bits) can fit in a Double value, without any unwanted truncations - Table table = createResultTableWithResult(hashFitInDoubleExponent); + Table table = GenericSensorResultsFactory.createResultTableWithResult(hashFitInDoubleExponent, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } } - Table dummyResultTable = createDummyResultTable(sensorRunParameters); + Table dummyResultTable = GenericSensorResultsFactory.createDummyResultTable(sensorRunParameters, + this.defaultTimeZoneProvider.getDefaultTimeZoneId()); return new SensorExecutionResult(sensorRunParameters, dummyResultTable); } catch (Throwable exception) { diff --git a/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnListUnorderedHashSensorRunner.java b/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnListUnorderedHashSensorRunner.java index 4657f362ad..f16a9deb10 100644 --- a/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnListUnorderedHashSensorRunner.java +++ b/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnListUnorderedHashSensorRunner.java @@ -29,6 +29,7 @@ import com.dqops.execution.sensors.progress.ExecutingSqlOnConnectionEvent; import com.dqops.execution.sensors.progress.SensorExecutionProgressListener; import com.dqops.execution.sensors.runners.AbstractSensorRunner; +import com.dqops.execution.sensors.runners.GenericSensorResultsFactory; import com.dqops.metadata.sources.ConnectionSpec; import com.dqops.metadata.sources.PhysicalTableName; import com.dqops.metadata.sources.TableSpec; @@ -117,7 +118,8 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont if (introspectedTableSpec == null) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } @@ -127,7 +129,8 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont long fullHash = Math.abs(Hashing.combineUnordered(elementHashes).asLong()); long hashFitInDoubleExponent = fullHash & ((1L << 52) - 1L); // because we are storing the results of data quality checks in a IEEE 754 double-precision floating-point value and we need exact match, we need to return only as many bits as the fraction part (52 bits) can fit in a Double value, without any unwanted truncations - Table table = createResultTableWithResult(hashFitInDoubleExponent); + Table table = GenericSensorResultsFactory.createResultTableWithResult(hashFitInDoubleExponent, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } catch (Throwable exception) { @@ -173,7 +176,8 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, if (retrievedTableSpecList.size() == 0) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } @@ -184,12 +188,14 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, long fullHash = Math.abs(Hashing.combineUnordered(elementHashes).asLong()); long hashFitInDoubleExponent = fullHash & ((1L << 52) - 1L); // because we are storing the results of data quality checks in a IEEE 754 double-precision floating-point value and we need exact match, we need to return only as many bits as the fraction part (52 bits) can fit in a Double value, without any unwanted truncations - Table table = createResultTableWithResult(hashFitInDoubleExponent); + Table table = GenericSensorResultsFactory.createResultTableWithResult(hashFitInDoubleExponent, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } } - Table dummyResultTable = createDummyResultTable(sensorRunParameters); + Table dummyResultTable = GenericSensorResultsFactory.createDummyResultTable(sensorRunParameters, + this.defaultTimeZoneProvider.getDefaultTimeZoneId()); return new SensorExecutionResult(sensorRunParameters, dummyResultTable); } catch (Throwable exception) { diff --git a/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnTypesHashSensorRunner.java b/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnTypesHashSensorRunner.java index 7f43c62013..8814e877ed 100644 --- a/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnTypesHashSensorRunner.java +++ b/dqops/src/main/java/com/dqops/sensors/table/schema/TableColumnTypesHashSensorRunner.java @@ -29,6 +29,7 @@ import com.dqops.execution.sensors.progress.ExecutingSqlOnConnectionEvent; import com.dqops.execution.sensors.progress.SensorExecutionProgressListener; import com.dqops.execution.sensors.runners.AbstractSensorRunner; +import com.dqops.execution.sensors.runners.GenericSensorResultsFactory; import com.dqops.metadata.sources.ConnectionSpec; import com.dqops.metadata.sources.PhysicalTableName; import com.dqops.metadata.sources.TableSpec; @@ -117,7 +118,8 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont if (introspectedTableSpec == null) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } @@ -130,7 +132,8 @@ public SensorExecutionResult extractSensorResults(ExecutionContext executionCont long fullHash = Math.abs(Hashing.combineUnordered(elementHashes).asLong()); long hashFitInDoubleExponent = fullHash & ((1L << 52) - 1L); // because we are storing the results of data quality checks in a IEEE 754 double-precision floating-point value and we need exact match, we need to return only as many bits as the fraction part (52 bits) can fit in a Double value, without any unwanted truncations - Table table = createResultTableWithResult(hashFitInDoubleExponent); + Table table = GenericSensorResultsFactory.createResultTableWithResult(hashFitInDoubleExponent, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } catch (Throwable exception) { @@ -176,7 +179,8 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, if (retrievedTableSpecList.size() == 0) { // table not found - Table table = createResultTableWithResult((Double) null); + Table table = GenericSensorResultsFactory.createResultTableWithResult((Double) null, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } @@ -190,12 +194,14 @@ public SensorExecutionResult executeSensor(ExecutionContext executionContext, long fullHash = Math.abs(Hashing.combineUnordered(elementHashes).asLong()); long hashFitInDoubleExponent = fullHash & ((1L << 52) - 1L); // because we are storing the results of data quality checks in a IEEE 754 double-precision floating-point value and we need exact match, we need to return only as many bits as the fraction part (52 bits) can fit in a Double value, without any unwanted truncations - Table table = createResultTableWithResult(hashFitInDoubleExponent); + Table table = GenericSensorResultsFactory.createResultTableWithResult(hashFitInDoubleExponent, + this.defaultTimeZoneProvider.getDefaultTimeZoneId(), sensorPrepareResult.getSensorRunParameters().getTimePeriodGradient()); return new SensorExecutionResult(sensorRunParameters, table); } } - Table dummyResultTable = createDummyResultTable(sensorRunParameters); + Table dummyResultTable = GenericSensorResultsFactory.createDummyResultTable(sensorRunParameters, + this.defaultTimeZoneProvider.getDefaultTimeZoneId()); return new SensorExecutionResult(sensorRunParameters, dummyResultTable); } catch (Throwable exception) { diff --git a/dqops/src/test/java/com/dqops/rules/comparison/MaxFailuresRuleParametersSpecTests.java b/dqops/src/test/java/com/dqops/rules/comparison/MaxFailuresRuleParametersSpecTests.java index b0c945025f..46bca8bccf 100644 --- a/dqops/src/test/java/com/dqops/rules/comparison/MaxFailuresRuleParametersSpecTests.java +++ b/dqops/src/test/java/com/dqops/rules/comparison/MaxFailuresRuleParametersSpecTests.java @@ -68,16 +68,16 @@ void timeWindowSettingsPredictionTimeWindow_whenMaxFailureSensor_thenRequiresHis void executeRuleMaxFailures_whenCurrentResultPassAndPreviousAreFailuresAndMaxFailures5_thenReturnsPassed() { this.sut.setMaxFailures(5L); - Double[] previousReadouts = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + Double[] previousReadouts = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; this.sensorReadouts = previousReadouts; HistoricDataPoint[] historicDataPoints = HistoricDataPointObjectMother.fillHistoricReadouts(this.timeWindowSettings, TimePeriodGradient.day, this.readoutTimestamp, this.sensorReadouts); - RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(1.0, + RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(0.0, this.sut, this.readoutTimestamp, historicDataPoints, this.timeWindowSettings); Assertions.assertTrue(ruleExecutionResult.isPassed()); - Assertions.assertNull(ruleExecutionResult.getExpectedValue()); + Assertions.assertEquals(0.0, ruleExecutionResult.getExpectedValue()); Assertions.assertNull(ruleExecutionResult.getLowerBound()); Assertions.assertEquals(5.0, ruleExecutionResult.getUpperBound()); } @@ -86,16 +86,16 @@ void executeRuleMaxFailures_whenCurrentResultPassAndPreviousAreFailuresAndMaxFai void executeRuleMaxFailures_whenCurrentResultPassAndPreviousAreFailuresAndMaxFailures0_thenReturnsPassed() { this.sut.setMaxFailures(0L); - Double[] previousReadouts = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + Double[] previousReadouts = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; this.sensorReadouts = previousReadouts; HistoricDataPoint[] historicDataPoints = HistoricDataPointObjectMother.fillHistoricReadouts(this.timeWindowSettings, TimePeriodGradient.day, this.readoutTimestamp, this.sensorReadouts); - RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(1.0, + RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(0.0, this.sut, this.readoutTimestamp, historicDataPoints, this.timeWindowSettings); Assertions.assertTrue(ruleExecutionResult.isPassed()); - Assertions.assertNull(ruleExecutionResult.getExpectedValue()); + Assertions.assertEquals(0.0, ruleExecutionResult.getExpectedValue()); Assertions.assertNull(ruleExecutionResult.getLowerBound()); Assertions.assertEquals(0.0, ruleExecutionResult.getUpperBound()); } @@ -104,7 +104,7 @@ void executeRuleMaxFailures_whenCurrentResultPassAndPreviousAreFailuresAndMaxFai void executeRuleMaxFailures_whenCurrentResultFailAndPreviousArePassAndMaxFailures1_thenReturnsPassed() { this.sut.setMaxFailures(1L); - Double[] previousReadouts = {1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; + Double[] previousReadouts = {0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; this.sensorReadouts = previousReadouts; HistoricDataPoint[] historicDataPoints = HistoricDataPointObjectMother.fillHistoricReadouts(this.timeWindowSettings, TimePeriodGradient.day, this.readoutTimestamp, this.sensorReadouts); @@ -113,16 +113,18 @@ void executeRuleMaxFailures_whenCurrentResultFailAndPreviousArePassAndMaxFailure this.sut, this.readoutTimestamp, historicDataPoints, this.timeWindowSettings); Assertions.assertTrue(ruleExecutionResult.isPassed()); - Assertions.assertNull(ruleExecutionResult.getExpectedValue()); + Assertions.assertEquals(0.0, ruleExecutionResult.getExpectedValue()); Assertions.assertNull(ruleExecutionResult.getLowerBound()); Assertions.assertEquals(1.0, ruleExecutionResult.getUpperBound()); + Assertions.assertEquals(1.0, ruleExecutionResult.getNewActualValue()); } @Test void executeRuleMaxFailures_whenCurrentResultFailAndPreviousArePassButRecentValueAreNullAndMaxFailures1_thenReturnsPassed() { this.sut.setMaxFailures(1L); - Double[] previousReadouts = {1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, null, null, null}; + Double[] previousReadouts = {0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, null, null, null}; + this.sensorReadouts = previousReadouts; HistoricDataPoint[] historicDataPoints = HistoricDataPointObjectMother.fillHistoricReadouts(this.timeWindowSettings, TimePeriodGradient.day, this.readoutTimestamp, this.sensorReadouts); @@ -131,7 +133,7 @@ void executeRuleMaxFailures_whenCurrentResultFailAndPreviousArePassButRecentValu this.sut, this.readoutTimestamp, historicDataPoints, this.timeWindowSettings); Assertions.assertTrue(ruleExecutionResult.isPassed()); - Assertions.assertNull(ruleExecutionResult.getExpectedValue()); + Assertions.assertEquals(0.0, ruleExecutionResult.getExpectedValue()); Assertions.assertNull(ruleExecutionResult.getLowerBound()); Assertions.assertEquals(1.0, ruleExecutionResult.getUpperBound()); } @@ -140,7 +142,7 @@ void executeRuleMaxFailures_whenCurrentResultFailAndPreviousArePassButRecentValu void executeRuleMaxFailures_whenCurrentResultFailAndPreviousArePassButRecentValueAreNullAndMaxFailures5_thenReturnsPassed() { this.sut.setMaxFailures(5L); - Double[] previousReadouts = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, null, 0.0, 0.0, null, null, null}; + Double[] previousReadouts = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, null, 1.0, 1.0, null, null, null}; this.sensorReadouts = previousReadouts; HistoricDataPoint[] historicDataPoints = HistoricDataPointObjectMother.fillHistoricReadouts(this.timeWindowSettings, TimePeriodGradient.day, this.readoutTimestamp, this.sensorReadouts); @@ -149,7 +151,7 @@ void executeRuleMaxFailures_whenCurrentResultFailAndPreviousArePassButRecentValu this.sut, this.readoutTimestamp, historicDataPoints, this.timeWindowSettings); Assertions.assertTrue(ruleExecutionResult.isPassed()); - Assertions.assertNull(ruleExecutionResult.getExpectedValue()); + Assertions.assertEquals(0.0, ruleExecutionResult.getExpectedValue()); Assertions.assertNull(ruleExecutionResult.getLowerBound()); Assertions.assertEquals(5.0, ruleExecutionResult.getUpperBound()); } @@ -158,16 +160,16 @@ void executeRuleMaxFailures_whenCurrentResultFailAndPreviousArePassButRecentValu void executeRuleMaxFailures_whenCurrentResultFailAndPreviousArePassAndMaxFailures0_thenReturnsFailed() { this.sut.setMaxFailures(0L); - Double[] previousReadouts = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; + Double[] previousReadouts = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; this.sensorReadouts = previousReadouts; HistoricDataPoint[] historicDataPoints = HistoricDataPointObjectMother.fillHistoricReadouts(this.timeWindowSettings, TimePeriodGradient.day, this.readoutTimestamp, this.sensorReadouts); - RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(0.0, + RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(1.0, this.sut, this.readoutTimestamp, historicDataPoints, this.timeWindowSettings); Assertions.assertFalse(ruleExecutionResult.isPassed()); - Assertions.assertNull(ruleExecutionResult.getExpectedValue()); + Assertions.assertEquals(0.0, ruleExecutionResult.getExpectedValue()); Assertions.assertNull(ruleExecutionResult.getLowerBound()); Assertions.assertEquals(0.0, ruleExecutionResult.getUpperBound()); } @@ -176,16 +178,16 @@ void executeRuleMaxFailures_whenCurrentResultFailAndPreviousArePassAndMaxFailure void executeRuleMaxFailures_whenCurrentResultFailAndPreviousAreAllFailuresAndMaxFailures1_thenReturnsFailed() { this.sut.setMaxFailures(1L); - Double[] previousReadouts = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + Double[] previousReadouts = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0}; this.sensorReadouts = previousReadouts; HistoricDataPoint[] historicDataPoints = HistoricDataPointObjectMother.fillHistoricReadouts(this.timeWindowSettings, TimePeriodGradient.day, this.readoutTimestamp, this.sensorReadouts); - RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(0.0, + RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(1.0, this.sut, this.readoutTimestamp, historicDataPoints, this.timeWindowSettings); Assertions.assertFalse(ruleExecutionResult.isPassed()); - Assertions.assertNull(ruleExecutionResult.getExpectedValue()); + Assertions.assertEquals(0.0, ruleExecutionResult.getExpectedValue()); Assertions.assertNull(ruleExecutionResult.getLowerBound()); Assertions.assertEquals(1.0, ruleExecutionResult.getUpperBound()); } @@ -194,16 +196,16 @@ void executeRuleMaxFailures_whenCurrentResultFailAndPreviousAreAllFailuresAndMax void executeRuleMaxFailures_whenCurrentResultFailAndPreviousAreAllFailuresAndMaxFailures5_thenReturnsFailed() { this.sut.setMaxFailures(5L); - Double[] previousReadouts = {0.0, 1.0, 0.0, 0.0, 0.0, null, 0.0, 0.0, null, 0.0, 0.0, 0.0}; + Double[] previousReadouts = {1.0, 0.0, 1.0, 1.0, 1.0, null, 1.0, 1.0, null, 1.0, 1.0, 1.0}; this.sensorReadouts = previousReadouts; HistoricDataPoint[] historicDataPoints = HistoricDataPointObjectMother.fillHistoricReadouts(this.timeWindowSettings, TimePeriodGradient.day, this.readoutTimestamp, this.sensorReadouts); - RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(0.0, + RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(1.0, this.sut, this.readoutTimestamp, historicDataPoints, this.timeWindowSettings); Assertions.assertFalse(ruleExecutionResult.isPassed()); - Assertions.assertNull(ruleExecutionResult.getExpectedValue()); + Assertions.assertEquals(0.0, ruleExecutionResult.getExpectedValue()); Assertions.assertNull(ruleExecutionResult.getLowerBound()); Assertions.assertEquals(5.0, ruleExecutionResult.getUpperBound()); } @@ -212,16 +214,16 @@ void executeRuleMaxFailures_whenCurrentResultFailAndPreviousAreAllFailuresAndMax void executeRuleMaxFailures_whenCurrentResultFailAndPreviousAreAllFailuresButOnly3AndMaxFailures5_thenReturnsPassed() { this.sut.setMaxFailures(5L); - Double[] previousReadouts = {0.0, null, null, null, null, null, 0.0, null, 0.0, null}; + Double[] previousReadouts = {1.0, null, null, null, null, null, 1.0, null, 1.0, null}; this.sensorReadouts = previousReadouts; HistoricDataPoint[] historicDataPoints = HistoricDataPointObjectMother.fillHistoricReadouts(this.timeWindowSettings, TimePeriodGradient.day, this.readoutTimestamp, this.sensorReadouts); - RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(0.0, + RuleExecutionResult ruleExecutionResult = PythonRuleRunnerObjectMother.executeBuiltInRule(1.0, this.sut, this.readoutTimestamp, historicDataPoints, this.timeWindowSettings); Assertions.assertTrue(ruleExecutionResult.isPassed()); - Assertions.assertNull(ruleExecutionResult.getExpectedValue()); + Assertions.assertEquals(0.0, ruleExecutionResult.getExpectedValue()); Assertions.assertNull(ruleExecutionResult.getLowerBound()); Assertions.assertEquals(5.0, ruleExecutionResult.getUpperBound()); } diff --git a/dqops/src/test/java/com/dqops/utils/docs/sensors/SensorDocumentationModelFactoryImplTests.java b/dqops/src/test/java/com/dqops/utils/docs/sensors/SensorDocumentationModelFactoryImplTests.java index 386f86ffe4..ae3429bfba 100644 --- a/dqops/src/test/java/com/dqops/utils/docs/sensors/SensorDocumentationModelFactoryImplTests.java +++ b/dqops/src/test/java/com/dqops/utils/docs/sensors/SensorDocumentationModelFactoryImplTests.java @@ -57,7 +57,7 @@ void createSensorDocumentation_whenCalledForClassWithSensorParameterFields_thenC Assertions.assertEquals("column/strings/string_match_regex_percent", sensorDocumentation.getFullSensorName()); Assertions.assertNotNull(sensorDocumentation.getSqlTemplates()); - Assertions.assertEquals(6,sensorDocumentation.getSqlTemplates().keySet().size()); + Assertions.assertEquals(7,sensorDocumentation.getSqlTemplates().keySet().size()); Assertions.assertTrue(sensorDocumentation.getSqlTemplates().keySet().stream() .map(ProviderTypeModel::getProviderTypeName) .anyMatch(provider -> provider.equals("bigquery"))); diff --git a/home/rules/comparison/max_failures.py b/home/rules/comparison/max_failures.py index c9c08d2932..fdffb63ec5 100644 --- a/home/rules/comparison/max_failures.py +++ b/home/rules/comparison/max_failures.py @@ -48,12 +48,14 @@ class RuleExecutionRunParameters: # what is the expected value for the rule and what are the upper and lower boundaries of accepted values (optional) class RuleExecutionResult: passed: bool + new_actual_value: float expected_value: float lower_bound: float upper_bound: float - def __init__(self, passed=True, expected_value=None, lower_bound=None, upper_bound=None): + def __init__(self, passed=True, new_actual_value=None, expected_value=None, lower_bound=None, upper_bound=None): self.passed = passed + self.new_actual_value = new_actual_value self.expected_value = expected_value self.lower_bound = lower_bound self.upper_bound = upper_bound @@ -72,16 +74,16 @@ def evaluate_rule(rule_parameters: RuleExecutionRunParameters) -> RuleExecutionR filtered.reverse() - recent_failures = 0 + recent_failures = 0 for i in filtered: - if i == 0: + if i > 0: recent_failures += 1 else: break - expected_value = None + expected_value = 0 lower_bound = None upper_bound = rule_parameters.parameters.max_failures - passed = recent_failures <= upper_bound + passed = recent_failures <= rule_parameters.parameters.max_failures - return RuleExecutionResult(passed, expected_value, lower_bound, upper_bound) + return RuleExecutionResult(passed, recent_failures, expected_value, lower_bound, upper_bound) diff --git a/home/rules/comparison/value_changed.dqorule.yaml b/home/rules/comparison/value_changed.dqorule.yaml index 021efcc55d..8091e83404 100644 --- a/home/rules/comparison/value_changed.dqorule.yaml +++ b/home/rules/comparison/value_changed.dqorule.yaml @@ -4,6 +4,6 @@ spec: type: python mode: previous_readouts time_window: - prediction_time_window: 30 + prediction_time_window: 60 min_periods_with_readouts: 0 historic_data_point_grouping: last_n_readouts diff --git a/home/rules/comparison/value_changed.py b/home/rules/comparison/value_changed.py index f0464cf0de..676ba59969 100644 --- a/home/rules/comparison/value_changed.py +++ b/home/rules/comparison/value_changed.py @@ -68,11 +68,11 @@ def evaluate_rule(rule_parameters: RuleExecutionRunParameters) -> RuleExecutionR if not hasattr(rule_parameters, 'previous_readouts'): return RuleExecutionResult() - filtered = [readouts.sensor_readout for readouts in rule_parameters.previous_readouts if readouts is not None and rule_parameters.actual_value is not None] + filtered = [readouts.sensor_readout for readouts in rule_parameters.previous_readouts if readouts is not None and hasattr(readouts, 'sensor_readout') and rule_parameters.actual_value is not None] - expected_value = None - lower_bound = None - upper_bound = None - passed = (filtered[-1] is not None and filtered[-1] == rule_parameters.actual_value) or filtered[-1] == None + expected_value = filtered[-1].sensor_readout if len(filtered) > 0 else None + lower_bound = expected_value + upper_bound = expected_value + passed = len(filtered) == 0 or (filtered[-1] is not None and filtered[-1] == rule_parameters.actual_value) or filtered[-1] == None return RuleExecutionResult(passed, expected_value, lower_bound, upper_bound) \ No newline at end of file From 84e5d418be49c65868ba103bad44d21f5b923fea Mon Sep 17 00:00:00 2001 From: Piotr Czarnas Date: Mon, 21 Aug 2023 17:13:34 +0200 Subject: [PATCH 4/6] Table model has a field to edit the result truncation for advanced profiling. --- .../rest/models/metadata/TableBasicModel.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dqops/src/main/java/com/dqops/rest/models/metadata/TableBasicModel.java b/dqops/src/main/java/com/dqops/rest/models/metadata/TableBasicModel.java index 9921a4ba62..babdb42aff 100644 --- a/dqops/src/main/java/com/dqops/rest/models/metadata/TableBasicModel.java +++ b/dqops/src/main/java/com/dqops/rest/models/metadata/TableBasicModel.java @@ -16,6 +16,8 @@ package com.dqops.rest.models.metadata; import com.dqops.checks.CheckType; +import com.dqops.checks.ProfilingTimePeriod; +import com.dqops.checks.table.profiling.TableProfilingCheckCategoriesSpec; import com.dqops.core.jobqueue.jobs.data.DeleteStoredDataQueueJobParameters; import com.dqops.metadata.search.CheckSearchFilters; import com.dqops.metadata.search.StatisticsCollectorSearchFilters; @@ -88,6 +90,15 @@ public class TableBasicModel { @JsonPropertyDescription("Table owner information like the data steward name or the business application name.") private TableOwnerSpec owner; + /** + * Defines how many advanced profiling results are stored for the table monthly. By default, DQO will use the 'one_per_month' configuration and store only the most recent + * advanced profiling result executed during the month. By changing this value, it is possible to store one value per day or even store all advanced profiling results. + */ + @JsonPropertyDescription("Defines how many advanced profiling results are stored for the table monthly. By default, DQO will use the 'one_per_month' configuration and store only the most recent " + + "advanced profiling result executed during the month. By changing this value, it is possible to store one value per day or even store all advanced profiling results.") + @JsonInclude(JsonInclude.Include.NON_NULL) + private ProfilingTimePeriod advancedProfilingResultTruncation; + /** * True when the table has any checks configured. */ @@ -172,6 +183,7 @@ public static TableBasicModel fromTableSpecificationForListEntry(String connecti setTableHash(tableSpec.getHierarchyId() != null ? tableSpec.getHierarchyId().hashCode64() : null); setTarget(tableSpec.getPhysicalTableName()); setDisabled(tableSpec.isDisabled()); + setAdvancedProfilingResultTruncation(tableSpec.getProfilingChecks() != null ? tableSpec.getProfilingChecks().getResultTruncation() : null); setPartitioningConfigurationMissing(tableSpec.getTimestampColumns() == null || Strings.isNullOrEmpty(tableSpec.getTimestampColumns().getPartitionByColumn())); setHasAnyConfiguredChecks(tableSpec.hasAnyChecksConfigured()); @@ -230,6 +242,7 @@ public static TableBasicModel fromTableSpecification(String connectionName, Tabl setFilter(tableSpec.getFilter()); setPriority(tableSpec.getPriority()); setOwner(tableSpec.getOwner()); + setAdvancedProfilingResultTruncation(tableSpec.getProfilingChecks() != null ? tableSpec.getProfilingChecks().getResultTruncation() : null); setPartitioningConfigurationMissing(tableSpec.getTimestampColumns() == null || Strings.isNullOrEmpty(tableSpec.getTimestampColumns().getPartitionByColumn())); setHasAnyConfiguredChecks(tableSpec.hasAnyChecksConfigured()); @@ -289,5 +302,10 @@ public void copyToTableSpecification(TableSpec targetTableSpec) { targetTableSpec.setFilter(this.getFilter()); targetTableSpec.setPriority(this.getPriority()); targetTableSpec.setOwner(this.getOwner()); + + if (targetTableSpec.getProfilingChecks() == null) { + targetTableSpec.setProfilingChecks(new TableProfilingCheckCategoriesSpec()); + } + targetTableSpec.getProfilingChecks().setResultTruncation(this.advancedProfilingResultTruncation); } } From 13e07b4442592cda74e93d954f1eff3fb56edb1f Mon Sep 17 00:00:00 2001 From: Piotr Czarnas Date: Mon, 21 Aug 2023 17:42:45 +0200 Subject: [PATCH 5/6] Unit test for sensor docs generation updated after adding a tested oracle sensor. --- .../docs/sensors/SensorDocumentationModelFactoryImplTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dqops/src/test/java/com/dqops/utils/docs/sensors/SensorDocumentationModelFactoryImplTests.java b/dqops/src/test/java/com/dqops/utils/docs/sensors/SensorDocumentationModelFactoryImplTests.java index ae3429bfba..310e2ce229 100644 --- a/dqops/src/test/java/com/dqops/utils/docs/sensors/SensorDocumentationModelFactoryImplTests.java +++ b/dqops/src/test/java/com/dqops/utils/docs/sensors/SensorDocumentationModelFactoryImplTests.java @@ -61,7 +61,7 @@ void createSensorDocumentation_whenCalledForClassWithSensorParameterFields_thenC Assertions.assertTrue(sensorDocumentation.getSqlTemplates().keySet().stream() .map(ProviderTypeModel::getProviderTypeName) .anyMatch(provider -> provider.equals("bigquery"))); - Assertions.assertEquals(6,sensorDocumentation.getSqlTemplates().values().size()); + Assertions.assertEquals(7,sensorDocumentation.getSqlTemplates().values().size()); Assertions.assertNotNull(sensorDocumentation.getDefinition()); Assertions.assertEquals(1, sensorDocumentation.getDefinition().getSpec().getFields().size()); From 9c93fcc63512dd93e0dac935922674980ecc75f0 Mon Sep 17 00:00:00 2001 From: Piotr Czarnas Date: Mon, 21 Aug 2023 17:50:21 +0200 Subject: [PATCH 6/6] Version updated to 0.2.1. --- .run/dqo run.run.xml | 2 +- VERSION | 2 +- distribution/pom.xml | 2 +- .../python/dqops/client/models/__init__.py | 4 + ...aly_differencing_sum_30_days_check_spec.py | 12 +- ...fferencing_row_count_30_days_check_spec.py | 12 +- .../dqops/client/models/table_basic_model.py | 37 + ...el_advanced_profiling_result_truncation.py | 12 + distribution/python/dqops/version.py | 4 +- distribution/python/setup.py | 3 +- .../integrity/foreign-key-match-percent.md | 650 ++++++++++++++ .../integrity/foreign-key-not-match-count.md | 690 +++++++++++++++ .../strings/string-invalid-email-count.md | 500 +++++++++++ .../string-invalid-ip4-address-count.md | 520 +++++++++++ .../string-match-date-regex-percent.md | 730 ++++++++++++++++ .../string-match-name-regex-percent.md | 580 +++++++++++++ .../strings/string-match-regex-percent.md | 670 +++++++++++++++ .../string-not-match-date-regex-count.md | 730 ++++++++++++++++ .../strings/string-not-match-regex-count.md | 670 +++++++++++++++ .../string-surrounded-by-whitespace-count.md | 550 ++++++++++++ ...string-surrounded-by-whitespace-percent.md | 610 +++++++++++++ .../string-valid-country-code-percent.md | 560 ++++++++++++ .../string-valid-currency-code-percent.md | 570 +++++++++++++ .../strings/string-value-in-set-percent.md | 806 +++++++++++++++++- .../table/availability/table-availability.md | 2 +- docs/reference/rules/Comparison.md | 26 +- .../column/integrity-column-sensors.md | 76 ++ .../sensors/column/strings-column-sensors.md | 395 +++++++++ .../table/availability-table-sensors.md | 2 +- docs/reference/yaml/ConnectionYaml.md | 228 ++--- docs/reference/yaml/DashboardYaml.md | 64 +- docs/reference/yaml/RuleDefinitionYaml.md | 44 +- docs/reference/yaml/SensorDefinitionYaml.md | 20 +- docs/reference/yaml/SettingsYaml.md | 152 ++-- docs/reference/yaml/TableYaml.md | 424 ++++----- .../column-daily-partitioned-checks.md | 330 +++---- .../column-monthly-partitioned-checks.md | 290 +++---- .../table-daily-partitioned-checks.md | 78 +- .../table-monthly-partitioned-checks.md | 46 +- .../yaml/profiling/column-profiling-checks.md | 506 +++++------ .../yaml/profiling/table-profiling-checks.md | 200 ++--- .../column-daily-recurring-checks.md | 262 +++--- .../column-monthly-recurring-checks.md | 290 +++---- .../recurring/table-daily-recurring-checks.md | 108 +-- .../table-monthly-recurring-checks.md | 92 +- dqo | 2 +- dqo.cmd | 2 +- dqops/pom.xml | 2 +- dqops/src/main/frontend/package.json | 2 +- dqops/src/main/resources/banner.txt | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- .../oracle.dqoprovidersensor.yaml | 2 +- lib/pom.xml | 2 +- pom.xml | 2 +- 64 files changed, 10952 insertions(+), 1647 deletions(-) create mode 100644 distribution/python/dqops/client/models/table_basic_model_advanced_profiling_result_truncation.py diff --git a/.run/dqo run.run.xml b/.run/dqo run.run.xml index a33953c9b0..27a4013f6b 100644 --- a/.run/dqo run.run.xml +++ b/.run/dqo run.run.xml @@ -1,6 +1,6 @@ -