Skip to content

Commit

Permalink
Fix StressTests Java BigQueryIO job flakiness (#32803)
Browse files Browse the repository at this point in the history
* Fix assert for at least once method

* Refactoring
  • Loading branch information
Amar3tto authored Oct 16, 2024
1 parent f7b987d commit 7311270
Showing 1 changed file with 7 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ public final class BigQueryIOST extends IOStressTestBase {
private static final String READ_ELEMENT_METRIC_NAME = "read_count";
private static final String STORAGE_WRITE_API_METHOD = "STORAGE_WRITE_API";
private static final String STORAGE_API_AT_LEAST_ONCE_METHOD = "STORAGE_API_AT_LEAST_ONCE";
private static final double STORAGE_API_AT_LEAST_ONCE_MAX_ALLOWED_DIFFERENCE_FRACTION = 0.00001;

private static BigQueryResourceManager resourceManager;
private static String tableName;
Expand Down Expand Up @@ -334,11 +335,14 @@ private void generateDataAndWrite(BigQueryIO.Write<byte[]> writeIO) throws IOExc

// Depending on writing method there might be duplicates on different sides (read or write).
if (configuration.writeMethod.equals(STORAGE_API_AT_LEAST_ONCE_METHOD)) {
long allowedDifference =
(long) (numRecords * STORAGE_API_AT_LEAST_ONCE_MAX_ALLOWED_DIFFERENCE_FRACTION);
long actualDifference = (long) numRecords - rowCount;
assertTrue(
String.format(
"Number of rows in the table (%d) is less than the expected number (%d). Missing records: %d",
rowCount, (long) numRecords, (long) numRecords - rowCount),
rowCount >= numRecords);
"Row difference (%d) exceeds the limit of %d. Rows: %d, Expected: %d",
actualDifference, allowedDifference, rowCount, (long) numRecords),
actualDifference <= allowedDifference);
} else {
assertTrue(
String.format(
Expand Down

0 comments on commit 7311270

Please sign in to comment.