Skip to content

Commit

Permalink
Replace random test values by discrete ones
Browse files Browse the repository at this point in the history 
The use of Random() function in the parquet.batchreader.decoders tests may cause flakiness.

Adds TestMode.java to parameterize tests with an arbitrary value and an upper-/lower-bounded value.

Resolves: prestodb#23840
  • Loading branch information
@ges1227
ges1227 committed Nov 6, 2024
1 parent c16711a commit 89ba2ab
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 94 deletions.
11 changes: 5 additions & 6 deletions ...java/com/facebook/presto/parquet/batchreader/decoders/TestFlatDefinitionLevelDecoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,10 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;

import static com.facebook.presto.parquet.batchreader.decoders.TestParquetUtils.addDLRLEBlock;
import static com.facebook.presto.parquet.batchreader.decoders.TestParquetUtils.addDLValues;
import static com.facebook.presto.parquet.batchreader.decoders.TestParquetUtils.randomValues;
import static com.facebook.presto.parquet.batchreader.decoders.TestParquetUtils.fillValues;
import static java.lang.Math.min;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;
Expand All @@ -35,19 +34,19 @@ public class TestFlatDefinitionLevelDecoder
private static int valueCount;
private static int nonNullCount;
private static byte[] pageBytes;
private static List<Integer> expectedValues = new ArrayList<>();
private static final List<Integer> expectedValues = new ArrayList<>();

@BeforeClass
public void setup()
throws IOException
{
Random random = new Random(200);
TestMode testMode = TestMode.ARBITRARY;
RunLengthBitPackingHybridEncoder encoder = TestParquetUtils.getSimpleDLEncoder();

addDLRLEBlock(1, 50, encoder, expectedValues);
addDLValues(randomValues(random, 457, 1), encoder, expectedValues);
addDLValues(fillValues(testMode, 457, 1), encoder, expectedValues);
addDLRLEBlock(0, 37, encoder, expectedValues);
addDLValues(randomValues(random, 186, 1), encoder, expectedValues);
addDLValues(fillValues(testMode, 186, 1), encoder, expectedValues);

valueCount = expectedValues.size();
for (Integer value : expectedValues) {
Expand Down
38 changes: 38 additions & 0 deletions presto-parquet/src/test/java/com/facebook/presto/parquet/batchreader/decoders/TestMode.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package com.facebook.presto.parquet.batchreader.decoders;

public enum TestMode
{
UPPER_BOUND(Integer.MAX_VALUE, Long.MAX_VALUE), LOWER_BOUND(Integer.MIN_VALUE, 0), ARBITRARY(237, 237 * (1L << 31));

private final int testInt;
private final long testLong;

TestMode(int i, long l)
{
this.testInt = i;
this.testLong = l;
}

public int getInt()
{
return testInt;
}

public long getLong()
{
return testLong;
}

public int getPositiveUpperBoundedInt(int upper)
{
if (this.name().equals(LOWER_BOUND.name()) || upper <= 0) {
return 0;
}

if (this.name().equals(UPPER_BOUND.name())) {
return upper;
}

return ARBITRARY.testInt % upper;
}
}
33 changes: 15 additions & 18 deletions ...quet/src/test/java/com/facebook/presto/parquet/batchreader/decoders/TestParquetUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
*/
package com.facebook.presto.parquet.batchreader.decoders;

import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime;
import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTimeUtils;
import org.apache.parquet.bytes.BytesUtils;
Expand All @@ -31,7 +30,6 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Random;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
Expand Down Expand Up @@ -80,16 +78,16 @@ public static void addDLValues(Iterator<Integer> values, RunLengthBitPackingHybr
}
}

public static Iterator<Integer> randomValues(Random random, int numValues, int maxValue)
public static Iterator<Integer> fillValues(TestMode testMode, int numValues, int maxValue)
{
List<Integer> values = new ArrayList<>();
for (int i = 0; i < numValues; i++) {
values.add(random.nextInt(maxValue + 1));
values.add(testMode.getPositiveUpperBoundedInt(maxValue));
}
return values.iterator();
}

public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits, Random random, List<Object> addedValues)
public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits, TestMode testMode, List<Object> addedValues)
{
ValuesWriter writer;

Expand All @@ -103,15 +101,15 @@ public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits,
switch (valueSizeBits) {
case 1: {
for (int i = 0; i < valueCount; i++) {
int value = random.nextInt(2);
int value = testMode.getPositiveUpperBoundedInt(1);
writer.writeInteger(value);
addedValues.add(value);
}
break;
}
case -1: {
for (int i = 0; i < valueCount; i++) {
String valueStr = RandomStringUtils.random(random.nextInt(10), 0, 0, true, true, null, random);
String valueStr = "4nY" + valueCount;
byte[] valueUtf8 = valueStr.getBytes(StandardCharsets.UTF_8);
writer.writeBytes(Binary.fromConstantByteArray(valueUtf8, 0, valueUtf8.length));
addedValues.add(valueStr);
Expand All @@ -120,23 +118,23 @@ public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits,
}
case 32: {
for (int i = 0; i < valueCount; i++) {
int value = random.nextInt();
int value = testMode.getInt();
writer.writeInteger(value);
addedValues.add(value);
}
break;
}
case 64: {
for (int i = 0; i < valueCount; i++) {
long value = random.nextLong();
long value = testMode.getLong();
writer.writeLong(value);
addedValues.add(value);
}
break;
}
case 96: {
for (int i = 0; i < valueCount; i++) {
long millisValue = Long.valueOf(random.nextInt(1572281176) * 1000);
long millisValue = testMode.getPositiveUpperBoundedInt(1572281175) * 1000L;
NanoTime nanoTime = NanoTimeUtils.getNanoTime(new Timestamp(millisValue), false);
writer.writeLong(nanoTime.getTimeOfDayNanos());
writer.writeInteger(nanoTime.getJulianDay());
Expand All @@ -146,10 +144,9 @@ public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits,
}
case 128:
for (int i = 0; i < valueCount; i++) {
long value = random.nextLong();
long value = testMode.getLong();
writer.writeLong(value);
addedValues.add(value);
value = random.nextLong();
writer.writeLong(value);
addedValues.add(value);
}
Expand All @@ -166,19 +163,19 @@ public static byte[] generatePlainValuesPage(int valueCount, int valueSizeBits,
}
}

public static byte[] generateDictionaryIdPage2048(int maxValue, Random random, List<Integer> addedValues)
public static byte[] generateDictionaryIdPage2048(int maxValue, TestMode testMode, List<Integer> addedValues)
{
RunLengthBitPackingHybridEncoder encoder = getDictionaryDataPageEncoder(maxValue);

addDLRLEBlock(maxValue / 2, 50, encoder, addedValues);
addDLValues(randomValues(random, 457, maxValue), encoder, addedValues);
addDLValues(fillValues(testMode, 457, maxValue), encoder, addedValues);
addDLRLEBlock(0, 37, encoder, addedValues);
addDLValues(randomValues(random, 186, maxValue), encoder, addedValues);
addDLValues(randomValues(random, 289, maxValue), encoder, addedValues);
addDLValues(fillValues(testMode, 186, maxValue), encoder, addedValues);
addDLValues(fillValues(testMode, 289, maxValue), encoder, addedValues);
addDLRLEBlock(maxValue - 1, 76, encoder, addedValues);
addDLValues(randomValues(random, 789, maxValue), encoder, addedValues);
addDLValues(fillValues(testMode, 789, maxValue), encoder, addedValues);
addDLRLEBlock(maxValue - 1, 137, encoder, addedValues);
addDLValues(randomValues(random, 27, maxValue), encoder, addedValues);
addDLValues(fillValues(testMode, 27, maxValue), encoder, addedValues);

checkState(addedValues.size() == 2048);

Expand Down
Loading

0 comments on commit 89ba2ab

Please sign in to comment.