use of org.apache.drill.test.rowSet.test.TestFixedWidthWriter.TestIndex in project drill by axbaretto.
the class TestVariableWidthWriter method testSizeLimit.
/**
* Test resize monitoring. Add a listener to an Varchar writer,
* capture each resize, and refuse a resize when the s
* of the vector exceeds 1 MB. This will trigger an overflow,
* which will throw an exception which we then check for.
*/
@Test
public void testSizeLimit() {
try (VarCharVector vector = allocVector(1000)) {
TestIndex index = new TestIndex();
VarCharColumnWriter writer = makeWriter(vector, index);
writer.bindListener(new ColumnWriterListener() {
// Because assumed array size is 10, so 10 * 1000 = 10,000
// rounded to 16K
int totalAlloc = 16384;
@Override
public void overflowed(ScalarWriter writer) {
throw new IllegalStateException("overflow called");
}
@Override
public boolean canExpand(ScalarWriter writer, int delta) {
// System.out.println("Delta: " + delta);
totalAlloc += delta;
return totalAlloc < 1024 * 1024;
}
});
writer.startWrite();
byte[] value = new byte[423];
Arrays.fill(value, (byte) 'X');
try {
for (int i = 0; ; i++) {
index.index = i;
writer.startRow();
writer.setBytes(value, value.length);
writer.saveRow();
}
} catch (IllegalStateException e) {
assertTrue(e.getMessage().contains("overflow called"));
}
}
}
use of org.apache.drill.test.rowSet.test.TestFixedWidthWriter.TestIndex in project drill by axbaretto.
the class TestVariableWidthWriter method testRolloverWithEmpties.
/**
* Simulate the case in which the tail end of an overflow
* batch has empties. <tt>preRollover()</tt> should back-fill
* them with the next offset prior to rollover.
*/
@Test
public void testRolloverWithEmpties() {
try (VarCharVector vector = allocVector(1000)) {
TestIndex index = new TestIndex();
VarCharColumnWriter writer = makeWriter(vector, index);
writer.startWrite();
// Simulate doing an overflow of 15 values,
// of which 5 are empty.
String base = "sample-value";
for (int i = 0; i < 10; i++) {
index.index = i;
writer.startRow();
writer.setString(base + i);
writer.saveRow();
}
for (int i = 10; i < 15; i++) {
index.index = i;
writer.startRow();
writer.saveRow();
}
// Overflow occurs before writing the 16th row
index.index = 15;
writer.startRow();
// Overflow occurs. This should fill empty offsets.
writer.preRollover();
for (int i = 0; i < 10; i++) {
assertEquals(base + i, stringAt(vector, i));
}
for (int i = 10; i < 15; i++) {
assertEquals("", stringAt(vector, i));
}
// Simulate rollover
byte[] dummy = new byte[] { (byte) 0x55 };
for (int i = 0; i < 500; i++) {
vector.getMutator().setSafe(i, dummy);
}
for (int i = 1; i < 15; i++) {
vector.getOffsetVector().getMutator().set(i, 0xdeadbeef);
}
vector.getMutator().setSafe(0, new byte[] {});
writer.postRollover();
index.index = 0;
writer.saveRow();
for (int i = 1; i < 5; i++) {
index.index = i;
writer.startRow();
writer.saveRow();
}
for (int i = 5; i < 10; i++) {
index.index = i;
writer.startRow();
writer.setString(base + (i + 20));
writer.saveRow();
}
writer.endWrite();
for (int i = 0; i < 5; i++) {
assertEquals("", stringAt(vector, i));
}
for (int i = 5; i < 10; i++) {
assertEquals(base + (i + 20), stringAt(vector, i));
}
}
}
use of org.apache.drill.test.rowSet.test.TestFixedWidthWriter.TestIndex in project drill by axbaretto.
the class TestVariableWidthWriter method testFillEmpties.
/**
* Filling empties in a variable-width row means carrying forward
* offsets (as tested elsewhere), leaving zero-length values.
*/
@Test
public void testFillEmpties() {
try (VarCharVector vector = allocVector(1000)) {
TestIndex index = new TestIndex();
VarCharColumnWriter writer = makeWriter(vector, index);
writer.startWrite();
// Write values, skipping four out of five positions,
// forcing backfill.
// The number of values is odd, forcing the writer to
// back-fill at the end as well as between values.
String base = "sample-value";
for (int i = 0; i < 501; i += 5) {
index.index = i;
writer.startRow();
writer.setString(base + i);
writer.saveRow();
}
// At end, vector index defined to point one past the
// last row. That is, the vector index gives the row count.
index.index = 504;
writer.endWrite();
for (int i = 0; i < 504; i++) {
assertEquals("Mismatch on " + i, (i % 5) == 0 ? base + i : "", stringAt(vector, i));
}
}
}
use of org.apache.drill.test.rowSet.test.TestFixedWidthWriter.TestIndex in project drill by axbaretto.
the class TestVariableWidthWriter method testSkipNulls.
/**
* Test the case in which a scalar vector is used in conjunction
* with a nullable bits vector. The nullable vector will call the
* <tt>skipNulls()</tt> method to avoid writing values for null
* entries. For variable-width, there is no difference between
* filling empties and skipping nulls: both result in zero-sized
* entries.
*/
@Test
public void testSkipNulls() {
try (VarCharVector vector = allocVector(1000)) {
TestIndex index = new TestIndex();
VarCharColumnWriter writer = makeWriter(vector, index);
writer.startWrite();
// Write values, skipping four out of five positions,
// skipping nulls.
// The number of values is odd, forcing the writer to
// skip nulls at the end as well as between values.
String base = "sample-value";
for (int i = 0; i < 3000; i += 5) {
index.index = i;
writer.startRow();
writer.skipNulls();
writer.setString(base + i);
writer.saveRow();
}
index.index = 3003;
writer.startRow();
writer.skipNulls();
writer.saveRow();
writer.endWrite();
for (int i = 0; i < 3000; i++) {
assertEquals("Mismatch at " + i, (i % 5) == 0 ? base + i : "", stringAt(vector, i));
}
}
}
use of org.apache.drill.test.rowSet.test.TestFixedWidthWriter.TestIndex in project drill by axbaretto.
the class TestOffsetVectorWriter method testRolloverWithEmpties.
/**
* Simulate the case in which the tail end of an overflow
* batch has empties. <tt>preRollover()</tt> should back-fill
* them with the next offset prior to rollover.
*/
@Test
public void testRolloverWithEmpties() {
try (UInt4Vector vector = allocVector(1000)) {
TestIndex index = new TestIndex();
OffsetVectorWriter writer = makeWriter(vector, index);
writer.startWrite();
for (int i = 0; i < 10; i++) {
index.index = i;
writer.startRow();
writer.setNextOffset((i + 1) * 10);
writer.saveRow();
}
for (int i = 10; i < 15; i++) {
index.index = i;
writer.startRow();
writer.saveRow();
}
// Overflow occurs before writing the 16th row
index.index = 15;
writer.startRow();
// Overflow occurs. This should fill empty offsets.
writer.preRollover();
for (int i = 0; i < 11; i++) {
assertEquals(i * 10, vector.getAccessor().get(i));
}
for (int i = 11; i < 16; i++) {
assertEquals("i = " + i, 100, vector.getAccessor().get(i));
}
for (int i = 0; i < 20; i++) {
vector.getMutator().set(i, 0xdeadbeef);
}
// Post rollover, slot 0 should be initialized.
// This is a rollover. This row must set the value
// for the new row 0 (which was presumably set/filled
// after the overflow.)
writer.postRollover();
index.index = 0;
writer.setNextOffset(0);
writer.saveRow();
for (int i = 1; i < 5; i++) {
index.index = i;
writer.startRow();
writer.saveRow();
}
for (int i = 5; i < 10; i++) {
index.index = i;
writer.startRow();
writer.setNextOffset((i - 4) * 10);
writer.saveRow();
}
writer.endWrite();
for (int i = 0; i < 6; i++) {
assertEquals(0, vector.getAccessor().get(i));
}
for (int i = 6; i < 11; i++) {
assertEquals((i - 5) * 10, vector.getAccessor().get(i));
}
}
}
Aggregations