use of org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriterImpl in project drill by apache.
the class TestOffsetVectorWriter method makeWriter.
private OffsetVectorWriterImpl makeWriter(UInt4Vector vector, TestIndex index) {
OffsetVectorWriterImpl writer = new OffsetVectorWriterImpl(vector);
writer.bindIndex(index);
assertEquals(ValueType.INTEGER, writer.valueType());
return writer;
}
use of org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriterImpl in project drill by apache.
the class TestOffsetVectorWriter method testSizeLimit.
/**
* Test resize monitoring. Add a listener to an offsets writer,
* capture each resize, and refuse a resize when the number
* of ints exceeds 8K values. This will trigger an overflow,
* which will throw an exception which we then check for.
*/
@Test
public void testSizeLimit() {
try (UInt4Vector vector = allocVector(1000)) {
TestIndex index = new TestIndex();
OffsetVectorWriterImpl writer = makeWriter(vector, index);
writer.bindListener(new ColumnWriterListener() {
int totalAlloc = 4096;
@Override
public void overflowed(ScalarWriter writer) {
throw new IllegalStateException("overflow called");
}
@Override
public boolean canExpand(ScalarWriter writer, int delta) {
totalAlloc += delta;
return totalAlloc < 16_384 * 4;
}
});
writer.startWrite();
try {
for (int i = 0; ; i++) {
index.index = i;
writer.startRow();
writer.setNextOffset(i);
writer.saveRow();
}
} catch (IllegalStateException e) {
assertTrue(e.getMessage().contains("overflow called"));
}
// Should have failed on 8191, which doubled vector
// to 16K, which was rejected. Note the 8191 value,
// because offsets are one ahead of the index.
assertEquals(8191, index.index);
}
}
use of org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriterImpl in project drill by apache.
the class TestOffsetVectorWriter method testRestartRow.
@Test
public void testRestartRow() {
try (UInt4Vector vector = allocVector(1000)) {
TestIndex index = new TestIndex();
OffsetVectorWriterImpl writer = makeWriter(vector, index);
writer.startWrite();
// Write rows, rewriting every other row.
writer.startRow();
index.index = 0;
for (int i = 0; i < 50; i++) {
if (i % 2 == 0) {
assertEquals(i == 0 ? 0 : (i - 1) * 10, writer.nextOffset());
writer.setNextOffset((i + 1) * 10);
writer.saveRow();
writer.startRow();
index.index++;
} else {
writer.setNextOffset((i + 1) * 10);
writer.restartRow();
}
}
writer.endWrite();
// Verify values
assertEquals(0, vector.getAccessor().get(0));
for (int i = 1; i < 25; i++) {
assertEquals((2 * i - 1) * 10, vector.getAccessor().get(i));
}
}
}
use of org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriterImpl in project drill by apache.
the class TestOffsetVectorWriter method testFillEmpties.
/**
* Offset vectors have specific behavior when back-filling missing values:
* the last offset must be carried forward into the missing slots. The
* slots cannot be zero-filled, or entries will end up with a negative
* length.
*/
@Test
public void testFillEmpties() {
try (UInt4Vector vector = allocVector(1000)) {
TestIndex index = new TestIndex();
OffsetVectorWriterImpl writer = makeWriter(vector, index);
writer.startWrite();
// Pretend to write offsets for values of width 10, but
// skip four out of five values, forcing backfill.
// The loop will cause the vector to double in size.
// The number of values is odd, forcing the writer to
// back-fill at the end as well as between values.
long origAddr = vector.getBuffer().addr();
for (int i = 5; i < 3001; i += 5) {
index.index = i;
writer.startRow();
int startOffset = writer.nextOffset();
assertEquals((i / 5 - 1) * 10, startOffset);
writer.setNextOffset(startOffset + 10);
writer.saveRow();
}
index.index = 3003;
writer.endWrite();
// Should have been reallocated.
assertNotEquals(origAddr, vector.getBuffer().addr());
for (int i = 0; i < 3004; i++) {
assertEquals(((i - 1) / 5) * 10, vector.getAccessor().get(i));
}
}
}
use of org.apache.drill.exec.vector.accessor.writer.OffsetVectorWriterImpl in project drill by apache.
the class TestOffsetVectorWriter method testRolloverWithEmpties.
/**
* Simulate the case in which the tail end of an overflow
* batch has empties. <tt>preRollover()</tt> should back-fill
* them with the next offset prior to rollover.
*/
@Test
public void testRolloverWithEmpties() {
try (UInt4Vector vector = allocVector(1000)) {
TestIndex index = new TestIndex();
OffsetVectorWriterImpl writer = makeWriter(vector, index);
writer.startWrite();
for (int i = 0; i < 10; i++) {
index.index = i;
writer.startRow();
writer.setNextOffset((i + 1) * 10);
writer.saveRow();
}
for (int i = 10; i < 15; i++) {
index.index = i;
writer.startRow();
writer.saveRow();
}
// Overflow occurs before writing the 16th row
index.index = 15;
writer.startRow();
// Overflow occurs. This should fill empty offsets.
writer.preRollover();
for (int i = 0; i < 11; i++) {
assertEquals("i = " + i, i * 10, vector.getAccessor().get(i));
}
for (int i = 11; i < 16; i++) {
assertEquals("i = " + i, 100, vector.getAccessor().get(i));
}
for (int i = 0; i < 20; i++) {
vector.getMutator().set(i, 0xdeadbeef);
}
// Simulate finishing the overflow row.
index.index++;
// Post rollover, slot 0 should be initialized.
// This is a rollover. This row must set the value
// for the new row 0 (which was presumably set/filled
// after the overflow.)
writer.postRollover();
index.index = 0;
writer.setNextOffset(0);
writer.saveRow();
for (int i = 1; i < 5; i++) {
index.index = i;
writer.startRow();
writer.saveRow();
}
for (int i = 5; i < 10; i++) {
index.index = i;
writer.startRow();
writer.setNextOffset((i - 4) * 10);
writer.saveRow();
}
writer.endWrite();
for (int i = 0; i < 6; i++) {
assertEquals("Index + " + i, 0, vector.getAccessor().get(i));
}
for (int i = 6; i < 11; i++) {
assertEquals("Index + " + i, (i - 5) * 10, vector.getAccessor().get(i));
}
}
}
Aggregations