use of org.apache.drill.exec.vector.accessor.ArrayReader in project drill by apache.
the class TestRowSet method testTopFixedWidthArray.
/**
* Test an array of ints (as an example fixed-width type)
* at the top level of a schema.
*/
@Test
public void testTopFixedWidthArray() {
final TupleMetadata schema = new SchemaBuilder().add("c", MinorType.INT).addArray("a", MinorType.INT).buildSchema();
final ExtendableRowSet rs1 = fixture.rowSet(schema);
final RowSetWriter writer = rs1.writer();
writer.scalar(0).setInt(10);
final ScalarWriter array = writer.array(1).scalar();
array.setInt(100);
array.setInt(110);
writer.save();
writer.scalar(0).setInt(20);
array.setInt(200);
array.setInt(120);
array.setInt(220);
writer.save();
writer.scalar(0).setInt(30);
writer.save();
final SingleRowSet result = writer.done();
final RowSetReader reader = result.reader();
final ArrayReader arrayReader = reader.array(1);
final ScalarReader elementReader = arrayReader.scalar();
assertTrue(reader.next());
assertEquals(10, reader.scalar(0).getInt());
assertEquals(2, arrayReader.size());
assertTrue(arrayReader.next());
assertEquals(100, elementReader.getInt());
assertTrue(arrayReader.next());
assertEquals(110, elementReader.getInt());
assertTrue(reader.next());
assertEquals(20, reader.scalar(0).getInt());
assertEquals(3, arrayReader.size());
assertTrue(arrayReader.next());
assertEquals(200, elementReader.getInt());
assertTrue(arrayReader.next());
assertEquals(120, elementReader.getInt());
assertTrue(arrayReader.next());
assertEquals(220, elementReader.getInt());
assertTrue(reader.next());
assertEquals(30, reader.scalar(0).getInt());
assertEquals(0, arrayReader.size());
assertFalse(reader.next());
final SingleRowSet rs2 = fixture.rowSetBuilder(schema).addRow(10, intArray(100, 110)).addRow(20, intArray(200, 120, 220)).addRow(30, null).build();
RowSetUtilities.verify(rs1, rs2);
}
use of org.apache.drill.exec.vector.accessor.ArrayReader in project drill by apache.
the class TestRowSet method testScalarArrayStructure.
/**
* Test a record with a top level array. The focus here is on the
* scalar array structure.
*
* @throws VectorOverflowException should never occur
*/
@Test
public void testScalarArrayStructure() {
final TupleMetadata schema = new SchemaBuilder().addArray("a", MinorType.INT).buildSchema();
final ExtendableRowSet rowSet = fixture.rowSet(schema);
final RowSetWriter writer = rowSet.writer();
// Repeated Int
// Verify the invariants of the "full" and "simple" access paths
assertEquals(ObjectType.ARRAY, writer.column("a").type());
assertSame(writer.column("a"), writer.column(0));
assertSame(writer.array("a"), writer.array(0));
assertSame(writer.column("a").array(), writer.array("a"));
assertSame(writer.column(0).array(), writer.array(0));
assertEquals(ObjectType.SCALAR, writer.column("a").array().entry().type());
assertEquals(ObjectType.SCALAR, writer.column("a").array().entryType());
assertSame(writer.array(0).entry().scalar(), writer.array(0).scalar());
assertEquals(ValueType.INTEGER, writer.array(0).scalar().valueType());
try {
writer.column(0).scalar();
fail();
} catch (final UnsupportedOperationException e) {
// Expected
}
try {
writer.column(0).tuple();
fail();
} catch (final UnsupportedOperationException e) {
// Expected
}
// Write some data
final ScalarWriter intWriter = writer.array("a").scalar();
intWriter.setInt(10);
intWriter.setInt(11);
writer.save();
intWriter.setInt(20);
intWriter.setInt(21);
intWriter.setInt(22);
writer.save();
intWriter.setInt(30);
writer.save();
intWriter.setInt(40);
intWriter.setInt(41);
writer.save();
// Finish the row set and get a reader.
final SingleRowSet actual = writer.done();
final RowSetReader reader = actual.reader();
// Verify the invariants of the "full" and "simple" access paths
assertEquals(ObjectType.ARRAY, writer.column("a").type());
assertSame(reader.column("a"), reader.column(0));
assertSame(reader.array("a"), reader.array(0));
assertSame(reader.column("a").array(), reader.array("a"));
assertSame(reader.column(0).array(), reader.array(0));
assertEquals(ObjectType.SCALAR, reader.column("a").array().entryType());
assertEquals(ValueType.INTEGER, reader.array(0).scalar().valueType());
// Read and verify the rows
final ArrayReader arrayReader = reader.array(0);
final ScalarReader intReader = arrayReader.scalar();
assertTrue(reader.next());
assertFalse(arrayReader.isNull());
assertEquals(2, arrayReader.size());
assertTrue(arrayReader.next());
assertEquals(10, intReader.getInt());
assertTrue(arrayReader.next());
assertEquals(11, intReader.getInt());
assertFalse(arrayReader.next());
assertTrue(reader.next());
assertFalse(arrayReader.isNull());
assertEquals(3, arrayReader.size());
assertTrue(arrayReader.next());
assertEquals(20, intReader.getInt());
assertTrue(arrayReader.next());
assertEquals(21, intReader.getInt());
assertTrue(arrayReader.next());
assertEquals(22, intReader.getInt());
assertFalse(arrayReader.next());
assertTrue(reader.next());
assertFalse(arrayReader.isNull());
assertEquals(1, arrayReader.size());
assertTrue(arrayReader.next());
assertEquals(30, intReader.getInt());
assertFalse(arrayReader.next());
assertTrue(reader.next());
assertFalse(arrayReader.isNull());
assertEquals(2, arrayReader.size());
assertTrue(arrayReader.next());
assertEquals(40, intReader.getInt());
assertTrue(arrayReader.next());
assertEquals(41, intReader.getInt());
assertFalse(arrayReader.next());
assertFalse(reader.next());
// Test the above again via the writer and reader
// utility classes.
final SingleRowSet expected = fixture.rowSetBuilder(schema).addSingleCol(intArray(10, 11)).addSingleCol(intArray(20, 21, 22)).addSingleCol(intArray(30)).addSingleCol(intArray(40, 41)).build();
RowSetUtilities.verify(expected, actual);
}
use of org.apache.drill.exec.vector.accessor.ArrayReader in project drill by apache.
the class TestResultSetLoaderOverflow method testMissingArrayValues.
/**
* Test the case that an array has "missing values" before the overflow.
*/
@Test
public void testMissingArrayValues() {
TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).addArray("c", MinorType.INT).buildSchema();
ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).readerSchema(schema).build();
ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
RowSetLoader rootWriter = rsLoader.writer();
byte[] value = new byte[512];
Arrays.fill(value, (byte) 'X');
int blankAfter = ValueVector.MAX_BUFFER_SIZE / 512 * 2 / 3;
ScalarWriter cWriter = rootWriter.array("c").scalar();
rsLoader.startBatch();
int rowId = 0;
while (rootWriter.start()) {
rootWriter.scalar("a").setInt(rowId);
rootWriter.scalar("b").setBytes(value, value.length);
if (rowId < blankAfter) {
for (int i = 0; i < 3; i++) {
cWriter.setInt(rowId * 3 + i);
}
}
rootWriter.save();
rowId++;
}
VectorContainer container = rsLoader.harvest();
BatchValidator.validate(container);
RowSet result = fixture.wrap(container);
assertEquals(rowId - 1, result.rowCount());
RowSetReader reader = result.reader();
ArrayReader cArray = reader.array("c");
ScalarReader cReader = cArray.scalar();
while (reader.next()) {
assertEquals(reader.offset(), reader.scalar("a").getInt());
assertArrayEquals(value, reader.scalar("b").getBytes());
if (reader.offset() < blankAfter) {
assertEquals(3, cArray.size());
for (int i = 0; i < 3; i++) {
assertTrue(cArray.next());
assertEquals(reader.offset() * 3 + i, cReader.getInt());
}
} else {
assertEquals(0, cArray.size());
}
}
result.clear();
rsLoader.close();
}
use of org.apache.drill.exec.vector.accessor.ArrayReader in project drill by apache.
the class TestResultSetLoaderOverflow method testSizeLimitOnArray.
/**
* Test a row with a single array column which overflows. Verifies
* that all the fiddly bits about offset vectors and so on works
* correctly. Run this test (the simplest case) if you change anything
* about the array handling code.
*/
@Test
public void testSizeLimitOnArray() {
TupleMetadata schema = new SchemaBuilder().addArray("s", MinorType.VARCHAR).buildSchema();
ResultSetOptions options = new ResultSetOptionBuilder().rowCountLimit(ValueVector.MAX_ROW_COUNT).readerSchema(schema).build();
ResultSetLoader rsLoader = new ResultSetLoaderImpl(fixture.allocator(), options);
RowSetLoader rootWriter = rsLoader.writer();
// Fill batch with rows of with a single array, three values each. Tack on
// a suffix to each so we can be sure the proper data is written and moved
// to the overflow batch.
rsLoader.startBatch();
byte[] value = new byte[473];
Arrays.fill(value, (byte) 'X');
String strValue = new String(value, Charsets.UTF_8);
int valuesPerArray = 13;
int count = 0;
{
int rowSize = 0;
int totalSize = 0;
while (rootWriter.start()) {
totalSize += rowSize;
rowSize = 0;
ScalarWriter array = rootWriter.array(0).scalar();
for (int i = 0; i < valuesPerArray; i++) {
String cellValue = strValue + (count + 1) + "." + i;
array.setString(cellValue);
rowSize += cellValue.length();
}
rootWriter.save();
count++;
}
// Row count should include the overflow row.
int expectedCount = count - 1;
// Size without overflow row should fit in the vector, size
// with overflow should not.
assertTrue(totalSize <= ValueVector.MAX_BUFFER_SIZE);
assertTrue(totalSize + rowSize > ValueVector.MAX_BUFFER_SIZE);
// Result should exclude the overflow row. Last row
// should hold the last full array.
VectorContainer container = rsLoader.harvest();
BatchValidator.validate(container);
RowSet result = fixture.wrap(container);
assertEquals(expectedCount, result.rowCount());
RowSetReader reader = result.reader();
reader.setPosition(expectedCount - 1);
ArrayReader arrayReader = reader.array(0);
ScalarReader strReader = arrayReader.scalar();
assertEquals(valuesPerArray, arrayReader.size());
for (int i = 0; i < valuesPerArray; i++) {
assertTrue(arrayReader.next());
String cellValue = strValue + (count - 1) + "." + i;
assertEquals(cellValue, strReader.getString());
}
result.clear();
}
// Next batch should start with the overflow row.
// The only row in this next batch should be the whole
// array being written at the time of overflow.
{
rsLoader.startBatch();
assertEquals(1, rootWriter.rowCount());
assertEquals(count, rsLoader.totalRowCount());
VectorContainer container = rsLoader.harvest();
BatchValidator.validate(container);
RowSet result = fixture.wrap(container);
assertEquals(1, result.rowCount());
RowSetReader reader = result.reader();
reader.next();
ArrayReader arrayReader = reader.array(0);
ScalarReader strReader = arrayReader.scalar();
assertEquals(valuesPerArray, arrayReader.size());
for (int i = 0; i < valuesPerArray; i++) {
assertTrue(arrayReader.next());
String cellValue = strValue + count + "." + i;
assertEquals(cellValue, strReader.getString());
}
result.clear();
}
rsLoader.close();
}
use of org.apache.drill.exec.vector.accessor.ArrayReader in project drill by apache.
the class TestScalarAccessors method longArrayTester.
private void longArrayTester(MinorType type) {
TupleMetadata schema = new SchemaBuilder().addArray("col", type).buildSchema();
SingleRowSet rs = fixture.rowSetBuilder(schema).addSingleCol(new long[] {}).addSingleCol(new long[] { 0, 20, 30 }).build();
assertEquals(2, rs.rowCount());
RowSetReader reader = rs.reader();
ArrayReader arrayReader = reader.array(0);
ScalarReader colReader = arrayReader.scalar();
assertEquals(ValueType.LONG, colReader.valueType());
assertTrue(reader.next());
assertEquals(0, arrayReader.size());
assertTrue(reader.next());
assertEquals(3, arrayReader.size());
assertTrue(arrayReader.next());
assertEquals(0, colReader.getLong());
if (colReader.extendedType() == ValueType.LONG) {
assertEquals(0L, colReader.getObject());
assertEquals("0", colReader.getAsString());
}
assertTrue(arrayReader.next());
assertEquals(20, colReader.getLong());
if (colReader.extendedType() == ValueType.LONG) {
assertEquals(20L, colReader.getObject());
assertEquals("20", colReader.getAsString());
}
assertTrue(arrayReader.next());
assertEquals(30, colReader.getLong());
if (colReader.extendedType() == ValueType.LONG) {
assertEquals(30L, colReader.getObject());
assertEquals("30", colReader.getAsString());
}
assertFalse(arrayReader.next());
if (colReader.extendedType() == ValueType.LONG) {
assertEquals("[0, 20, 30]", arrayReader.getAsString());
assertEquals(Arrays.asList(0L, 20L, 30L), arrayReader.getObject());
}
assertFalse(reader.next());
rs.clear();
}
Aggregations