use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.
the class TestFillEmpties method doFillEmptiesScalar.
private void doFillEmptiesScalar(MajorType majorType) {
TupleMetadata schema = new SchemaBuilder().add("a", majorType).buildSchema();
ExtendableRowSet rs = fixture.rowSet(schema);
RowSetWriter writer = rs.writer();
ScalarWriter colWriter = writer.scalar(0);
ValueType valueType = colWriter.valueType();
boolean nullable = majorType.getMode() == DataMode.OPTIONAL;
for (int i = 0; i < ROW_COUNT; i++) {
if (i % 5 == 0) {
colWriter.setObject(RowSetUtilities.testDataFromInt(valueType, majorType, i));
}
writer.save();
}
SingleRowSet result = writer.done();
RowSetReader reader = result.reader();
ScalarReader colReader = reader.scalar(0);
MinorType type = majorType.getMinorType();
boolean isVariable = Types.isVarWidthType(type);
for (int i = 0; i < ROW_COUNT; i++) {
assertTrue(reader.next());
if (i % 5 != 0) {
if (nullable) {
// Nullable types fill with nulls.
assertTrue(colReader.isNull());
continue;
}
if (isVariable) {
// Variable width types fill with a zero-length value.
assertEquals(0, colReader.getBytes().length);
continue;
}
}
// All other types fill with zero-bytes, interpreted as some form
// of zero for each type.
Object actual = colReader.getObject();
Object expected = RowSetUtilities.testDataFromInt(valueType, majorType, i % 5 == 0 ? i : 0);
RowSetUtilities.assertEqualValues(majorType.toString().replace('\n', ' ') + "[" + i + "]", valueType, expected, actual);
}
result.clear();
}
use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.
the class TestFixedWidthWriter method testSizeLimit.
/**
* Test resize monitoring. Add a listener to an int writer,
* capture each resize, and refuse a resize when the number
* of ints exceeds 8K values. This will trigger an overflow,
* which will throw an exception which we then check for.
*/
@Test
public void testSizeLimit() {
try (IntVector vector = allocVector(1000)) {
TestIndex index = new TestIndex();
IntColumnWriter writer = makeWriter(vector, index);
writer.bindListener(new ColumnWriterListener() {
int totalAlloc = 4096;
@Override
public void overflowed(ScalarWriter writer) {
throw new IllegalStateException("overflow called");
}
@Override
public boolean canExpand(ScalarWriter writer, int delta) {
totalAlloc += delta;
return totalAlloc < 16_384 * 4;
}
});
writer.startWrite();
try {
for (int i = 0; ; i++) {
index.index = i;
writer.startRow();
writer.setInt(i);
writer.saveRow();
}
} catch (IllegalStateException e) {
assertTrue(e.getMessage().contains("overflow called"));
}
// Should have failed on 8192, which doubled vector
// to 16K, which was rejected.
assertEquals(8192, index.index);
}
}
use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.
the class TestRowSet method testRepeatedDictStructure.
@Test
public void testRepeatedDictStructure() {
final String dictName = "d";
final TupleMetadata schema = new SchemaBuilder().add("id", MinorType.INT).addDictArray(dictName, MinorType.FLOAT8).value(MinorType.VARCHAR).resumeSchema().buildSchema();
final ExtendableRowSet rowSet = fixture.rowSet(schema);
final RowSetWriter writer = rowSet.writer();
// Repeated dict
assertEquals(ObjectType.ARRAY, writer.column(dictName).type());
final ScalarWriter idWriter = writer.scalar(0);
final ArrayWriter dictArrayWriter = writer.column(1).array();
assertEquals(ObjectType.ARRAY, dictArrayWriter.entryType());
DictWriter dictWriter = dictArrayWriter.dict();
assertEquals(ValueType.DOUBLE, dictWriter.keyType());
assertEquals(ObjectType.SCALAR, dictWriter.valueType());
final ScalarWriter keyWriter = dictWriter.keyWriter();
final ScalarWriter valueWriter = dictWriter.valueWriter().scalar();
assertEquals(ValueType.DOUBLE, keyWriter.valueType());
assertEquals(ValueType.STRING, valueWriter.valueType());
// Write data
idWriter.setInt(1);
keyWriter.setDouble(1);
valueWriter.setString("a");
// advance to next entry position
dictWriter.save();
keyWriter.setDouble(2);
valueWriter.setString("b");
dictWriter.save();
// advance to next array position
dictArrayWriter.save();
keyWriter.setDouble(3);
valueWriter.setString("c");
dictWriter.save();
dictArrayWriter.save();
// advance to next row
writer.save();
idWriter.setInt(2);
keyWriter.setDouble(11);
valueWriter.setString("d");
dictWriter.save();
keyWriter.setDouble(12);
valueWriter.setString("e");
dictWriter.save();
dictArrayWriter.save();
writer.save();
idWriter.setInt(3);
keyWriter.setDouble(21);
valueWriter.setString("f");
dictWriter.save();
keyWriter.setDouble(22);
valueWriter.setString("g");
dictWriter.save();
keyWriter.setDouble(23);
valueWriter.setString("h");
dictWriter.save();
dictArrayWriter.save();
keyWriter.setDouble(24);
valueWriter.setString("i");
dictWriter.save();
keyWriter.setDouble(25);
valueWriter.setString("j");
dictWriter.save();
keyWriter.setDouble(26.5);
valueWriter.setString("k");
dictWriter.save();
keyWriter.setDouble(27);
valueWriter.setString("l");
dictWriter.save();
keyWriter.setDouble(28);
valueWriter.setString("m");
dictWriter.save();
dictArrayWriter.save();
writer.save();
// Finish the row set and get a reader.
final SingleRowSet actual = writer.done();
final RowSetReader reader = actual.reader();
// Verify reader structure
assertEquals(ObjectType.ARRAY, reader.column(dictName).type());
final ArrayReader dictArrayReader = reader.array(1);
assertEquals(ObjectType.ARRAY, dictArrayReader.entryType());
final DictReader dictReader = dictArrayReader.entry().dict();
assertEquals(ValueType.DOUBLE, dictReader.keyColumnType());
assertEquals(ObjectType.SCALAR, dictReader.valueColumnType());
final KeyAccessor keyAccessor = dictReader.keyAccessor();
final ScalarReader valueReader = dictReader.valueReader().scalar();
// Row 1
assertTrue(reader.next());
// array is not null
assertFalse(dictArrayReader.isNull());
assertTrue(dictArrayReader.next());
// first dict is not null
assertFalse(dictArrayReader.isNull());
assertTrue(keyAccessor.find(2.0f));
assertEquals("b", valueReader.getObject());
assertTrue(keyAccessor.find(1.0f));
assertEquals("a", valueReader.getObject());
// no entry for given key
assertFalse(keyAccessor.find(1.1f));
assertTrue(dictArrayReader.next());
assertTrue(keyAccessor.find(3.0f));
assertEquals("c", valueReader.getObject());
assertFalse(keyAccessor.find(1.0f));
assertEquals(Arrays.asList(map(1.0, "a", 2.0, "b"), map(3.0, "c")), dictArrayReader.getObject());
// Row 2
assertTrue(reader.next());
assertTrue(dictArrayReader.next());
assertTrue(keyAccessor.find(11.0f));
assertEquals("d", valueReader.getString());
assertFalse(keyAccessor.find(1.0f));
assertTrue(keyAccessor.find(12.0f));
assertEquals("e", valueReader.getString());
// Row 3: use explicit positioning
assertTrue(reader.next());
dictArrayReader.setPosn(1);
assertTrue(keyAccessor.find(24.0f));
assertEquals("i", valueReader.getString());
assertTrue(keyAccessor.find(26.5f));
assertEquals("k", valueReader.getString());
assertTrue(keyAccessor.find(28.0f));
assertEquals("m", valueReader.getString());
assertFalse(keyAccessor.find(35.0f));
assertTrue(keyAccessor.find(27.0f));
assertEquals("l", valueReader.getString());
Map<Object, Object> element1 = map(24.0, "i", 25.0, "j", 26.5, "k", 27.0, "l", 28.0, "m");
assertEquals(element1, dictReader.getObject());
dictArrayReader.setPosn(0);
assertTrue(keyAccessor.find(23.0f));
assertEquals("h", valueReader.getObject());
assertTrue(keyAccessor.find(21.0f));
assertEquals("f", valueReader.getObject());
assertFalse(keyAccessor.find(23.05f));
Map<Object, Object> element0 = map(21.0, "f", 22.0, "g", 23.0, "h");
assertEquals(element0, dictReader.getObject());
assertEquals(Arrays.asList(element0, element1), dictArrayReader.getObject());
assertFalse(reader.next());
// Verify that the dict accessor's value count was set.
final RepeatedDictVector vector = (RepeatedDictVector) actual.container().getValueVector(1).getValueVector();
assertEquals(3, vector.getAccessor().getValueCount());
final SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(1, objArray(map(1.0f, "a", 2.0f, "b"), map(3.0f, "c"))).addRow(2, objArray(singleObjArray(map(11.0f, "d", 12.0f, "e")))).addRow(3, objArray(map(21.0f, "f", 22.0f, "g", 23.0f, "h"), map(24.0f, "i", 25.0f, "j", 26.5f, "k", 27.0f, "l", 28.0f, "m"))).build();
RowSetUtilities.verify(expected, actual);
}
use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.
the class TestRowSet method testMapStructure.
/**
* Test a simple map structure at the top level of a row.
*
* @throws VectorOverflowException should never occur
*/
@Test
public void testMapStructure() {
final TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).addMap("m").addArray("b", MinorType.INT).resumeSchema().buildSchema();
final ExtendableRowSet rowSet = fixture.rowSet(schema);
final RowSetWriter writer = rowSet.writer();
// Map and Int
// Test Invariants
assertEquals(ObjectType.SCALAR, writer.column("a").type());
assertEquals(ObjectType.SCALAR, writer.column(0).type());
assertEquals(ObjectType.TUPLE, writer.column("m").type());
assertEquals(ObjectType.TUPLE, writer.column(1).type());
assertSame(writer.column(1).tuple(), writer.tuple(1));
final TupleWriter mapWriter = writer.column(1).tuple();
assertEquals(ObjectType.SCALAR, mapWriter.column("b").array().entry().type());
assertEquals(ObjectType.SCALAR, mapWriter.column("b").array().entryType());
final ScalarWriter aWriter = writer.column("a").scalar();
final ScalarWriter bWriter = writer.column("m").tuple().column("b").array().entry().scalar();
assertSame(bWriter, writer.tuple(1).array(0).scalar());
assertEquals(ValueType.INTEGER, bWriter.valueType());
try {
writer.column(1).scalar();
fail();
} catch (final UnsupportedOperationException e) {
// Expected
}
try {
writer.column(1).array();
fail();
} catch (final UnsupportedOperationException e) {
// Expected
}
// Write data
aWriter.setInt(10);
bWriter.setInt(11);
bWriter.setInt(12);
writer.save();
aWriter.setInt(20);
bWriter.setInt(21);
bWriter.setInt(22);
writer.save();
aWriter.setInt(30);
bWriter.setInt(31);
bWriter.setInt(32);
writer.save();
// Finish the row set and get a reader.
final SingleRowSet actual = writer.done();
final RowSetReader reader = actual.reader();
assertEquals(ObjectType.SCALAR, reader.column("a").type());
assertEquals(ObjectType.SCALAR, reader.column(0).type());
assertEquals(ObjectType.TUPLE, reader.column("m").type());
assertEquals(ObjectType.TUPLE, reader.column(1).type());
assertSame(reader.column(1).tuple(), reader.tuple(1));
final ScalarReader aReader = reader.column(0).scalar();
final TupleReader mReader = reader.column(1).tuple();
final ArrayReader bArray = mReader.column("b").array();
assertEquals(ObjectType.SCALAR, bArray.entryType());
final ScalarReader bReader = bArray.scalar();
assertEquals(ValueType.INTEGER, bReader.valueType());
// Row 1: (10, {[11, 12]})
assertTrue(reader.next());
assertEquals(10, aReader.getInt());
assertFalse(mReader.isNull());
assertTrue(bArray.next());
assertFalse(bReader.isNull());
assertEquals(11, bReader.getInt());
assertTrue(bArray.next());
assertFalse(bReader.isNull());
assertEquals(12, bReader.getInt());
assertFalse(bArray.next());
// Row 2: (20, {[21, 22]})
assertTrue(reader.next());
assertEquals(20, aReader.getInt());
assertFalse(mReader.isNull());
assertTrue(bArray.next());
assertEquals(21, bReader.getInt());
assertTrue(bArray.next());
assertEquals(22, bReader.getInt());
// Row 3: (30, {[31, 32]})
assertTrue(reader.next());
assertEquals(30, aReader.getInt());
assertFalse(mReader.isNull());
assertTrue(bArray.next());
assertEquals(31, bReader.getInt());
assertTrue(bArray.next());
assertEquals(32, bReader.getInt());
assertFalse(reader.next());
// Verify that the map accessor's value count was set.
final MapVector mapVector = (MapVector) actual.container().getValueVector(1).getValueVector();
assertEquals(actual.rowCount(), mapVector.getAccessor().getValueCount());
final SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(10, objArray(intArray(11, 12))).addRow(20, objArray(intArray(21, 22))).addRow(30, objArray(intArray(31, 32))).build();
RowSetUtilities.verify(expected, actual);
}
use of org.apache.drill.exec.vector.accessor.ScalarWriter in project drill by apache.
the class TestRowSet method testRepeatedMapStructure.
@Test
public void testRepeatedMapStructure() {
final TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).addMapArray("m").add("b", MinorType.INT).add("c", MinorType.INT).resumeSchema().buildSchema();
final ExtendableRowSet rowSet = fixture.rowSet(schema);
final RowSetWriter writer = rowSet.writer();
// Map and Int
// Pick out components and lightly test. (Assumes structure
// tested earlier is still valid, so no need to exhaustively
// test again.)
assertEquals(ObjectType.SCALAR, writer.column("a").type());
assertEquals(ObjectType.ARRAY, writer.column("m").type());
final ArrayWriter maWriter = writer.column(1).array();
assertEquals(ObjectType.TUPLE, maWriter.entryType());
final TupleWriter mapWriter = maWriter.tuple();
assertEquals(ObjectType.SCALAR, mapWriter.column("b").type());
assertEquals(ObjectType.SCALAR, mapWriter.column("c").type());
final ScalarWriter aWriter = writer.column("a").scalar();
final ScalarWriter bWriter = mapWriter.scalar("b");
final ScalarWriter cWriter = mapWriter.scalar("c");
assertEquals(ValueType.INTEGER, aWriter.valueType());
assertEquals(ValueType.INTEGER, bWriter.valueType());
assertEquals(ValueType.INTEGER, cWriter.valueType());
// Write data
aWriter.setInt(10);
bWriter.setInt(101);
cWriter.setInt(102);
// Advance to next array position
maWriter.save();
bWriter.setInt(111);
cWriter.setInt(112);
maWriter.save();
writer.save();
aWriter.setInt(20);
bWriter.setInt(201);
cWriter.setInt(202);
maWriter.save();
bWriter.setInt(211);
cWriter.setInt(212);
maWriter.save();
writer.save();
aWriter.setInt(30);
bWriter.setInt(301);
cWriter.setInt(302);
maWriter.save();
bWriter.setInt(311);
cWriter.setInt(312);
maWriter.save();
writer.save();
// Finish the row set and get a reader.
final SingleRowSet actual = writer.done();
final RowSetReader reader = actual.reader();
// Verify reader structure
assertEquals(ObjectType.SCALAR, reader.column("a").type());
assertEquals(ObjectType.ARRAY, reader.column("m").type());
final ArrayReader maReader = reader.column(1).array();
assertEquals(ObjectType.TUPLE, maReader.entryType());
final TupleReader mapReader = maReader.tuple();
assertEquals(ObjectType.SCALAR, mapReader.column("b").type());
assertEquals(ObjectType.SCALAR, mapReader.column("c").type());
final ScalarReader aReader = reader.column("a").scalar();
final ScalarReader bReader = mapReader.scalar("b");
final ScalarReader cReader = mapReader.scalar("c");
assertEquals(ValueType.INTEGER, aReader.valueType());
assertEquals(ValueType.INTEGER, bReader.valueType());
assertEquals(ValueType.INTEGER, cReader.valueType());
// Row 1: Use iterator-like accessors
assertTrue(reader.next());
assertEquals(10, aReader.getInt());
// Array itself is not null
assertFalse(maReader.isNull());
assertTrue(maReader.next());
// Tuple 0 is not null
assertFalse(mapReader.isNull());
assertEquals(101, mapReader.scalar(0).getInt());
assertEquals(102, mapReader.scalar(1).getInt());
assertTrue(maReader.next());
assertEquals(111, mapReader.scalar(0).getInt());
assertEquals(112, mapReader.scalar(1).getInt());
// Row 2: use explicit positioning,
// but access scalars through the map reader.
assertTrue(reader.next());
assertEquals(20, aReader.getInt());
maReader.setPosn(0);
assertEquals(201, mapReader.scalar(0).getInt());
assertEquals(202, mapReader.scalar(1).getInt());
maReader.setPosn(1);
assertEquals(211, mapReader.scalar(0).getInt());
assertEquals(212, mapReader.scalar(1).getInt());
// Row 3: use scalar accessor
assertTrue(reader.next());
assertEquals(30, aReader.getInt());
assertTrue(maReader.next());
assertEquals(301, bReader.getInt());
assertEquals(302, cReader.getInt());
assertTrue(maReader.next());
assertEquals(311, bReader.getInt());
assertEquals(312, cReader.getInt());
assertFalse(reader.next());
// Verify that the map accessor's value count was set.
final RepeatedMapVector mapVector = (RepeatedMapVector) actual.container().getValueVector(1).getValueVector();
assertEquals(3, mapVector.getAccessor().getValueCount());
// Verify the readers and writers again using the testing tools.
final SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(10, objArray(objArray(101, 102), objArray(111, 112))).addRow(20, objArray(objArray(201, 202), objArray(211, 212))).addRow(30, objArray(objArray(301, 302), objArray(311, 312))).build();
RowSetUtilities.verify(expected, actual);
}
Aggregations