use of org.apache.drill.exec.vector.complex.MapVector in project drill by apache.
the class ComplexWriterImpl method rootAsMap.
@Override
public MapWriter rootAsMap() {
switch(mode) {
case INIT:
MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class);
mapRoot = new SingleMapWriter(map, this, unionEnabled);
mapRoot.setPosition(idx());
mode = Mode.MAP;
break;
case MAP:
break;
default:
check(Mode.INIT, Mode.MAP);
}
return mapRoot;
}
use of org.apache.drill.exec.vector.complex.MapVector in project drill by apache.
the class TestValueVector method testVectors.
/**
* Convenience method that allows running tests on various {@link ValueVector vector} instances.
*
* @param test test function to execute
*/
private void testVectors(VectorVerifier test) throws Exception {
final MaterializedField[] fields = { MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedListVector.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, MapVector.TYPE), MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedMapVector.TYPE) };
final ValueVector[] vectors = { new UInt4Vector(fields[0], allocator), new BitVector(fields[1], allocator), new VarCharVector(fields[2], allocator), new NullableVarCharVector(fields[3], allocator), new RepeatedListVector(fields[4], allocator, null), new MapVector(fields[5], allocator, null), new RepeatedMapVector(fields[6], allocator, null) };
try {
for (final ValueVector vector : vectors) {
test.verify(vector);
}
} finally {
AutoCloseables.close(vectors);
}
}
use of org.apache.drill.exec.vector.complex.MapVector in project drill by apache.
the class TestRecordBatchSizer method testSizerMap.
@Test
public void testSizerMap() {
TupleMetadata schema = new SchemaBuilder().addMap("map").add("key", MinorType.INT).add("value", MinorType.VARCHAR).resumeSchema().buildSchema();
RowSetBuilder builder = fixture.rowSetBuilder(schema);
for (int i = 0; i < 10; i++) {
builder.addRow((Object) (new Object[] { 10, "a" }));
}
RowSet rows = builder.build();
// Run the record batch sizer on the resulting batch.
RecordBatchSizer sizer = new RecordBatchSizer(rows.container());
assertEquals(1, sizer.columns().size());
/**
* stdDataSize:50+4, stdNetSize:50+4+4, dataSizePerEntry:4+1,
* netSizePerEntry: 4+1+4,
* totalDataSize:5*10, totalNetSize:4*10+4*10+1*10,
* valueCount:10,
* elementCount:10, cardinality:1, isVariableWidth:true
*/
verifyColumnValues(sizer.columns().get("map"), 54, 58, 5, 9, 50, 90, 10, 10, 1, false);
SingleRowSet empty = fixture.rowSet(schema);
VectorAccessible accessible = empty.vectorAccessible();
for (VectorWrapper<?> vw : accessible) {
ValueVector v = vw.getValueVector();
RecordBatchSizer.ColumnSize colSize = sizer.getColumn(v.getField().getName());
// Allocates to nearest power of two
colSize.allocateVector(v, testRowCount);
MapVector mapVector = (MapVector) v;
ValueVector keyVector = mapVector.getChild("key");
ValueVector valueVector1 = mapVector.getChild("value");
assertEquals((Integer.highestOneBit(testRowCount) << 1), keyVector.getValueCapacity());
UInt4Vector offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals((Integer.highestOneBit(testRowCount) << 1), offsetVector.getValueCapacity());
assertEquals(Integer.highestOneBit(testRowCount << 1) - 1, valueVector1.getValueCapacity());
// Allocates the same as value passed since it is already power of two.
colSize.allocateVector(v, testRowCountPowerTwo - 1);
mapVector = (MapVector) v;
keyVector = mapVector.getChild("key");
valueVector1 = mapVector.getChild("value");
assertEquals((Integer.highestOneBit(testRowCountPowerTwo - 1) << 1), keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(testRowCountPowerTwo, offsetVector.getValueCapacity());
assertEquals(Integer.highestOneBit(testRowCountPowerTwo) - 1, valueVector1.getValueCapacity());
// Allocate for max rows.
colSize.allocateVector(v, ValueVector.MAX_ROW_COUNT - 1);
mapVector = (MapVector) v;
keyVector = mapVector.getChild("key");
valueVector1 = mapVector.getChild("value");
assertEquals(ValueVector.MAX_ROW_COUNT, keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(ValueVector.MAX_ROW_COUNT, offsetVector.getValueCapacity());
assertEquals(ValueVector.MAX_ROW_COUNT - 1, valueVector1.getValueCapacity());
// Allocate for 0 rows. should atleast do allocation for 1 row.
colSize.allocateVector(v, 0);
mapVector = (MapVector) v;
keyVector = mapVector.getChild("key");
valueVector1 = mapVector.getChild("value");
assertEquals(ValueVector.MIN_ROW_COUNT, keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(ValueVector.MIN_ROW_COUNT + 1, offsetVector.getValueCapacity());
assertEquals(ValueVector.MIN_ROW_COUNT, valueVector1.getValueCapacity());
v.clear();
}
empty.clear();
rows.clear();
}
use of org.apache.drill.exec.vector.complex.MapVector in project drill by apache.
the class TestRecordBatchSizer method testSizerNestedMap.
@Test
public void testSizerNestedMap() {
TupleMetadata schema = new SchemaBuilder().addMap("map").add("key", MinorType.INT).add("value", MinorType.VARCHAR).addMap("childMap").add("childKey", MinorType.INT).add("childValue", MinorType.VARCHAR).resumeMap().resumeSchema().buildSchema();
RowSetBuilder builder = fixture.rowSetBuilder(schema);
for (int i = 0; i < 10; i++) {
builder.addRow((Object) (new Object[] { 10, "a", new Object[] { 5, "b" } }));
}
RowSet rows = builder.build();
// Run the record batch sizer on the resulting batch.
RecordBatchSizer sizer = new RecordBatchSizer(rows.container());
assertEquals(1, sizer.columns().size());
/**
* stdDataSize:(50+4)*2, stdNetSize:(50+4)*2+4+4, dataSizePerEntry:(4+1)*2,
* netSizePerEntry: 4*2+1*2+4*2,
* totalDataSize:5*2*10, totalNetSize:netSizePerEntry*2,
* valueCount:10,
* elementCount:10, cardinality:1, isVariableWidth:true
*/
verifyColumnValues(sizer.columns().get("map"), 108, 116, 10, 18, 100, 180, 10, 10, 1, false);
SingleRowSet empty = fixture.rowSet(schema);
VectorAccessible accessible = empty.vectorAccessible();
UInt4Vector offsetVector;
for (VectorWrapper<?> vw : accessible) {
ValueVector v = vw.getValueVector();
RecordBatchSizer.ColumnSize colSize = sizer.getColumn(v.getField().getName());
// Allocates to nearest power of two
colSize.allocateVector(v, testRowCount);
MapVector mapVector = (MapVector) v;
ValueVector keyVector = mapVector.getChild("key");
ValueVector valueVector1 = mapVector.getChild("value");
assertEquals((Integer.highestOneBit(testRowCount) << 1), keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals((Integer.highestOneBit(testRowCount) << 1), offsetVector.getValueCapacity());
assertEquals(Integer.highestOneBit(testRowCount << 1) - 1, valueVector1.getValueCapacity());
MapVector childMapVector = (MapVector) mapVector.getChild("childMap");
ValueVector childKeyVector = childMapVector.getChild("childKey");
ValueVector childValueVector1 = childMapVector.getChild("childValue");
assertEquals((Integer.highestOneBit(testRowCount) << 1), childKeyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals((Integer.highestOneBit(testRowCount) << 1), offsetVector.getValueCapacity());
assertEquals(Integer.highestOneBit(testRowCount << 1) - 1, childValueVector1.getValueCapacity());
// Allocates the same as value passed since it is already power of two.
colSize.allocateVector(v, testRowCountPowerTwo - 1);
mapVector = (MapVector) v;
keyVector = mapVector.getChild("key");
valueVector1 = mapVector.getChild("value");
assertEquals(testRowCountPowerTwo, keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(testRowCountPowerTwo, offsetVector.getValueCapacity());
assertEquals(testRowCountPowerTwo - 1, valueVector1.getValueCapacity());
childMapVector = (MapVector) mapVector.getChild("childMap");
childKeyVector = childMapVector.getChild("childKey");
childValueVector1 = childMapVector.getChild("childValue");
assertEquals(testRowCountPowerTwo, childKeyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(testRowCountPowerTwo, offsetVector.getValueCapacity());
assertEquals(testRowCountPowerTwo - 1, childValueVector1.getValueCapacity());
// Allocate for max rows.
colSize.allocateVector(v, ValueVector.MAX_ROW_COUNT - 1);
mapVector = (MapVector) v;
keyVector = mapVector.getChild("key");
valueVector1 = mapVector.getChild("value");
assertEquals(ValueVector.MAX_ROW_COUNT, keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(ValueVector.MAX_ROW_COUNT, offsetVector.getValueCapacity());
assertEquals(ValueVector.MAX_ROW_COUNT - 1, valueVector1.getValueCapacity());
childMapVector = (MapVector) mapVector.getChild("childMap");
childKeyVector = childMapVector.getChild("childKey");
childValueVector1 = childMapVector.getChild("childValue");
assertEquals(ValueVector.MAX_ROW_COUNT, childKeyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(ValueVector.MAX_ROW_COUNT, offsetVector.getValueCapacity());
assertEquals(ValueVector.MAX_ROW_COUNT - 1, childValueVector1.getValueCapacity());
// Allocate for 0 rows. should atleast do allocation for 1 row.
colSize.allocateVector(v, 0);
mapVector = (MapVector) v;
keyVector = mapVector.getChild("key");
valueVector1 = mapVector.getChild("value");
assertEquals(ValueVector.MIN_ROW_COUNT, keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(ValueVector.MIN_ROW_COUNT + 1, offsetVector.getValueCapacity());
assertEquals(ValueVector.MIN_ROW_COUNT, valueVector1.getValueCapacity());
childMapVector = (MapVector) mapVector.getChild("childMap");
childKeyVector = childMapVector.getChild("childKey");
childValueVector1 = childMapVector.getChild("childValue");
assertEquals(ValueVector.MIN_ROW_COUNT, childKeyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(ValueVector.MIN_ROW_COUNT + 1, offsetVector.getValueCapacity());
assertEquals(ValueVector.MIN_ROW_COUNT, childValueVector1.getValueCapacity());
v.clear();
}
empty.clear();
rows.clear();
}
use of org.apache.drill.exec.vector.complex.MapVector in project drill by apache.
the class TestRecordBatchSizer method testEmptyBatchNestedMap.
@Test
public void testEmptyBatchNestedMap() {
TupleMetadata schema = new SchemaBuilder().addMap("map").add("key", MinorType.INT).add("value", MinorType.VARCHAR).addMap("childMap").add("childKey", MinorType.INT).add("childValue", MinorType.VARCHAR).resumeMap().resumeSchema().buildSchema();
RowSetBuilder builder = fixture.rowSetBuilder(schema);
RowSet rows = builder.build();
// Run the record batch sizer on the resulting batch.
RecordBatchSizer sizer = new RecordBatchSizer(rows.container());
assertEquals(1, sizer.columns().size());
/**
* stdDataSize:(50+4)*2, stdNetSize:(50+4)*2+4+4, dataSizePerEntry:0,
* netSizePerEntry: 0,
* totalDataSize:0, totalNetSize:0,
* valueCount:0,
* elementCount:0, cardinality:0, isVariableWidth:true
*/
verifyColumnValues(sizer.columns().get("map"), 108, 116, 0, 0, 0, 0, 0, 0, 0, false);
// Verify memory allocation is done correctly based on std size for empty batch.
SingleRowSet empty = fixture.rowSet(schema);
VectorAccessible accessible = empty.vectorAccessible();
UInt4Vector offsetVector;
for (VectorWrapper<?> vw : accessible) {
ValueVector v = vw.getValueVector();
RecordBatchSizer.ColumnSize colSize = sizer.getColumn(v.getField().getName());
// Allocates to nearest power of two
colSize.allocateVector(v, testRowCount);
MapVector mapVector = (MapVector) v;
ValueVector keyVector = mapVector.getChild("key");
ValueVector valueVector1 = mapVector.getChild("value");
assertEquals((Integer.highestOneBit(testRowCount) << 1), keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals((Integer.highestOneBit(testRowCount) << 1), offsetVector.getValueCapacity());
assertEquals(Integer.highestOneBit(testRowCount << 1) - 1, valueVector1.getValueCapacity());
MapVector childMapVector = (MapVector) mapVector.getChild("childMap");
ValueVector childKeyVector = childMapVector.getChild("childKey");
ValueVector childValueVector1 = childMapVector.getChild("childValue");
assertEquals((Integer.highestOneBit(testRowCount) << 1), childKeyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals((Integer.highestOneBit(testRowCount) << 1), offsetVector.getValueCapacity());
assertEquals(Integer.highestOneBit(testRowCount << 1) - 1, childValueVector1.getValueCapacity());
// Allocates the same as value passed since it is already power of two.
colSize.allocateVector(v, testRowCountPowerTwo - 1);
mapVector = (MapVector) v;
keyVector = mapVector.getChild("key");
valueVector1 = mapVector.getChild("value");
assertEquals(testRowCountPowerTwo, keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(testRowCountPowerTwo, offsetVector.getValueCapacity());
assertEquals(testRowCountPowerTwo - 1, valueVector1.getValueCapacity());
childMapVector = (MapVector) mapVector.getChild("childMap");
childKeyVector = childMapVector.getChild("childKey");
childValueVector1 = childMapVector.getChild("childValue");
assertEquals(testRowCountPowerTwo, childKeyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(testRowCountPowerTwo, offsetVector.getValueCapacity());
assertEquals(testRowCountPowerTwo - 1, childValueVector1.getValueCapacity());
// Allocate for max rows.
colSize.allocateVector(v, ValueVector.MAX_ROW_COUNT - 1);
mapVector = (MapVector) v;
keyVector = mapVector.getChild("key");
valueVector1 = mapVector.getChild("value");
assertEquals(ValueVector.MAX_ROW_COUNT, keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(ValueVector.MAX_ROW_COUNT, offsetVector.getValueCapacity());
assertEquals(ValueVector.MAX_ROW_COUNT - 1, valueVector1.getValueCapacity());
childMapVector = (MapVector) mapVector.getChild("childMap");
childKeyVector = childMapVector.getChild("childKey");
childValueVector1 = childMapVector.getChild("childValue");
assertEquals(ValueVector.MAX_ROW_COUNT, childKeyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(ValueVector.MAX_ROW_COUNT, offsetVector.getValueCapacity());
assertEquals(ValueVector.MAX_ROW_COUNT - 1, childValueVector1.getValueCapacity());
// Allocate for 0 rows. should atleast do allocation for 1 row.
colSize.allocateVector(v, 0);
mapVector = (MapVector) v;
keyVector = mapVector.getChild("key");
valueVector1 = mapVector.getChild("value");
assertEquals(ValueVector.MIN_ROW_COUNT, keyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(ValueVector.MIN_ROW_COUNT + 1, offsetVector.getValueCapacity());
assertEquals(ValueVector.MIN_ROW_COUNT, valueVector1.getValueCapacity());
childMapVector = (MapVector) mapVector.getChild("childMap");
childKeyVector = childMapVector.getChild("childKey");
childValueVector1 = childMapVector.getChild("childValue");
assertEquals(ValueVector.MIN_ROW_COUNT, childKeyVector.getValueCapacity());
offsetVector = ((VariableWidthVector) valueVector1).getOffsetVector();
assertEquals(ValueVector.MIN_ROW_COUNT + 1, offsetVector.getValueCapacity());
assertEquals(ValueVector.MIN_ROW_COUNT, childValueVector1.getValueCapacity());
v.clear();
}
empty.clear();
rows.clear();
}
Aggregations