use of org.apache.drill.exec.vector.complex.MapVector in project drill by apache.
the class TestUnnestWithLateralCorrectness method testUnnest.
// test unnest for various input conditions optionally invoking kill. if the kill or killBatch
// parameter is greater than 0 then the record batch is sent a kill after that many batches have been processed
private <T> void testUnnest(TupleMetadata[] incomingSchemas, RecordBatch.IterOutcome[] iterOutcomes, // kill unnest after every 'unnestLimit' number of values in every record
int unnestLimit, // number of batches after which to kill the execution (!)
int execKill, T[][] data, T[][][] baseline, boolean excludeUnnestColumn) throws Exception {
// Get the incoming container with dummy data for LJ
final List<VectorContainer> incomingContainer = new ArrayList<>(data.length);
// Create data
ArrayList<RowSet.SingleRowSet> rowSets = new ArrayList<>();
int rowNumber = 0;
int batchNum = 0;
for (Object[] recordBatch : data) {
RowSetBuilder rowSetBuilder = fixture.rowSetBuilder(incomingSchemas[batchNum]);
for (Object rowData : recordBatch) {
rowSetBuilder.addRow(++rowNumber, rowData);
}
RowSet.SingleRowSet rowSet = rowSetBuilder.build();
rowSets.add(rowSet);
incomingContainer.add(rowSet.container());
batchNum++;
}
// Get the unnest POPConfig
final UnnestPOP unnestPopConfig = new UnnestPOP(null, SchemaPath.getCompoundPath("unnestColumn"), DrillUnnestRelBase.IMPLICIT_COLUMN);
// Get the IterOutcomes for LJ
final List<RecordBatch.IterOutcome> outcomes = new ArrayList<>(iterOutcomes.length);
for (RecordBatch.IterOutcome o : iterOutcomes) {
outcomes.add(o);
}
// Create incoming MockRecordBatch
final MockRecordBatch incomingMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, incomingContainer, outcomes, incomingContainer.get(0).getSchema());
// setup Unnest record batch
final UnnestRecordBatch unnestBatch = new UnnestRecordBatch(unnestPopConfig, fixture.getFragmentContext());
// project is required to rename the columns so as to disambiguate the same column name from
// unnest operator and the regular scan.
final Project projectPopConfig = new Project(DrillLogicalTestUtils.parseExprs("unnestColumn", "unnestColumn1", unnestPopConfig.getImplicitColumn(), unnestPopConfig.getImplicitColumn()), null);
final ProjectRecordBatch projectBatch = new ProjectRecordBatch(projectPopConfig, unnestBatch, fixture.getFragmentContext());
final LateralJoinBatch lateralJoinBatch = new LateralJoinBatch(ljPopConfig, fixture.getFragmentContext(), incomingMockBatch, projectBatch);
// set pointer to Lateral in unnest
unnestBatch.setIncoming((LateralContract) lateralJoinBatch);
// Simulate the pipeline by calling next on the incoming
// results is an array of batches, each batch being an array of output vectors.
List<List<ValueVector>> resultList = new ArrayList<>();
List<List<ValueVector>> results = null;
int batchesProcessed = 0;
try {
try {
while (!isTerminal(lateralJoinBatch.next())) {
if (lateralJoinBatch.getRecordCount() > 0) {
addBatchToResults(resultList, lateralJoinBatch);
}
batchesProcessed++;
if (batchesProcessed == execKill) {
// Simulate by skipping out of the loop
break;
}
// else nothing to do
}
} catch (UserException e) {
throw e;
} catch (Exception e) {
fail(e.getMessage());
}
// Check results against baseline
results = resultList;
int batchIndex = 0;
int vectorIndex = 0;
// int valueIndex = 0;
for (List<ValueVector> batch : results) {
int vectorCount = batch.size();
int expectedVectorCount = (excludeUnnestColumn) ? 0 : 1;
expectedVectorCount += baseline[batchIndex].length;
if (vectorCount != expectedVectorCount) {
// baseline does not include the original unnest column
fail("Test failed in validating unnest output. Batch column count mismatch.");
}
for (ValueVector vv : batch) {
if (vv.getField().getName().equals("unnestColumn")) {
// skip the original input column
continue;
}
int valueCount = vv.getAccessor().getValueCount();
if (valueCount != baseline[batchIndex][vectorIndex].length) {
fail("Test failed in validating unnest output. Value count mismatch in batch number " + (batchIndex + 1) + "" + ".");
}
for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) {
if (vv instanceof MapVector) {
if (!compareMapBaseline(baseline[batchIndex][vectorIndex][valueIndex], vv.getAccessor().getObject(valueIndex))) {
fail("Test failed in validating unnest(Map) output. Value mismatch");
}
} else if (vv instanceof VarCharVector) {
Object val = vv.getAccessor().getObject(valueIndex);
if (((String) baseline[batchIndex][vectorIndex][valueIndex]).compareTo(val.toString()) != 0) {
fail("Test failed in validating unnest output. Value mismatch. Baseline value[]" + vectorIndex + "][" + valueIndex + "]" + ": " + baseline[vectorIndex][valueIndex] + " VV.getObject(valueIndex): " + val);
}
} else {
Object val = vv.getAccessor().getObject(valueIndex);
if (!baseline[batchIndex][vectorIndex][valueIndex].equals(val)) {
fail("Test failed in validating unnest output. Value mismatch. Baseline value[" + vectorIndex + "][" + valueIndex + "]" + ": " + baseline[batchIndex][vectorIndex][valueIndex] + " VV.getObject(valueIndex): " + val);
}
}
}
vectorIndex++;
}
vectorIndex = 0;
batchIndex++;
}
} catch (UserException e) {
// Valid exception
throw e;
} catch (Exception e) {
fail("Test failed. Exception : " + e.getMessage());
} finally {
// Close all the resources for this test case
unnestBatch.close();
lateralJoinBatch.close();
incomingMockBatch.close();
if (results != null) {
for (List<ValueVector> batch : results) {
for (ValueVector vv : batch) {
vv.clear();
}
}
}
for (RowSet.SingleRowSet rowSet : rowSets) {
rowSet.clear();
}
}
}
use of org.apache.drill.exec.vector.complex.MapVector in project drill by apache.
the class TestRepeated method listOfList.
//
// @Test
// public void repeatedMap() {
//
// /**
// * We're going to try to create an object that looks like:
// *
// * {
// * a: [
// * {x: 1, y: 2}
// * {x: 2, y: 1}
// * ]
// * }
// *
// */
// MapVector v = new MapVector("", allocator);
// ComplexWriter writer = new ComplexWriterImpl("col", v);
//
// MapWriter map = writer.rootAsMap();
//
// map.start();
// ListWriter list = map.list("a");
// MapWriter inner = list.map();
//
// IntHolder holder = new IntHolder();
// IntWriter xCol = inner.integer("x");
// IntWriter yCol = inner.integer("y");
//
// inner.start();
//
// holder.value = 1;
// xCol.write(holder);
// holder.value = 2;
// yCol.write(holder);
//
// inner.end();
//
// inner.start();
//
// holder.value = 2;
// xCol.write(holder);
// holder.value = 1;
// yCol.write(holder);
//
// inner.end();
//
// IntWriter numCol = map.integer("nums");
// holder.value = 14;
// numCol.write(holder);
//
// map.end();
//
//
// assertTrue(writer.ok());
//
// }
@Test
public void listOfList() throws Exception {
/**
* We're going to try to create an object that looks like:
*
* {
* a: [
* [1,2,3],
* [2,3,4]
* ],
* nums: 14,
* b: [
* { c: 1 },
* { c: 2 , x: 15}
* ]
* }
*/
final MapVector mapVector = new MapVector("", allocator, null);
final ComplexWriterImpl writer = new ComplexWriterImpl("col", mapVector);
writer.allocate();
{
final MapWriter map = writer.rootAsMap();
final ListWriter list = map.list("a");
list.startList();
final ListWriter innerList = list.list();
final IntWriter innerInt = innerList.integer();
innerList.startList();
final IntHolder holder = new IntHolder();
holder.value = 1;
innerInt.write(holder);
holder.value = 2;
innerInt.write(holder);
holder.value = 3;
innerInt.write(holder);
innerList.endList();
innerList.startList();
holder.value = 4;
innerInt.write(holder);
holder.value = 5;
innerInt.write(holder);
innerList.endList();
list.endList();
final IntWriter numCol = map.integer("nums");
holder.value = 14;
numCol.write(holder);
final MapWriter repeatedMap = map.list("b").map();
repeatedMap.start();
holder.value = 1;
repeatedMap.integer("c").write(holder);
repeatedMap.end();
repeatedMap.start();
holder.value = 2;
repeatedMap.integer("c").write(holder);
final BigIntHolder h = new BigIntHolder();
h.value = 15;
repeatedMap.bigInt("x").write(h);
repeatedMap.end();
map.end();
}
{
writer.setPosition(1);
final MapWriter map = writer.rootAsMap();
final ListWriter list = map.list("a");
list.startList();
final ListWriter innerList = list.list();
final IntWriter innerInt = innerList.integer();
innerList.startList();
final IntHolder holder = new IntHolder();
holder.value = -1;
innerInt.write(holder);
holder.value = -2;
innerInt.write(holder);
holder.value = -3;
innerInt.write(holder);
innerList.endList();
innerList.startList();
holder.value = -4;
innerInt.write(holder);
holder.value = -5;
innerInt.write(holder);
innerList.endList();
list.endList();
final IntWriter numCol = map.integer("nums");
holder.value = -28;
numCol.write(holder);
final MapWriter repeatedMap = map.list("b").map();
repeatedMap.start();
holder.value = -1;
repeatedMap.integer("c").write(holder);
repeatedMap.end();
repeatedMap.start();
holder.value = -2;
repeatedMap.integer("c").write(holder);
final BigIntHolder h = new BigIntHolder();
h.value = -30;
repeatedMap.bigInt("x").write(h);
repeatedMap.end();
map.end();
}
final ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter();
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
final JsonWriter jsonWriter = new JsonWriter(stream, true, true);
final FieldReader reader = mapVector.getChild("col", MapVector.class).getReader();
reader.setPosition(0);
jsonWriter.write(reader);
reader.setPosition(1);
jsonWriter.write(reader);
writer.close();
}
use of org.apache.drill.exec.vector.complex.MapVector in project drill by axbaretto.
the class TestRepeated method listOfList.
//
// @Test
// public void repeatedMap() {
//
// /**
// * We're going to try to create an object that looks like:
// *
// * {
// * a: [
// * {x: 1, y: 2}
// * {x: 2, y: 1}
// * ]
// * }
// *
// */
// MapVector v = new MapVector("", allocator);
// ComplexWriter writer = new ComplexWriterImpl("col", v);
//
// MapWriter map = writer.rootAsMap();
//
// map.start();
// ListWriter list = map.list("a");
// MapWriter inner = list.map();
//
// IntHolder holder = new IntHolder();
// IntWriter xCol = inner.integer("x");
// IntWriter yCol = inner.integer("y");
//
// inner.start();
//
// holder.value = 1;
// xCol.write(holder);
// holder.value = 2;
// yCol.write(holder);
//
// inner.end();
//
// inner.start();
//
// holder.value = 2;
// xCol.write(holder);
// holder.value = 1;
// yCol.write(holder);
//
// inner.end();
//
// IntWriter numCol = map.integer("nums");
// holder.value = 14;
// numCol.write(holder);
//
// map.end();
//
//
// assertTrue(writer.ok());
//
// System.out.println(v.getAccessor().getObject(0));
//
// }
@Test
public void listOfList() throws Exception {
/**
* We're going to try to create an object that looks like:
*
* {
* a: [
* [1,2,3],
* [2,3,4]
* ],
* nums: 14,
* b: [
* { c: 1 },
* { c: 2 , x: 15}
* ]
* }
*/
final MapVector mapVector = new MapVector("", allocator, null);
final ComplexWriterImpl writer = new ComplexWriterImpl("col", mapVector);
writer.allocate();
{
final MapWriter map = writer.rootAsMap();
final ListWriter list = map.list("a");
list.startList();
final ListWriter innerList = list.list();
final IntWriter innerInt = innerList.integer();
innerList.startList();
final IntHolder holder = new IntHolder();
holder.value = 1;
innerInt.write(holder);
holder.value = 2;
innerInt.write(holder);
holder.value = 3;
innerInt.write(holder);
innerList.endList();
innerList.startList();
holder.value = 4;
innerInt.write(holder);
holder.value = 5;
innerInt.write(holder);
innerList.endList();
list.endList();
final IntWriter numCol = map.integer("nums");
holder.value = 14;
numCol.write(holder);
final MapWriter repeatedMap = map.list("b").map();
repeatedMap.start();
holder.value = 1;
repeatedMap.integer("c").write(holder);
repeatedMap.end();
repeatedMap.start();
holder.value = 2;
repeatedMap.integer("c").write(holder);
final BigIntHolder h = new BigIntHolder();
h.value = 15;
repeatedMap.bigInt("x").write(h);
repeatedMap.end();
map.end();
}
{
writer.setPosition(1);
final MapWriter map = writer.rootAsMap();
final ListWriter list = map.list("a");
list.startList();
final ListWriter innerList = list.list();
final IntWriter innerInt = innerList.integer();
innerList.startList();
final IntHolder holder = new IntHolder();
holder.value = -1;
innerInt.write(holder);
holder.value = -2;
innerInt.write(holder);
holder.value = -3;
innerInt.write(holder);
innerList.endList();
innerList.startList();
holder.value = -4;
innerInt.write(holder);
holder.value = -5;
innerInt.write(holder);
innerList.endList();
list.endList();
final IntWriter numCol = map.integer("nums");
holder.value = -28;
numCol.write(holder);
final MapWriter repeatedMap = map.list("b").map();
repeatedMap.start();
holder.value = -1;
repeatedMap.integer("c").write(holder);
repeatedMap.end();
repeatedMap.start();
holder.value = -2;
repeatedMap.integer("c").write(holder);
final BigIntHolder h = new BigIntHolder();
h.value = -30;
repeatedMap.bigInt("x").write(h);
repeatedMap.end();
map.end();
}
final ObjectWriter ow = new ObjectMapper().writer().withDefaultPrettyPrinter();
System.out.println("Map of Object[0]: " + ow.writeValueAsString(mapVector.getAccessor().getObject(0)));
System.out.println("Map of Object[1]: " + ow.writeValueAsString(mapVector.getAccessor().getObject(1)));
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
final JsonWriter jsonWriter = new JsonWriter(stream, true, true);
final FieldReader reader = mapVector.getChild("col", MapVector.class).getReader();
reader.setPosition(0);
jsonWriter.write(reader);
reader.setPosition(1);
jsonWriter.write(reader);
System.out.print("Json Read: ");
System.out.println(new String(stream.toByteArray(), Charsets.UTF_8));
writer.close();
}
use of org.apache.drill.exec.vector.complex.MapVector in project drill by axbaretto.
the class HBaseRecordReader method next.
@Override
public int next() {
Stopwatch watch = Stopwatch.createStarted();
if (rowKeyVector != null) {
rowKeyVector.clear();
rowKeyVector.allocateNew();
}
for (ValueVector v : familyVectorMap.values()) {
v.clear();
v.allocateNew();
}
int rowCount = 0;
// if allocated memory for the first row is larger than allowed max in batch, it will be added anyway
do {
Result result = null;
final OperatorStats operatorStats = operatorContext == null ? null : operatorContext.getStats();
try {
if (operatorStats != null) {
operatorStats.startWait();
}
try {
result = resultScanner.next();
} finally {
if (operatorStats != null) {
operatorStats.stopWait();
}
}
} catch (IOException e) {
throw new DrillRuntimeException(e);
}
if (result == null) {
break;
}
// parse the result and populate the value vectors
Cell[] cells = result.rawCells();
if (rowKeyVector != null) {
rowKeyVector.getMutator().setSafe(rowCount, cells[0].getRowArray(), cells[0].getRowOffset(), cells[0].getRowLength());
}
if (!rowKeyOnly) {
for (final Cell cell : cells) {
final int familyOffset = cell.getFamilyOffset();
final int familyLength = cell.getFamilyLength();
final byte[] familyArray = cell.getFamilyArray();
final MapVector mv = getOrCreateFamilyVector(new String(familyArray, familyOffset, familyLength), true);
final int qualifierOffset = cell.getQualifierOffset();
final int qualifierLength = cell.getQualifierLength();
final byte[] qualifierArray = cell.getQualifierArray();
final NullableVarBinaryVector v = getOrCreateColumnVector(mv, new String(qualifierArray, qualifierOffset, qualifierLength));
final int valueOffset = cell.getValueOffset();
final int valueLength = cell.getValueLength();
final byte[] valueArray = cell.getValueArray();
v.getMutator().setSafe(rowCount, valueArray, valueOffset, valueLength);
}
}
rowCount++;
} while (canAddNewRow(rowCount));
setOutputRowCount(rowCount);
logger.debug("Took {} ms to get {} records", watch.elapsed(TimeUnit.MILLISECONDS), rowCount);
return rowCount;
}
use of org.apache.drill.exec.vector.complex.MapVector in project drill by axbaretto.
the class HBaseRecordReader method getOrCreateFamilyVector.
private MapVector getOrCreateFamilyVector(String familyName, boolean allocateOnCreate) {
try {
MapVector v = familyVectorMap.get(familyName);
if (v == null) {
SchemaPath column = SchemaPath.getSimplePath(familyName);
MaterializedField field = MaterializedField.create(column.getAsNamePart().getName(), COLUMN_FAMILY_TYPE);
v = outputMutator.addField(field, MapVector.class);
if (allocateOnCreate) {
v.allocateNew();
}
getColumns().add(column);
familyVectorMap.put(familyName, v);
}
return v;
} catch (SchemaChangeException e) {
throw new DrillRuntimeException(e);
}
}
Aggregations