use of org.apache.drill.exec.physical.impl.join.LateralJoinBatch in project drill by apache.
the class TestUnnestWithLateralCorrectness method testNestedUnnest.
/**
* Run a plan like the following for various input batches :
* Lateral1
* / \
* / Lateral2
* Scan / \
* / \
* Project1 Project2
* / \
* / \
* Unnest1 Unnest2
*
* @param incomingSchemas
* @param iterOutcomes
* @param execKill
* @param data
* @param baseline
* @param <T>
* @throws Exception
*/
private <T> void testNestedUnnest(TupleMetadata[] incomingSchemas, RecordBatch.IterOutcome[] iterOutcomes, // number of batches after which to kill the execution (!)
int execKill, T[][] data, T[][][] baseline) throws Exception {
// Get the incoming container with dummy data for LJ
final List<VectorContainer> incomingContainer = new ArrayList<>(data.length);
// Create data
ArrayList<RowSet.SingleRowSet> rowSets = new ArrayList<>();
int rowNumber = 0;
int batchNum = 0;
for (Object[] recordBatch : data) {
RowSetBuilder rowSetBuilder = fixture.rowSetBuilder(incomingSchemas[batchNum]);
for (Object rowData : recordBatch) {
rowSetBuilder.addRow(++rowNumber, rowData);
}
RowSet.SingleRowSet rowSet = rowSetBuilder.build();
rowSets.add(rowSet);
incomingContainer.add(rowSet.container());
batchNum++;
}
// Get the unnest POPConfig
final UnnestPOP unnestPopConfig1 = new UnnestPOP(null, SchemaPath.getSimplePath("unnestColumn"), DrillUnnestRelBase.IMPLICIT_COLUMN);
final UnnestPOP unnestPopConfig2 = new UnnestPOP(null, SchemaPath.getSimplePath("colB"), DrillUnnestRelBase.IMPLICIT_COLUMN);
// Get the IterOutcomes for LJ
final List<RecordBatch.IterOutcome> outcomes = new ArrayList<>(iterOutcomes.length);
for (RecordBatch.IterOutcome o : iterOutcomes) {
outcomes.add(o);
}
// Create incoming MockRecordBatch
final MockRecordBatch incomingMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, incomingContainer, outcomes, incomingContainer.get(0).getSchema());
// setup Unnest record batch
final UnnestRecordBatch unnestBatch1 = new UnnestRecordBatch(unnestPopConfig1, fixture.getFragmentContext());
final UnnestRecordBatch unnestBatch2 = new UnnestRecordBatch(unnestPopConfig2, fixture.getFragmentContext());
// Create intermediate Project
final Project projectPopConfig1 = new Project(DrillLogicalTestUtils.parseExprs("unnestColumn.colB", "colB", unnestPopConfig1.getImplicitColumn(), unnestPopConfig1.getImplicitColumn()), unnestPopConfig1);
final ProjectRecordBatch projectBatch1 = new ProjectRecordBatch(projectPopConfig1, unnestBatch1, fixture.getFragmentContext());
final Project projectPopConfig2 = new Project(DrillLogicalTestUtils.parseExprs("colB", "unnestColumn2", unnestPopConfig2.getImplicitColumn(), unnestPopConfig2.getImplicitColumn()), unnestPopConfig2);
final ProjectRecordBatch projectBatch2 = new ProjectRecordBatch(projectPopConfig2, unnestBatch2, fixture.getFragmentContext());
final LateralJoinPOP ljPopConfig2 = new LateralJoinPOP(projectPopConfig1, projectPopConfig2, JoinRelType.INNER, DrillLateralJoinRelBase.IMPLICIT_COLUMN, Lists.newArrayList());
final LateralJoinPOP ljPopConfig1 = new LateralJoinPOP(mockPopConfig, ljPopConfig2, JoinRelType.INNER, DrillLateralJoinRelBase.IMPLICIT_COLUMN, Lists.newArrayList());
final LateralJoinBatch lateralJoinBatch2 = new LateralJoinBatch(ljPopConfig2, fixture.getFragmentContext(), projectBatch1, projectBatch2);
final LateralJoinBatch lateralJoinBatch1 = new LateralJoinBatch(ljPopConfig1, fixture.getFragmentContext(), incomingMockBatch, lateralJoinBatch2);
// set pointer to Lateral in unnest
unnestBatch1.setIncoming((LateralContract) lateralJoinBatch1);
unnestBatch2.setIncoming((LateralContract) lateralJoinBatch2);
// Simulate the pipeline by calling next on the incoming
// results is an array ot batches, each batch being an array of output vectors.
List<List<ValueVector>> resultList = new ArrayList<>();
List<List<ValueVector>> results = null;
int batchesProcessed = 0;
try {
try {
while (!isTerminal(lateralJoinBatch1.next())) {
if (lateralJoinBatch1.getRecordCount() > 0) {
addBatchToResults(resultList, lateralJoinBatch1);
}
batchesProcessed++;
if (batchesProcessed == execKill) {
lateralJoinBatch1.getContext().getExecutorState().fail(new DrillException("Testing failure of execution."));
lateralJoinBatch1.cancel();
}
// else nothing to do
}
} catch (UserException e) {
throw e;
} catch (Exception e) {
throw new Exception("Test failed to execute lateralJoinBatch.next() because: " + e.getMessage());
}
// Check results against baseline
results = resultList;
int batchIndex = 0;
int vectorIndex = 0;
// int valueIndex = 0;
for (List<ValueVector> batch : results) {
int vectorCount = batch.size();
if (vectorCount != baseline[batchIndex].length + 2) {
// baseline does not include the original unnest column(s)
fail("Test failed in validating unnest output. Batch column count mismatch.");
}
for (ValueVector vv : batch) {
if (vv.getField().getName().equals("unnestColumn") || vv.getField().getName().equals("colB")) {
// skip the original input column
continue;
}
int valueCount = vv.getAccessor().getValueCount();
if (valueCount != baseline[batchIndex][vectorIndex].length) {
fail("Test failed in validating unnest output. Value count mismatch in batch number " + (batchIndex + 1) + "" + ".");
}
for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) {
if (vv instanceof MapVector) {
if (!compareMapBaseline(baseline[batchIndex][vectorIndex][valueIndex], vv.getAccessor().getObject(valueIndex))) {
fail("Test failed in validating unnest(Map) output. Value mismatch");
}
} else if (vv instanceof VarCharVector) {
Object val = vv.getAccessor().getObject(valueIndex);
if (((String) baseline[batchIndex][vectorIndex][valueIndex]).compareTo(val.toString()) != 0) {
fail("Test failed in validating unnest output. Value mismatch. Baseline value[]" + vectorIndex + "][" + valueIndex + "]" + ": " + baseline[vectorIndex][valueIndex] + " VV.getObject(valueIndex): " + val);
}
} else {
Object val = vv.getAccessor().getObject(valueIndex);
if (!baseline[batchIndex][vectorIndex][valueIndex].equals(val)) {
fail("Test failed in validating unnest output. Value mismatch. Baseline value[" + vectorIndex + "][" + valueIndex + "]" + ": " + baseline[batchIndex][vectorIndex][valueIndex] + " VV.getObject(valueIndex): " + val);
}
}
}
vectorIndex++;
}
vectorIndex = 0;
batchIndex++;
}
} catch (UserException e) {
// Valid exception
throw e;
} catch (Exception e) {
fail("Test failed. Exception : " + e.getMessage());
} finally {
// Close all the resources for this test case
unnestBatch1.close();
lateralJoinBatch1.close();
unnestBatch2.close();
lateralJoinBatch2.close();
incomingMockBatch.close();
if (results != null) {
for (List<ValueVector> batch : results) {
for (ValueVector vv : batch) {
vv.clear();
}
}
}
for (RowSet.SingleRowSet rowSet : rowSets) {
rowSet.clear();
}
}
}
use of org.apache.drill.exec.physical.impl.join.LateralJoinBatch in project drill by apache.
the class TestUnnestWithLateralCorrectness method testUnnest.
// test unnest for various input conditions optionally invoking kill. if the kill or killBatch
// parameter is greater than 0 then the record batch is sent a kill after that many batches have been processed
private <T> void testUnnest(TupleMetadata[] incomingSchemas, RecordBatch.IterOutcome[] iterOutcomes, // kill unnest after every 'unnestLimit' number of values in every record
int unnestLimit, // number of batches after which to kill the execution (!)
int execKill, T[][] data, T[][][] baseline, boolean excludeUnnestColumn) throws Exception {
// Get the incoming container with dummy data for LJ
final List<VectorContainer> incomingContainer = new ArrayList<>(data.length);
// Create data
ArrayList<RowSet.SingleRowSet> rowSets = new ArrayList<>();
int rowNumber = 0;
int batchNum = 0;
for (Object[] recordBatch : data) {
RowSetBuilder rowSetBuilder = fixture.rowSetBuilder(incomingSchemas[batchNum]);
for (Object rowData : recordBatch) {
rowSetBuilder.addRow(++rowNumber, rowData);
}
RowSet.SingleRowSet rowSet = rowSetBuilder.build();
rowSets.add(rowSet);
incomingContainer.add(rowSet.container());
batchNum++;
}
// Get the unnest POPConfig
final UnnestPOP unnestPopConfig = new UnnestPOP(null, SchemaPath.getCompoundPath("unnestColumn"), DrillUnnestRelBase.IMPLICIT_COLUMN);
// Get the IterOutcomes for LJ
final List<RecordBatch.IterOutcome> outcomes = new ArrayList<>(iterOutcomes.length);
for (RecordBatch.IterOutcome o : iterOutcomes) {
outcomes.add(o);
}
// Create incoming MockRecordBatch
final MockRecordBatch incomingMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, incomingContainer, outcomes, incomingContainer.get(0).getSchema());
// setup Unnest record batch
final UnnestRecordBatch unnestBatch = new UnnestRecordBatch(unnestPopConfig, fixture.getFragmentContext());
// project is required to rename the columns so as to disambiguate the same column name from
// unnest operator and the regular scan.
final Project projectPopConfig = new Project(DrillLogicalTestUtils.parseExprs("unnestColumn", "unnestColumn1", unnestPopConfig.getImplicitColumn(), unnestPopConfig.getImplicitColumn()), null);
final ProjectRecordBatch projectBatch = new ProjectRecordBatch(projectPopConfig, unnestBatch, fixture.getFragmentContext());
final LateralJoinBatch lateralJoinBatch = new LateralJoinBatch(ljPopConfig, fixture.getFragmentContext(), incomingMockBatch, projectBatch);
// set pointer to Lateral in unnest
unnestBatch.setIncoming((LateralContract) lateralJoinBatch);
// Simulate the pipeline by calling next on the incoming
// results is an array of batches, each batch being an array of output vectors.
List<List<ValueVector>> resultList = new ArrayList<>();
List<List<ValueVector>> results = null;
int batchesProcessed = 0;
try {
try {
while (!isTerminal(lateralJoinBatch.next())) {
if (lateralJoinBatch.getRecordCount() > 0) {
addBatchToResults(resultList, lateralJoinBatch);
}
batchesProcessed++;
if (batchesProcessed == execKill) {
// Simulate by skipping out of the loop
break;
}
// else nothing to do
}
} catch (UserException e) {
throw e;
} catch (Exception e) {
fail(e.getMessage());
}
// Check results against baseline
results = resultList;
int batchIndex = 0;
int vectorIndex = 0;
// int valueIndex = 0;
for (List<ValueVector> batch : results) {
int vectorCount = batch.size();
int expectedVectorCount = (excludeUnnestColumn) ? 0 : 1;
expectedVectorCount += baseline[batchIndex].length;
if (vectorCount != expectedVectorCount) {
// baseline does not include the original unnest column
fail("Test failed in validating unnest output. Batch column count mismatch.");
}
for (ValueVector vv : batch) {
if (vv.getField().getName().equals("unnestColumn")) {
// skip the original input column
continue;
}
int valueCount = vv.getAccessor().getValueCount();
if (valueCount != baseline[batchIndex][vectorIndex].length) {
fail("Test failed in validating unnest output. Value count mismatch in batch number " + (batchIndex + 1) + "" + ".");
}
for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) {
if (vv instanceof MapVector) {
if (!compareMapBaseline(baseline[batchIndex][vectorIndex][valueIndex], vv.getAccessor().getObject(valueIndex))) {
fail("Test failed in validating unnest(Map) output. Value mismatch");
}
} else if (vv instanceof VarCharVector) {
Object val = vv.getAccessor().getObject(valueIndex);
if (((String) baseline[batchIndex][vectorIndex][valueIndex]).compareTo(val.toString()) != 0) {
fail("Test failed in validating unnest output. Value mismatch. Baseline value[]" + vectorIndex + "][" + valueIndex + "]" + ": " + baseline[vectorIndex][valueIndex] + " VV.getObject(valueIndex): " + val);
}
} else {
Object val = vv.getAccessor().getObject(valueIndex);
if (!baseline[batchIndex][vectorIndex][valueIndex].equals(val)) {
fail("Test failed in validating unnest output. Value mismatch. Baseline value[" + vectorIndex + "][" + valueIndex + "]" + ": " + baseline[batchIndex][vectorIndex][valueIndex] + " VV.getObject(valueIndex): " + val);
}
}
}
vectorIndex++;
}
vectorIndex = 0;
batchIndex++;
}
} catch (UserException e) {
// Valid exception
throw e;
} catch (Exception e) {
fail("Test failed. Exception : " + e.getMessage());
} finally {
// Close all the resources for this test case
unnestBatch.close();
lateralJoinBatch.close();
incomingMockBatch.close();
if (results != null) {
for (List<ValueVector> batch : results) {
for (ValueVector vv : batch) {
vv.clear();
}
}
}
for (RowSet.SingleRowSet rowSet : rowSets) {
rowSet.clear();
}
}
}
Aggregations