use of org.apache.drill.exec.physical.impl.MockRecordBatch in project drill by apache.
the class TestLateralJoinCorrectness method testBuildSchemaNonEmptyLEmptyRBatch.
/**
* With non-empty left and empty right batch, the {@link LateralJoinBatch#buildSchema()} phase
* should still build the output container schema and return empty batch
*
* @throws Exception
*/
@Test
public void testBuildSchemaNonEmptyLEmptyRBatch() throws Exception {
// Get the left container with dummy data for Lateral Join
leftContainer.add(nonEmptyLeftRowSet.container());
// Get the left IterOutcomes for Lateral Join
leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
// Create Left MockRecordBatch
final CloseableRecordBatch leftMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, leftContainer, leftOutcomes, leftContainer.get(0).getSchema());
// Get the right container with dummy data
rightContainer.add(emptyRightRowSet.container());
rightOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
final CloseableRecordBatch rightMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, rightContainer, rightOutcomes, rightContainer.get(0).getSchema());
final LateralJoinBatch ljBatch = new LateralJoinBatch(ljPopConfig, fixture.getFragmentContext(), leftMockBatch, rightMockBatch);
try {
assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == ljBatch.next());
assertTrue(ljBatch.getRecordCount() == 0);
// Since Right batch is empty it should drain left and return NONE
assertTrue(RecordBatch.IterOutcome.NONE == ljBatch.next());
} catch (AssertionError | Exception error) {
fail();
} finally {
// Close all the resources for this test case
ljBatch.close();
leftMockBatch.close();
rightMockBatch.close();
}
}
use of org.apache.drill.exec.physical.impl.MockRecordBatch in project drill by apache.
the class TestLateralJoinCorrectness method test1RecordLeftBatchTo1RightRecordBatch.
/**
* Test to show correct IterOutcome produced by LATERAL when one record in left batch produces only 1 right batch
* with EMIT outcome. Then output of LATERAL should be produced by OK outcome after the join. It verifies the number
* of records in the output batch based on left and right input batches.
*
* @throws Exception
*/
@Test
public void test1RecordLeftBatchTo1RightRecordBatch() throws Exception {
// Get the left container with dummy data for Lateral Join
leftContainer.add(nonEmptyLeftRowSet.container());
// Get the left IterOutcomes for Lateral Join
leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
// Create Left MockRecordBatch
final CloseableRecordBatch leftMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, leftContainer, leftOutcomes, leftContainer.get(0).getSchema());
// Get the right container with dummy data
rightContainer.add(emptyRightRowSet.container());
rightContainer.add(nonEmptyRightRowSet.container());
rightOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
rightOutcomes.add(RecordBatch.IterOutcome.EMIT);
final CloseableRecordBatch rightMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, rightContainer, rightOutcomes, rightContainer.get(0).getSchema());
final LateralJoinBatch ljBatch = new LateralJoinBatch(ljPopConfig, fixture.getFragmentContext(), leftMockBatch, rightMockBatch);
try {
assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == ljBatch.next());
assertTrue(RecordBatch.IterOutcome.OK == ljBatch.next());
assertTrue(ljBatch.getRecordCount() == (nonEmptyLeftRowSet.rowCount() * nonEmptyRightRowSet.rowCount()));
assertTrue(RecordBatch.IterOutcome.NONE == ljBatch.next());
} catch (AssertionError | Exception error) {
fail();
} finally {
// Close all the resources for this test case
ljBatch.close();
leftMockBatch.close();
rightMockBatch.close();
}
}
use of org.apache.drill.exec.physical.impl.MockRecordBatch in project drill by apache.
the class TestLateralJoinCorrectness method testMultipleUnnestAtSameLevel.
/**
* This test generates an operator tree for multiple UNNEST at same level by stacking 2 LATERAL and UNNEST pair on
* top of each other. Then we call next() on top level LATERAL to simulate the operator tree and compare the
* outcome and record count generated with expected values.
* @throws Exception
*/
@Test
public void testMultipleUnnestAtSameLevel() throws Exception {
// ** Prepare first pair of left batch and right batch for Lateral_1 **
leftContainer.add(nonEmptyLeftRowSet.container());
// Get the left IterOutcomes for Lateral Join
leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
final CloseableRecordBatch leftMockBatch_1 = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, leftContainer, leftOutcomes, leftContainer.get(0).getSchema());
// Get the right container with dummy data
rightContainer.add(emptyRightRowSet.container());
rightContainer.add(nonEmptyRightRowSet.container());
rightOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
rightOutcomes.add(RecordBatch.IterOutcome.EMIT);
final CloseableRecordBatch rightMockBatch_1 = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, rightContainer, rightOutcomes, rightContainer.get(0).getSchema());
final LateralJoinPOP popConfig_1 = new LateralJoinPOP(null, null, JoinRelType.INNER, DrillLateralJoinRelBase.IMPLICIT_COLUMN, Lists.newArrayList());
final LateralJoinBatch ljBatch_1 = new LateralJoinBatch(popConfig_1, fixture.getFragmentContext(), leftMockBatch_1, rightMockBatch_1);
// ** Prepare second pair of left and right batch for Lateral_2 **
// Get the right container with dummy data for Lateral Join_2
// Create right input schema
TupleMetadata rightSchema2 = new SchemaBuilder().add(popConfig_1.getImplicitRIDColumn(), TypeProtos.MinorType.INT).add("id_right_1", TypeProtos.MinorType.INT).add("cost_right_1", TypeProtos.MinorType.INT).add("name_right_1", TypeProtos.MinorType.VARCHAR).buildSchema();
final RowSet.SingleRowSet emptyRightRowSet2 = fixture.rowSetBuilder(rightSchema2).build();
final RowSet.SingleRowSet nonEmptyRightRowSet2 = fixture.rowSetBuilder(rightSchema2).addRow(1, 6, 60, "item61").addRow(1, 7, 70, "item71").addRow(1, 8, 80, "item81").build();
final List<VectorContainer> rightContainer2 = new ArrayList<>(5);
// Get the right container with dummy data
rightContainer2.add(emptyRightRowSet2.container());
rightContainer2.add(nonEmptyRightRowSet2.container());
rightContainer2.add(emptyRightRowSet2.container());
rightContainer2.add(emptyRightRowSet2.container());
final List<RecordBatch.IterOutcome> rightOutcomes2 = new ArrayList<>(5);
rightOutcomes2.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
rightOutcomes2.add(RecordBatch.IterOutcome.OK);
rightOutcomes2.add(RecordBatch.IterOutcome.OK);
rightOutcomes2.add(RecordBatch.IterOutcome.EMIT);
final CloseableRecordBatch rightMockBatch_2 = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, rightContainer2, rightOutcomes2, rightContainer2.get(0).getSchema());
final LateralJoinBatch ljBatch_2 = new LateralJoinBatch(popConfig_1, fixture.getFragmentContext(), ljBatch_1, rightMockBatch_2);
try {
// 3 from the lower level lateral and then finally 3 for 1st row in
final int expectedOutputRecordCount = 3;
// second lateral and 0 for other 2 rows.
assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == ljBatch_2.next());
assertTrue(RecordBatch.IterOutcome.OK == ljBatch_2.next());
int actualOutputRecordCount = ljBatch_2.getRecordCount();
assertTrue(RecordBatch.IterOutcome.NONE == ljBatch_2.next());
assertTrue(actualOutputRecordCount == expectedOutputRecordCount);
} catch (AssertionError | Exception error) {
fail();
} finally {
// Close all the resources for this test case
ljBatch_2.close();
rightMockBatch_2.close();
ljBatch_1.close();
leftMockBatch_1.close();
rightMockBatch_1.close();
rightContainer2.clear();
rightOutcomes2.clear();
}
}
use of org.apache.drill.exec.physical.impl.MockRecordBatch in project drill by apache.
the class TestLateralJoinCorrectness method test_OK_NEW_SCHEMAFromLeft_EmitFromRight_PostBuildSchema.
/**
* Test to verify if OK_NEW_SCHEMA is received from left side of LATERAL post build schema phase and EMIT is
* received from right side of LATERAL for each row on left side, then Lateral sends OK_NEW_SCHEMA downstream with
* the output batch. LATERAL shouldn't send any batch with EMIT outcome to the downstream operator as it is the
* consumer of all the EMIT outcomes. It will work fine in case of Multilevel LATERAL too since there the lower
* LATERAL only sends EMIT after it receives it from left UNNEST.
* @throws Exception
*/
@Test
public void test_OK_NEW_SCHEMAFromLeft_EmitFromRight_PostBuildSchema() throws Exception {
// Get the left container with dummy data for Lateral Join
TupleMetadata leftSchema3 = new SchemaBuilder().add("id_left_left", TypeProtos.MinorType.INT).add("cost_left_left", TypeProtos.MinorType.VARCHAR).add("name_left_left", TypeProtos.MinorType.VARCHAR).buildSchema();
final RowSet.SingleRowSet nonEmptyLeftRowSet_leftSchema3 = fixture.rowSetBuilder(leftSchema3).addRow(6, "60", "item6").addRow(7, "70", "item7").build();
leftContainer.add(emptyLeftRowSet.container());
leftContainer.add(nonEmptyLeftRowSet_leftSchema3.container());
// Get the left IterOutcomes for Lateral Join
leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
// Create Left MockRecordBatch
final CloseableRecordBatch leftMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, leftContainer, leftOutcomes, leftContainer.get(0).getSchema());
// Get the right container with dummy data
final RowSet.SingleRowSet nonEmptyRightRowSet2 = fixture.rowSetBuilder(rightSchema).addRow(2, 10, 100, "list10").build();
rightContainer.add(emptyRightRowSet.container());
rightContainer.add(nonEmptyRightRowSet.container());
rightContainer.add(nonEmptyRightRowSet2.container());
rightOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
rightOutcomes.add(RecordBatch.IterOutcome.OK);
rightOutcomes.add(RecordBatch.IterOutcome.EMIT);
final CloseableRecordBatch rightMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, rightContainer, rightOutcomes, rightContainer.get(0).getSchema());
final LateralJoinBatch ljBatch = new LateralJoinBatch(ljPopConfig, fixture.getFragmentContext(), leftMockBatch, rightMockBatch);
try {
assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == ljBatch.next());
assertTrue(ljBatch.getRecordCount() == 0);
// Since Right batch is empty it should drain left and return NONE
assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == ljBatch.next());
final int expectedOutputCount = nonEmptyRightRowSet.rowCount() + nonEmptyRightRowSet2.rowCount();
assertTrue(ljBatch.getRecordCount() == expectedOutputCount);
} catch (AssertionError | Exception error) {
fail();
} finally {
// Close all the resources for this test case
ljBatch.close();
leftMockBatch.close();
rightMockBatch.close();
}
}
use of org.apache.drill.exec.physical.impl.MockRecordBatch in project drill by apache.
the class TestLateralJoinCorrectness method testMultiLevelLateral_SchemaChange_LeftRightUnnest_NonEmptyBatch.
/**
* Test to verify in case of Multilevel lateral when a non-empty OK_NEW_SCHEMA batch post build schema phase is
* received from right most UNNEST of lower LATERAL then pipeline works fine.
* @throws Exception
*/
@Test
public void testMultiLevelLateral_SchemaChange_LeftRightUnnest_NonEmptyBatch() throws Exception {
// ** Prepare first pair of left batch and right batch for lower level LATERAL Lateral_1 **
final LateralJoinPOP popConfig_1 = new LateralJoinPOP(null, null, JoinRelType.INNER, DrillLateralJoinRelBase.IMPLICIT_COLUMN, Lists.newArrayList());
TupleMetadata leftSchemaWithImplicit = new SchemaBuilder().add(popConfig_1.getImplicitRIDColumn(), TypeProtos.MinorType.INT).add("id_left", TypeProtos.MinorType.INT).add("cost_left", TypeProtos.MinorType.INT).add("name_left", TypeProtos.MinorType.VARCHAR).buildSchema();
final RowSet.SingleRowSet emptyLeftRowSet_1 = fixture.rowSetBuilder(leftSchemaWithImplicit).build();
final RowSet.SingleRowSet nonEmptyLeftRowSet_1 = fixture.rowSetBuilder(leftSchemaWithImplicit).addRow(1, 1, 10, "item1").build();
// Create left input schema for first batch
TupleMetadata leftSchema2 = new SchemaBuilder().add(popConfig_1.getImplicitRIDColumn(), TypeProtos.MinorType.INT).add("id_left_new", TypeProtos.MinorType.INT).add("cost_left_new", TypeProtos.MinorType.INT).add("name_left_new", TypeProtos.MinorType.VARCHAR).buildSchema();
final RowSet.SingleRowSet emptyLeftRowSet_leftSchema2 = fixture.rowSetBuilder(leftSchema2).build();
final RowSet.SingleRowSet nonEmptyLeftRowSet_leftSchema2 = fixture.rowSetBuilder(leftSchema2).addRow(1, 6, 60, "item6").build();
leftContainer.add(emptyLeftRowSet_1.container());
leftContainer.add(nonEmptyLeftRowSet_1.container());
leftContainer.add(emptyLeftRowSet_leftSchema2.container());
leftContainer.add(nonEmptyLeftRowSet_leftSchema2.container());
// Get the left IterOutcomes for Lateral Join
leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
leftOutcomes.add(RecordBatch.IterOutcome.EMIT);
leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
leftOutcomes.add(RecordBatch.IterOutcome.EMIT);
final CloseableRecordBatch leftMockBatch_1 = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, leftContainer, leftOutcomes, leftContainer.get(0).getSchema());
// Get the right container with dummy data
TupleMetadata rightSchema2 = new SchemaBuilder().add(popConfig_1.getImplicitRIDColumn(), TypeProtos.MinorType.INT).add("id_right_new", TypeProtos.MinorType.INT).add("cost_right_new", TypeProtos.MinorType.VARCHAR).add("name_right_new", TypeProtos.MinorType.VARCHAR).buildSchema();
final RowSet.SingleRowSet emptyRightRowSet_rightSchema2 = fixture.rowSetBuilder(rightSchema2).build();
final RowSet.SingleRowSet nonEmptyRightRowSet_rightSchema2 = fixture.rowSetBuilder(rightSchema2).addRow(1, 5, "51", "item51").addRow(1, 6, "61", "item61").addRow(1, 7, "71", "item71").build();
rightContainer.add(emptyRightRowSet.container());
rightContainer.add(nonEmptyRightRowSet.container());
// non-empty batch with Ok_new_schema
rightContainer.add(nonEmptyRightRowSet_rightSchema2.container());
rightContainer.add(emptyRightRowSet_rightSchema2.container());
rightOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
rightOutcomes.add(RecordBatch.IterOutcome.EMIT);
rightOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
rightOutcomes.add(RecordBatch.IterOutcome.EMIT);
final CloseableRecordBatch rightMockBatch_1 = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, rightContainer, rightOutcomes, rightContainer.get(0).getSchema());
final LateralJoinBatch lowerLevelLateral = new LateralJoinBatch(popConfig_1, fixture.getFragmentContext(), leftMockBatch_1, rightMockBatch_1);
// ** Prepare second pair of left and right batch for upper level Lateral_2 **
// Create left input schema for first batch
TupleMetadata leftSchema3 = new SchemaBuilder().add("id_left_left", TypeProtos.MinorType.INT).add("cost_left_left", TypeProtos.MinorType.INT).add("name_left_left", TypeProtos.MinorType.VARCHAR).buildSchema();
final RowSet.SingleRowSet emptyLeftRowSet_leftSchema3 = fixture.rowSetBuilder(leftSchema3).build();
final RowSet.SingleRowSet nonEmptyLeftRowSet_leftSchema3 = fixture.rowSetBuilder(leftSchema3).addRow(6, 60, "item6").build();
// Get left input schema for second left batch
TupleMetadata leftSchema4 = new SchemaBuilder().add("id_left_left_new", TypeProtos.MinorType.INT).add("cost_left_left_new", TypeProtos.MinorType.VARCHAR).add("name_left_left_new", TypeProtos.MinorType.VARCHAR).buildSchema();
final RowSet.SingleRowSet nonEmptyLeftRowSet_leftSchema4 = fixture.rowSetBuilder(leftSchema4).addRow(100, "100", "item100").build();
// Build Left container for upper level LATERAL operator
final List<VectorContainer> leftContainer2 = new ArrayList<>(5);
// Get the left container with dummy data
leftContainer2.add(emptyLeftRowSet_leftSchema3.container());
leftContainer2.add(nonEmptyLeftRowSet_leftSchema3.container());
leftContainer2.add(nonEmptyLeftRowSet_leftSchema4.container());
// Get the left container outcomes for upper level LATERAL operator
final List<RecordBatch.IterOutcome> leftOutcomes2 = new ArrayList<>(5);
leftOutcomes2.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
leftOutcomes2.add(RecordBatch.IterOutcome.OK);
leftOutcomes2.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
final CloseableRecordBatch leftMockBatch_2 = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, leftContainer2, leftOutcomes2, leftContainer2.get(0).getSchema());
final LateralJoinBatch upperLevelLateral = new LateralJoinBatch(popConfig_1, fixture.getFragmentContext(), leftMockBatch_2, lowerLevelLateral);
try {
// 3 for first batch on left side and another 3 for next left batch
final int expectedOutputRecordCount = 6;
int actualOutputRecordCount = 0;
assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == upperLevelLateral.next());
assertTrue(RecordBatch.IterOutcome.OK == upperLevelLateral.next());
actualOutputRecordCount += upperLevelLateral.getRecordCount();
assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == upperLevelLateral.next());
actualOutputRecordCount += upperLevelLateral.getRecordCount();
assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == upperLevelLateral.next());
actualOutputRecordCount += upperLevelLateral.getRecordCount();
assertTrue(RecordBatch.IterOutcome.OK == upperLevelLateral.next());
actualOutputRecordCount += upperLevelLateral.getRecordCount();
assertTrue(RecordBatch.IterOutcome.NONE == upperLevelLateral.next());
assertTrue(actualOutputRecordCount == expectedOutputRecordCount);
} catch (AssertionError | Exception error) {
fail();
} finally {
// Close all the resources for this test case
upperLevelLateral.close();
leftMockBatch_2.close();
lowerLevelLateral.close();
leftMockBatch_1.close();
rightMockBatch_1.close();
leftContainer2.clear();
leftOutcomes2.clear();
}
}
Aggregations