Examples with MockRecordBatch org.apache.drill.exec.physical.impl.MockRecordBatch used on opensource projects

Example 31 with MockRecordBatch

use of org.apache.drill.exec.physical.impl.MockRecordBatch in project drill by apache.

the class TestHashJoinOutcome method testHashJoinWhenProbeIsNONE.

/**
 * Testing for DRILL-6755: No Hash Table is built when the first probe batch is NONE
 */
@Test
public void testHashJoinWhenProbeIsNONE() {
    inputOutcomesLeft.add(RecordBatch.IterOutcome.NONE);
    inputOutcomesRight.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    inputOutcomesRight.add(RecordBatch.IterOutcome.OK);
    inputOutcomesRight.add(RecordBatch.IterOutcome.NONE);
    // for the probe side input - use multiple batches (to check that they are all cleared/drained)
    final List<VectorContainer> buildSideinputContainer = new ArrayList<>(5);
    buildSideinputContainer.add(emptyInputRowSetRight.container());
    buildSideinputContainer.add(nonEmptyInputRowSetRight.container());
    RowSet.SingleRowSet secondInputRowSetRight = operatorFixture.rowSetBuilder(inputSchemaRight).addRow(456).build();
    RowSet.SingleRowSet thirdInputRowSetRight = operatorFixture.rowSetBuilder(inputSchemaRight).addRow(789).build();
    buildSideinputContainer.add(secondInputRowSetRight.container());
    buildSideinputContainer.add(thirdInputRowSetRight.container());
    final MockRecordBatch mockInputBatchRight = new MockRecordBatch(operatorFixture.getFragmentContext(), opContext, buildSideinputContainer, inputOutcomesRight, batchSchemaRight);
    final MockRecordBatch mockInputBatchLeft = new MockRecordBatch(operatorFixture.getFragmentContext(), opContext, inputContainerLeft, inputOutcomesLeft, batchSchemaLeft);
    List<JoinCondition> conditions = Lists.newArrayList();
    conditions.add(new JoinCondition(SqlKind.EQUALS.toString(), FieldReference.getWithQuotedRef("leftcol"), FieldReference.getWithQuotedRef("rightcol")));
    HashJoinPOP hjConf = new HashJoinPOP(null, null, conditions, JoinRelType.INNER);
    HashJoinBatch hjBatch = new HashJoinBatch(hjConf, operatorFixture.getFragmentContext(), mockInputBatchLeft, mockInputBatchRight);
    RecordBatch.IterOutcome gotOutcome = hjBatch.next();
    assertSame(gotOutcome, RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    gotOutcome = hjBatch.next();
    assertSame(gotOutcome, RecordBatch.IterOutcome.NONE);
    secondInputRowSetRight.clear();
    thirdInputRowSetRight.clear();
    buildSideinputContainer.clear();
}

Example 32 with MockRecordBatch

use of org.apache.drill.exec.physical.impl.MockRecordBatch in project drill by apache.

the class TestLateralJoinCorrectnessBatchProcessing method testLeftAndRightWithLastMissingRows_MultipleBatch.

@Test
public void testLeftAndRightWithLastMissingRows_MultipleBatch() throws Exception {
    leftContainer.add(nonEmptyLeftRowSet.container());
    // Get the left IterOutcomes for Lateral Join
    leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    // Create Left MockRecordBatch
    final CloseableRecordBatch leftMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, leftContainer, leftOutcomes, leftContainer.get(0).getSchema());
    // Get the right container with dummy data
    final RowSet.SingleRowSet nonEmptyRightRowSet2 = fixture.rowSetBuilder(rightSchema).addRow(1, 11, 110, "item11").addRow(2, 22, 220, "item22").build();
    final RowSet.SingleRowSet nonEmptyRightRowSet3 = fixture.rowSetBuilder(rightSchema).addRow(3, 33, 330, "item33_1").addRow(3, 33, 330, "item33_2").build();
    final RowSet.SingleRowSet expectedRowSet = fixture.rowSetBuilder(expectedSchema).addRow(1, 10, "item1", 11, 110, "item11").addRow(2, 20, "item2", 22, 220, "item22").addRow(3, 30, "item3", 33, 330, "item33_1").addRow(3, 30, "item3", 33, 330, "item33_2").build();
    rightContainer.add(emptyRightRowSet.container());
    rightContainer.add(nonEmptyRightRowSet2.container());
    rightContainer.add(nonEmptyRightRowSet3.container());
    rightOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    rightOutcomes.add(RecordBatch.IterOutcome.OK);
    rightOutcomes.add(RecordBatch.IterOutcome.EMIT);
    final CloseableRecordBatch rightMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, rightContainer, rightOutcomes, rightContainer.get(0).getSchema());
    final LateralJoinBatch ljBatch = new LateralJoinBatch(ljPopConfig, fixture.getFragmentContext(), leftMockBatch, rightMockBatch);
    try {
        assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == ljBatch.next());
        assertTrue(RecordBatch.IterOutcome.OK == ljBatch.next());
        assertTrue(ljBatch.getRecordCount() == (nonEmptyRightRowSet2.rowCount() + nonEmptyRightRowSet3.rowCount()));
        // verify results
        RowSet actualRowSet = DirectRowSet.fromContainer(ljBatch.getContainer());
        new RowSetComparison(expectedRowSet).verify(actualRowSet);
        assertTrue(RecordBatch.IterOutcome.NONE == ljBatch.next());
    } finally {
        // Close all the resources for this test case
        ljBatch.close();
        leftMockBatch.close();
        rightMockBatch.close();
        nonEmptyRightRowSet2.clear();
        expectedRowSet.clear();
    }
}

Example 33 with MockRecordBatch

use of org.apache.drill.exec.physical.impl.MockRecordBatch in project drill by apache.

the class TestLateralJoinCorrectnessBatchProcessing method testMultipleLeftAndRight_OutputFull_WithPendingLeftRow_LeftJoin.

@Test
public void testMultipleLeftAndRight_OutputFull_WithPendingLeftRow_LeftJoin() throws Exception {
    // Get the right container with dummy data
    final RowSet.SingleRowSet nonEmptyLeftRowSet2 = fixture.rowSetBuilder(leftSchema).addRow(5, 50, "item5").addRow(6, 60, "item6").build();
    leftContainer.add(nonEmptyLeftRowSet.container());
    leftContainer.add(nonEmptyLeftRowSet2.container());
    // Get the left IterOutcomes for Lateral Join
    leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    leftOutcomes.add(RecordBatch.IterOutcome.OK);
    // Create Left MockRecordBatch
    final CloseableRecordBatch leftMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, leftContainer, leftOutcomes, leftContainer.get(0).getSchema());
    // Get the right container with dummy data
    final RowSet.SingleRowSet nonEmptyRightRowSet2 = fixture.rowSetBuilder(rightSchema).addRow(1, 11, 110, "item11").addRow(2, 22, 220, "item22").build();
    final RowSet.SingleRowSet expectedRowSet = fixture.rowSetBuilder(expectedSchemaLeftJoin).addRow(1, 10, "item1", 11, 110, "item11").addRow(2, 20, "item2", 22, 220, "item22").addRow(3, 30, "item3", null, null, null).build();
    final RowSet.SingleRowSet expectedRowSet1 = fixture.rowSetBuilder(expectedSchemaLeftJoin).addRow(4, 40, "item4", null, null, null).addRow(5, 50, "item5", null, null, null).addRow(6, 60, "item6", null, null, null).build();
    rightContainer.add(emptyRightRowSet.container());
    rightContainer.add(nonEmptyRightRowSet2.container());
    rightContainer.add(emptyRightRowSet.container());
    rightOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    rightOutcomes.add(RecordBatch.IterOutcome.EMIT);
    rightOutcomes.add(RecordBatch.IterOutcome.EMIT);
    final CloseableRecordBatch rightMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, rightContainer, rightOutcomes, rightContainer.get(0).getSchema());
    LateralJoinPOP ljPopConfig = new LateralJoinPOP(null, null, JoinRelType.LEFT, DrillLateralJoinRelBase.IMPLICIT_COLUMN, Lists.newArrayList());
    final LateralJoinBatch ljBatch = new LateralJoinBatch(ljPopConfig, fixture.getFragmentContext(), leftMockBatch, rightMockBatch);
    ljBatch.setMaxOutputRowCount(3);
    ljBatch.setUseMemoryManager(false);
    try {
        assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == ljBatch.next());
        assertTrue(RecordBatch.IterOutcome.OK == ljBatch.next());
        assertTrue(ljBatch.getRecordCount() == 3);
        // verify results
        RowSet actualRowSet = DirectRowSet.fromContainer(ljBatch.getContainer());
        new RowSetComparison(expectedRowSet).verify(actualRowSet);
        // Release output container memory for this batch as other operators will do
        VectorAccessibleUtilities.clear(ljBatch);
        assertTrue(RecordBatch.IterOutcome.OK == ljBatch.next());
        assertTrue(ljBatch.getRecordCount() == (nonEmptyLeftRowSet.rowCount() + nonEmptyLeftRowSet2.rowCount() - 3));
        // verify results
        RowSet actualRowSet2 = DirectRowSet.fromContainer(ljBatch.getContainer());
        new RowSetComparison(expectedRowSet1).verify(actualRowSet2);
        assertTrue(RecordBatch.IterOutcome.NONE == ljBatch.next());
    } finally {
        // Close all the resources for this test case
        ljBatch.close();
        leftMockBatch.close();
        rightMockBatch.close();
        nonEmptyLeftRowSet2.clear();
        nonEmptyRightRowSet2.clear();
        expectedRowSet.clear();
        expectedRowSet1.clear();
    }
}

Example 34 with MockRecordBatch

use of org.apache.drill.exec.physical.impl.MockRecordBatch in project drill by apache.

the class TestLateralJoinCorrectnessBatchProcessing method testLeftAndRightWithLastMissingRows_LeftJoin_MultipleBatch.

@Test
public void testLeftAndRightWithLastMissingRows_LeftJoin_MultipleBatch() throws Exception {
    leftContainer.add(nonEmptyLeftRowSet.container());
    // Get the left IterOutcomes for Lateral Join
    leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    // Create Left MockRecordBatch
    final CloseableRecordBatch leftMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, leftContainer, leftOutcomes, leftContainer.get(0).getSchema());
    // Get the right container with dummy data
    final RowSet.SingleRowSet nonEmptyRightRowSet2 = fixture.rowSetBuilder(rightSchema).addRow(1, 11, 110, "item11").addRow(2, 22, 220, "item22").build();
    final RowSet.SingleRowSet nonEmptyRightRowSet3 = fixture.rowSetBuilder(rightSchema).addRow(3, 33, 330, "item33_1").addRow(3, 33, 330, "item33_2").build();
    final RowSet.SingleRowSet expectedRowSet = fixture.rowSetBuilder(expectedSchemaLeftJoin).addRow(1, 10, "item1", 11, 110, "item11").addRow(2, 20, "item2", 22, 220, "item22").addRow(3, 30, "item3", 33, 330, "item33_1").addRow(3, 30, "item3", 33, 330, "item33_2").addRow(4, 40, "item4", null, null, null).build();
    rightContainer.add(emptyRightRowSet.container());
    rightContainer.add(nonEmptyRightRowSet2.container());
    rightContainer.add(nonEmptyRightRowSet3.container());
    rightOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    rightOutcomes.add(RecordBatch.IterOutcome.OK);
    rightOutcomes.add(RecordBatch.IterOutcome.EMIT);
    final CloseableRecordBatch rightMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, rightContainer, rightOutcomes, rightContainer.get(0).getSchema());
    LateralJoinPOP ljPopConfig = new LateralJoinPOP(null, null, JoinRelType.LEFT, DrillLateralJoinRelBase.IMPLICIT_COLUMN, Lists.newArrayList());
    final LateralJoinBatch ljBatch = new LateralJoinBatch(ljPopConfig, fixture.getFragmentContext(), leftMockBatch, rightMockBatch);
    try {
        assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == ljBatch.next());
        assertTrue(RecordBatch.IterOutcome.OK == ljBatch.next());
        assertTrue(ljBatch.getRecordCount() == (1 + nonEmptyRightRowSet2.rowCount() + nonEmptyRightRowSet3.rowCount()));
        // verify results
        RowSet actualRowSet = DirectRowSet.fromContainer(ljBatch.getContainer());
        new RowSetComparison(expectedRowSet).verify(actualRowSet);
        assertTrue(RecordBatch.IterOutcome.NONE == ljBatch.next());
    } finally {
        // Close all the resources for this test case
        ljBatch.close();
        leftMockBatch.close();
        rightMockBatch.close();
        nonEmptyRightRowSet2.clear();
        expectedRowSet.clear();
    }
}

Example 35 with MockRecordBatch

use of org.apache.drill.exec.physical.impl.MockRecordBatch in project drill by apache.

the class TestLateralJoinCorrectnessBatchProcessing method testLeftAndRightWithMissingRows_LeftJoin_SingleBatch.

@Test
public void testLeftAndRightWithMissingRows_LeftJoin_SingleBatch() throws Exception {
    leftContainer.add(nonEmptyLeftRowSet.container());
    // Get the left IterOutcomes for Lateral Join
    leftOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    // Create Left MockRecordBatch
    final CloseableRecordBatch leftMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, leftContainer, leftOutcomes, leftContainer.get(0).getSchema());
    // Get the right container with dummy data
    final RowSet.SingleRowSet nonEmptyRightRowSet2 = fixture.rowSetBuilder(rightSchema).addRow(1, 11, 110, "item11").addRow(4, 44, 440, "item44").build();
    final RowSet.SingleRowSet expectedRowSet = fixture.rowSetBuilder(expectedSchemaLeftJoin).addRow(1, 10, "item1", 11, 110, "item11").addRow(2, 20, "item2", null, null, null).addRow(3, 30, "item3", null, null, null).addRow(4, 40, "item4", 44, 440, "item44").build();
    rightContainer.add(emptyRightRowSet.container());
    rightContainer.add(nonEmptyRightRowSet2.container());
    rightOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    rightOutcomes.add(RecordBatch.IterOutcome.EMIT);
    final CloseableRecordBatch rightMockBatch = new MockRecordBatch(fixture.getFragmentContext(), operatorContext, rightContainer, rightOutcomes, rightContainer.get(0).getSchema());
    LateralJoinPOP ljPopConfig = new LateralJoinPOP(null, null, JoinRelType.LEFT, DrillLateralJoinRelBase.IMPLICIT_COLUMN, Lists.newArrayList());
    final LateralJoinBatch ljBatch = new LateralJoinBatch(ljPopConfig, fixture.getFragmentContext(), leftMockBatch, rightMockBatch);
    try {
        assertTrue(RecordBatch.IterOutcome.OK_NEW_SCHEMA == ljBatch.next());
        assertTrue(RecordBatch.IterOutcome.OK == ljBatch.next());
        assertTrue(ljBatch.getRecordCount() == nonEmptyLeftRowSet.rowCount());
        // verify results
        RowSet actualRowSet = DirectRowSet.fromContainer(ljBatch.getContainer());
        new RowSetComparison(expectedRowSet).verify(actualRowSet);
        assertTrue(RecordBatch.IterOutcome.NONE == ljBatch.next());
    } finally {
        // Close all the resources for this test case
        ljBatch.close();
        leftMockBatch.close();
        rightMockBatch.close();
        nonEmptyRightRowSet2.clear();
        expectedRowSet.clear();
    }
}