Examples with Filter org.apache.drill.exec.physical.config.Filter used on opensource projects

Example 1 with Filter

use of org.apache.drill.exec.physical.config.Filter in project drill by apache.

the class TestMiniPlan method testUnionFilterAll.

@Test
@Ignore("DRILL-5327: A bug in UnionAll handling empty inputs from both sides")
public void testUnionFilterAll() throws Exception {
    List<String> leftJsonBatches = Lists.newArrayList("[{\"a\": 5, \"b\" : 1 }]");
    List<String> rightJsonBatches = Lists.newArrayList("[{\"a\": 50, \"b\" : 10 }]");
    RecordBatch leftScan = new JsonScanBuilder().jsonBatches(leftJsonBatches).columnsToRead("a", "b").build();
    RecordBatch leftFilter = new PopBuilder().physicalOperator(new Filter(null, parseExpr("a < 0"), 1.0f)).addInput(leftScan).build();
    RecordBatch rightScan = new JsonScanBuilder().jsonBatches(rightJsonBatches).columnsToRead("a", "b").build();
    RecordBatch rightFilter = new PopBuilder().physicalOperator(new Filter(null, parseExpr("a < 0"), 1.0f)).addInput(rightScan).build();
    RecordBatch batch = new PopBuilder().physicalOperator(// Children list is provided through RecordBatch
    new UnionAll(Collections.EMPTY_LIST)).addInput(leftFilter).addInput(rightFilter).build();
    BatchSchema expectedSchema = new SchemaBuilder().addNullable("a", TypeProtos.MinorType.BIGINT).addNullable("b", TypeProtos.MinorType.BIGINT).withSVMode(BatchSchema.SelectionVectorMode.NONE).build();
    new MiniPlanTestBuilder().root(batch).expectedSchema(expectedSchema).go();
}

Example 2 with Filter

use of org.apache.drill.exec.physical.config.Filter in project drill by axbaretto.

the class TestMiniPlan method testUnionFilter.

@Test
public void testUnionFilter() throws Exception {
    List<String> leftJsonBatches = Lists.newArrayList("[{\"a\": 5, \"b\" : 1 }]", "[{\"a\": 5, \"b\" : 5},{\"a\": 3, \"b\" : 8}]", "[{\"a\": 40, \"b\" : 3},{\"a\": 13, \"b\" : 100}]");
    List<String> rightJsonBatches = Lists.newArrayList("[{\"a\": 5, \"b\" : 10 }]", "[{\"a\": 50, \"b\" : 100}]");
    RecordBatch batch = new PopBuilder().physicalOperator(// Children list is provided through RecordBatch
    new UnionAll(Collections.<PhysicalOperator>emptyList())).addInputAsChild().physicalOperator(new Filter(null, parseExpr("a=5"), 1.0f)).addJsonScanAsChild().jsonBatches(leftJsonBatches).columnsToRead("a", "b").buildAddAsInput().buildAddAsInput().addInputAsChild().physicalOperator(new Filter(null, parseExpr("a=50"), 1.0f)).addJsonScanAsChild().jsonBatches(rightJsonBatches).columnsToRead("a", "b").buildAddAsInput().buildAddAsInput().build();
    BatchSchema expectedSchema = new SchemaBuilder().addNullable("a", TypeProtos.MinorType.BIGINT).addNullable("b", TypeProtos.MinorType.BIGINT).withSVMode(BatchSchema.SelectionVectorMode.NONE).build();
    new MiniPlanTestBuilder().root(batch).expectSchema(expectedSchema).baselineValues(5l, 1l).baselineValues(5l, 5l).baselineValues(50l, 100l).go();
}

Example 3 with Filter

use of org.apache.drill.exec.physical.config.Filter in project drill by apache.

the class TestFilterBatchEmitOutcome method testFilterNonEmptyFirst_NonEmptyOK_EmptyBatchEmitOutcome.

/**
 * Test to show empty batch with OK outcome is ignore and later non-empty batch with OK outcome produces an output
 * batch. Whereas a empty batch with EMIT outcome is not ignored and a empty output batch is returned with EMIT
 * outcome.
 * @throws Throwable
 */
@Test
public void testFilterNonEmptyFirst_NonEmptyOK_EmptyBatchEmitOutcome() throws Throwable {
    final RowSet.SingleRowSet nonEmptyInputRowSet2 = operatorFixture.rowSetBuilder(inputSchema).addRow(2, 20, "item2").build();
    inputContainer.add(nonEmptyInputRowSet.container());
    inputContainer.add(emptyInputRowSet.container());
    inputContainer.add(nonEmptyInputRowSet2.container());
    inputContainer.add(emptyInputRowSet.container());
    inputOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    inputOutcomes.add(RecordBatch.IterOutcome.OK);
    inputOutcomes.add(RecordBatch.IterOutcome.OK);
    inputOutcomes.add(RecordBatch.IterOutcome.EMIT);
    final MockRecordBatch mockInputBatch = new MockRecordBatch(operatorFixture.getFragmentContext(), opContext, inputContainer, inputOutcomes, emptyInputRowSet.container().getSchema());
    final Filter filterConf = new Filter(null, parseExpr("id_left>=1"), 1.0f);
    final FilterRecordBatch filterRecordBatch = new FilterRecordBatch(filterConf, mockInputBatch, operatorFixture.getFragmentContext());
    assertTrue(filterRecordBatch.next() == RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    outputRecordCount += filterRecordBatch.getRecordCount();
    assertTrue(filterRecordBatch.next() == RecordBatch.IterOutcome.OK);
    outputRecordCount += filterRecordBatch.getRecordCount();
    assertTrue(filterRecordBatch.next() == RecordBatch.IterOutcome.EMIT);
    outputRecordCount += filterRecordBatch.getRecordCount();
    assertEquals(2, outputRecordCount);
    // free up resources
    nonEmptyInputRowSet2.clear();
}

Example 4 with Filter

use of org.apache.drill.exec.physical.config.Filter in project drill by apache.

the class TestFilterBatchEmitOutcome method testFilterNonEmptyBatchEmitOutcome_WithNonMatchingCondition.

/**
 * Test to show if a non-empty batch is accompanied with EMIT outcome then Filter operator produces empty output
 * batch since filter condition is not satisfied by any data in incoming batch. This empty output batch is
 * accompanied with EMIT outcome.
 * @throws Throwable
 */
@Test
public void testFilterNonEmptyBatchEmitOutcome_WithNonMatchingCondition() throws Throwable {
    inputContainer.add(emptyInputRowSet.container());
    inputContainer.add(nonEmptyInputRowSet.container());
    inputOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    inputOutcomes.add(RecordBatch.IterOutcome.EMIT);
    final MockRecordBatch mockInputBatch = new MockRecordBatch(operatorFixture.getFragmentContext(), opContext, inputContainer, inputOutcomes, emptyInputRowSet.container().getSchema());
    final Filter filterConf = new Filter(null, parseExpr("id_left=2"), 1.0f);
    final FilterRecordBatch filterRecordBatch = new FilterRecordBatch(filterConf, mockInputBatch, operatorFixture.getFragmentContext());
    assertTrue(filterRecordBatch.next() == RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    outputRecordCount += filterRecordBatch.getRecordCount();
    assertTrue(filterRecordBatch.next() == RecordBatch.IterOutcome.EMIT);
    outputRecordCount += filterRecordBatch.getRecordCount();
    assertEquals(0, outputRecordCount);
}

Example 5 with Filter

use of org.apache.drill.exec.physical.config.Filter in project drill by apache.

the class TestFilterBatchEmitOutcome method testFilterNonEmptyFirst_EmptyOK_EmptyBatchEmitOutcome.

/**
 * Test to show if an empty batch is accompanied with OK outcome then that batch is ignored by Filter operator and
 * it doesn't return anything instead call's next() to get another batch. If the subsequent next() call returns empty
 * batch with EMIT outcome then Filter returns the EMIT outcome correctly rather than ignoring it because of empty
 * batch.
 * @throws Throwable
 */
@Test
public void testFilterNonEmptyFirst_EmptyOK_EmptyBatchEmitOutcome() throws Throwable {
    inputContainer.add(nonEmptyInputRowSet.container());
    inputContainer.add(emptyInputRowSet.container());
    inputContainer.add(emptyInputRowSet.container());
    inputContainer.add(emptyInputRowSet.container());
    inputOutcomes.add(RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    inputOutcomes.add(RecordBatch.IterOutcome.OK);
    inputOutcomes.add(RecordBatch.IterOutcome.EMIT);
    inputOutcomes.add(RecordBatch.IterOutcome.NONE);
    final MockRecordBatch mockInputBatch = new MockRecordBatch(operatorFixture.getFragmentContext(), opContext, inputContainer, inputOutcomes, emptyInputRowSet.container().getSchema());
    final Filter filterConf = new Filter(null, parseExpr("id_left=1"), 1.0f);
    final FilterRecordBatch filterRecordBatch = new FilterRecordBatch(filterConf, mockInputBatch, operatorFixture.getFragmentContext());
    assertTrue(filterRecordBatch.next() == RecordBatch.IterOutcome.OK_NEW_SCHEMA);
    outputRecordCount += filterRecordBatch.getRecordCount();
    // OK will not be received since it's was accompanied with empty batch
    assertTrue(filterRecordBatch.next() == RecordBatch.IterOutcome.EMIT);
    outputRecordCount += filterRecordBatch.getRecordCount();
    assertTrue(filterRecordBatch.next() == RecordBatch.IterOutcome.NONE);
    assertEquals(1, outputRecordCount);
}