Examples with MarkDistinctOperatorFactory com.facebook.presto.operator.MarkDistinctOperator.MarkDistinctOperatorFactory used on opensource projects

Example 1 with MarkDistinctOperatorFactory

use of com.facebook.presto.operator.MarkDistinctOperator.MarkDistinctOperatorFactory in project presto by prestodb.

the class TestMarkDistinctOperator method testMarkDistinct.

@Test(dataProvider = "hashEnabledValues")
public void testMarkDistinct(boolean hashEnabled) {
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(100, 0).addSequencePage(100, 0).build();
    OperatorFactory operatorFactory = new MarkDistinctOperatorFactory(0, new PlanNodeId("test"), rowPagesBuilder.getTypes(), ImmutableList.of(0), rowPagesBuilder.getHashChannel(), joinCompiler);
    MaterializedResult.Builder expected = resultBuilder(driverContext.getSession(), BIGINT, BOOLEAN);
    for (long i = 0; i < 100; i++) {
        expected.row(i, true);
        expected.row(i, false);
    }
    OperatorAssertion.assertOperatorEqualsIgnoreOrder(operatorFactory, driverContext, input, expected.build(), hashEnabled, Optional.of(1));
}

Example 2 with MarkDistinctOperatorFactory

use of com.facebook.presto.operator.MarkDistinctOperator.MarkDistinctOperatorFactory in project presto by prestodb.

the class TestMarkDistinctOperator method testMemoryReservationYield.

@Test(dataProvider = "dataType")
public void testMemoryReservationYield(Type type) {
    List<Page> input = createPagesWithDistinctHashKeys(type, 6_000, 600);
    OperatorFactory operatorFactory = new MarkDistinctOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(type), ImmutableList.of(0), Optional.of(1), joinCompiler);
    // get result with yield; pick a relatively small buffer for partitionRowCount's memory usage
    GroupByHashYieldAssertion.GroupByHashYieldResult result = finishOperatorWithYieldingGroupByHash(input, type, operatorFactory, operator -> ((MarkDistinctOperator) operator).getCapacity(), 1_400_000);
    assertGreaterThan(result.getYieldCount(), 5);
    assertGreaterThan(result.getMaxReservedBytes(), 20L << 20);
    int count = 0;
    for (Page page : result.getOutput()) {
        assertEquals(page.getChannelCount(), 3);
        for (int i = 0; i < page.getPositionCount(); i++) {
            assertEquals(page.getBlock(2).getByte(i), 1);
            count++;
        }
    }
    assertEquals(count, 6_000 * 600);
}

Example 3 with MarkDistinctOperatorFactory

use of com.facebook.presto.operator.MarkDistinctOperator.MarkDistinctOperatorFactory in project presto by prestodb.

the class TestMarkDistinctOperator method testRleDistinctMask.

@Test(dataProvider = "hashEnabledValues")
public void testRleDistinctMask(boolean hashEnabled) {
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT);
    List<Page> inputs = rowPagesBuilder.addSequencePage(100, 0).addSequencePage(100, 50).addSequencePage(1, 200).addSequencePage(1, 100).build();
    Page firstInput = inputs.get(0);
    Page secondInput = inputs.get(1);
    Page singleDistinctPage = inputs.get(2);
    Page singleNotDistinctPage = inputs.get(3);
    OperatorFactory operatorFactory = new MarkDistinctOperatorFactory(0, new PlanNodeId("test"), rowPagesBuilder.getTypes(), ImmutableList.of(0), rowPagesBuilder.getHashChannel(), joinCompiler);
    // mask channel is appended to the input
    int maskChannel = firstInput.getChannelCount();
    try (Operator operator = operatorFactory.createOperator(driverContext)) {
        operator.addInput(firstInput);
        Block allDistinctOutput = operator.getOutput().getBlock(maskChannel);
        operator.addInput(firstInput);
        Block noDistinctOutput = operator.getOutput().getBlock(maskChannel);
        // all distinct and no distinct conditions produce RLE blocks
        assertInstanceOf(allDistinctOutput, RunLengthEncodedBlock.class);
        assertTrue(BOOLEAN.getBoolean(allDistinctOutput, 0));
        assertInstanceOf(noDistinctOutput, RunLengthEncodedBlock.class);
        assertFalse(BOOLEAN.getBoolean(noDistinctOutput, 0));
        operator.addInput(secondInput);
        Block halfDistinctOutput = operator.getOutput().getBlock(maskChannel);
        // [0,50) is not distinct
        for (int position = 0; position < 50; position++) {
            assertFalse(BOOLEAN.getBoolean(halfDistinctOutput, position));
        }
        for (int position = 50; position < 100; position++) {
            assertTrue(BOOLEAN.getBoolean(halfDistinctOutput, position));
        }
        operator.addInput(singleDistinctPage);
        Block singleDistinctBlock = operator.getOutput().getBlock(maskChannel);
        assertFalse(singleDistinctBlock instanceof RunLengthEncodedBlock, "single position inputs should not be RLE");
        assertTrue(BOOLEAN.getBoolean(singleDistinctBlock, 0));
        operator.addInput(singleNotDistinctPage);
        Block singleNotDistinctBlock = operator.getOutput().getBlock(maskChannel);
        assertFalse(singleNotDistinctBlock instanceof RunLengthEncodedBlock, "single position inputs should not be RLE");
        assertFalse(BOOLEAN.getBoolean(singleNotDistinctBlock, 0));
    } catch (Exception e) {
        throwIfUnchecked(e);
        throw new RuntimeException(e);
    }
}