Examples with PlanNodeId com.facebook.presto.spi.plan.PlanNodeId used on opensource projects

Example 41 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestHashAggregationOperator method testMergeWithMemorySpill.

@Test
public void testMergeWithMemorySpill() {
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(BIGINT);
    int smallPagesSpillThresholdSize = 150000;
    List<Page> input = rowPagesBuilder.addSequencePage(smallPagesSpillThresholdSize, 0).addSequencePage(10, smallPagesSpillThresholdSize).build();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), ImmutableList.of(0), ImmutableList.of(), Step.SINGLE, false, ImmutableList.of(LONG_SUM.bind(ImmutableList.of(0), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 1, Optional.of(new DataSize(16, MEGABYTE)), true, new DataSize(smallPagesSpillThresholdSize, Unit.BYTE), succinctBytes(Integer.MAX_VALUE), spillerFactory, joinCompiler, false);
    DriverContext driverContext = createDriverContext(smallPagesSpillThresholdSize);
    MaterializedResult.Builder resultBuilder = resultBuilder(driverContext.getSession(), BIGINT, BIGINT);
    for (int i = 0; i < smallPagesSpillThresholdSize + 10; ++i) {
        resultBuilder.row((long) i, (long) i);
    }
    assertOperatorEqualsIgnoreOrder(operatorFactory, driverContext, input, resultBuilder.build());
}

Example 42 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestHashAggregationOperator method testMask.

@Test
public void testMask() {
    int positions = 4;
    Block groupingBlock = RunLengthEncodedBlock.create(BIGINT, 1L, positions);
    Block countBlock = RunLengthEncodedBlock.create(BIGINT, 1L, positions);
    Block maskBlock = new ByteArrayBlock(positions, Optional.of(new boolean[] { false, false, true, true }), new byte[] { (byte) 0, (byte) 1, (byte) 0, (byte) 1 });
    Page page = new Page(groupingBlock, countBlock, maskBlock);
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), ImmutableList.of(0), ImmutableList.of(), Step.SINGLE, false, ImmutableList.of(COUNT.bind(ImmutableList.of(1), Optional.of(2))), Optional.empty(), Optional.empty(), 1, Optional.of(new DataSize(16, MEGABYTE)), false, new DataSize(16, MEGABYTE), new DataSize(16, MEGABYTE), new FailingSpillerFactory(), joinCompiler, false);
    List<Page> outputPages = toPages(operatorFactory, createDriverContext(), ImmutableList.of(page)).stream().filter(p -> p.getPositionCount() > 0).collect(toImmutableList());
    assertEquals(outputPages.size(), 1);
    Page outputPage = outputPages.get(0);
    assertEquals(outputPage.getBlock(0).getLong(0), 1L);
    assertEquals(outputPage.getBlock(1).getLong(0), 1L);
}

Example 43 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestHashAggregationOperator method testHashAggregationWithGlobals.

@Test(dataProvider = "hashEnabledAndMemoryLimitForMergeValues")
public void testHashAggregationWithGlobals(boolean hashEnabled, boolean spillEnabled, boolean revokeMemoryWhenAddingPages, long memoryLimitForMerge, long memoryLimitForMergeWithMemory) {
    InternalAggregationFunction countVarcharColumn = getAggregation("count", VARCHAR);
    InternalAggregationFunction countBooleanColumn = getAggregation("count", BOOLEAN);
    InternalAggregationFunction maxVarcharColumn = getAggregation("max", VARCHAR);
    Optional<Integer> groupIdChannel = Optional.of(1);
    List<Integer> groupByChannels = Ints.asList(1, 2);
    List<Integer> globalAggregationGroupIds = Ints.asList(42, 49);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, groupByChannels, VARCHAR, VARCHAR, VARCHAR, BIGINT, BIGINT, BOOLEAN);
    List<Page> input = rowPagesBuilder.build();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(VARCHAR, BIGINT), groupByChannels, globalAggregationGroupIds, Step.SINGLE, true, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(4), Optional.empty()), LONG_AVERAGE.bind(ImmutableList.of(4), Optional.empty()), maxVarcharColumn.bind(ImmutableList.of(2), Optional.empty()), countVarcharColumn.bind(ImmutableList.of(0), Optional.empty()), countBooleanColumn.bind(ImmutableList.of(5), Optional.empty())), rowPagesBuilder.getHashChannel(), groupIdChannel, 100_000, Optional.of(new DataSize(16, MEGABYTE)), spillEnabled, succinctBytes(memoryLimitForMerge), succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler, false);
    DriverContext driverContext = createDriverContext(memoryLimitForMerge);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), VARCHAR, BIGINT, BIGINT, BIGINT, DOUBLE, VARCHAR, BIGINT, BIGINT).row(null, 42L, 0L, null, null, null, 0L, 0L).row(null, 49L, 0L, null, null, null, 0L, 0L).build();
    assertOperatorEqualsIgnoreOrder(operatorFactory, driverContext, input, expected, hashEnabled, Optional.of(groupByChannels.size()), revokeMemoryWhenAddingPages);
}

Example 44 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestHashAggregationOperator method testMemoryTracking.

private void testMemoryTracking(boolean useSystemMemory) throws Exception {
    List<Integer> hashChannels = Ints.asList(0);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(false, hashChannels, BIGINT);
    Page input = getOnlyElement(rowPagesBuilder.addSequencePage(500, 0).build());
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), Step.SINGLE, ImmutableList.of(LONG_SUM.bind(ImmutableList.of(0), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, Optional.of(new DataSize(16, MEGABYTE)), joinCompiler, useSystemMemory);
    DriverContext driverContext = createDriverContext(1024);
    try (Operator operator = operatorFactory.createOperator(driverContext)) {
        assertTrue(operator.needsInput());
        operator.addInput(input);
        if (useSystemMemory) {
            assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
            assertEquals(driverContext.getMemoryUsage(), 0);
        } else {
            assertEquals(driverContext.getSystemMemoryUsage(), 0);
            assertThat(driverContext.getMemoryUsage()).isGreaterThan(0);
        }
        toPages(operator, emptyIterator());
    }
    assertEquals(driverContext.getSystemMemoryUsage(), 0);
    assertEquals(driverContext.getMemoryUsage(), 0);
}

Example 45 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestHashAggregationOperator method testMemoryLimit.

@Test(dataProvider = "hashEnabled", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded per-node user memory limit of 10B.*")
public void testMemoryLimit(boolean hashEnabled) {
    InternalAggregationFunction maxVarcharColumn = getAggregation("max", VARCHAR);
    List<Integer> hashChannels = Ints.asList(1);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, hashChannels, VARCHAR, BIGINT, VARCHAR, BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(10, 100, 0, 100, 0).addSequencePage(10, 100, 0, 200, 0).addSequencePage(10, 100, 0, 300, 0).build();
    DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION, new DataSize(10, Unit.BYTE)).addPipelineContext(0, true, true, false).addDriverContext();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), Step.SINGLE, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(3), Optional.empty()), LONG_AVERAGE.bind(ImmutableList.of(3), Optional.empty()), maxVarcharColumn.bind(ImmutableList.of(2), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, Optional.of(new DataSize(16, MEGABYTE)), joinCompiler, false);
    toPages(operatorFactory, driverContext, input);
}