Examples with PlanNodeId com.facebook.presto.sql.planner.plan.PlanNodeId used on opensource projects

Example 26 with PlanNodeId

use of com.facebook.presto.sql.planner.plan.PlanNodeId in project presto by prestodb.

the class TestHashAggregationOperator method testMemoryLimit.

@Test(dataProvider = "hashEnabled", expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded local memory limit of 10B")
public void testMemoryLimit(boolean hashEnabled) {
    MetadataManager metadata = MetadataManager.createTestMetadataManager();
    InternalAggregationFunction maxVarcharColumn = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature("max", AGGREGATE, parseTypeSignature(StandardTypes.VARCHAR), parseTypeSignature(StandardTypes.VARCHAR)));
    List<Integer> hashChannels = Ints.asList(1);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, hashChannels, VARCHAR, BIGINT, VARCHAR, BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(10, 100, 0, 100, 0).addSequencePage(10, 100, 0, 200, 0).addSequencePage(10, 100, 0, 300, 0).build();
    DriverContext driverContext = createTaskContext(executor, TEST_SESSION, new DataSize(10, Unit.BYTE)).addPipelineContext(0, true, true).addDriverContext();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), Step.SINGLE, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(3), Optional.empty()), LONG_AVERAGE.bind(ImmutableList.of(3), Optional.empty()), maxVarcharColumn.bind(ImmutableList.of(2), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, new DataSize(16, MEGABYTE), joinCompiler);
    toPages(operatorFactory, driverContext, input);
}

Example 27 with PlanNodeId

use of com.facebook.presto.sql.planner.plan.PlanNodeId in project presto by prestodb.

the class TestHashAggregationOperator method testMultiplePartialFlushes.

@Test(dataProvider = "hashEnabledAndMemoryLimitBeforeSpillValues")
public void testMultiplePartialFlushes(boolean hashEnabled, long memoryLimitBeforeSpill, long memoryLimitForMergeWithMemory) throws Exception {
    List<Integer> hashChannels = Ints.asList(0);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, hashChannels, BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(500, 0).addSequencePage(500, 500).addSequencePage(500, 1000).addSequencePage(500, 1500).build();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), Step.PARTIAL, ImmutableList.of(LONG_SUM.bind(ImmutableList.of(0), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 100_000, new DataSize(1, Unit.KILOBYTE), memoryLimitBeforeSpill > 0, succinctBytes(memoryLimitBeforeSpill), succinctBytes(memoryLimitForMergeWithMemory), spillerFactory, joinCompiler);
    DriverContext driverContext = createTaskContext(executor, TEST_SESSION, new DataSize(4, Unit.KILOBYTE)).addPipelineContext(0, true, true).addDriverContext();
    try (Operator operator = operatorFactory.createOperator(driverContext)) {
        List<Page> expectedPages = rowPagesBuilder(BIGINT, BIGINT).addSequencePage(2000, 0, 0).build();
        MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT, BIGINT).pages(expectedPages).build();
        Iterator<Page> inputIterator = input.iterator();
        // Fill up the aggregation
        while (operator.needsInput() && inputIterator.hasNext()) {
            operator.addInput(inputIterator.next());
        }
        // Drain the output (partial flush)
        List<Page> outputPages = new ArrayList<>();
        while (true) {
            Page output = operator.getOutput();
            if (output == null) {
                break;
            }
            outputPages.add(output);
        }
        // There should be some pages that were drained
        assertTrue(!outputPages.isEmpty());
        // The operator need input again since this was a partial flush
        assertTrue(operator.needsInput());
        // Now, drive the operator to completion
        outputPages.addAll(toPages(operator, inputIterator));
        MaterializedResult actual;
        if (hashEnabled) {
            // Drop the hashChannel for all pages
            List<Page> actualPages = dropChannel(outputPages, ImmutableList.of(1));
            List<Type> expectedTypes = without(operator.getTypes(), ImmutableList.of(1));
            actual = toMaterializedResult(operator.getOperatorContext().getSession(), expectedTypes, actualPages);
        } else {
            actual = toMaterializedResult(operator.getOperatorContext().getSession(), operator.getTypes(), outputPages);
        }
        assertEquals(actual.getTypes(), expected.getTypes());
        assertEqualsIgnoreOrder(actual.getMaterializedRows(), expected.getMaterializedRows());
    }
}

Example 28 with PlanNodeId

use of com.facebook.presto.sql.planner.plan.PlanNodeId in project presto by prestodb.

the class TestHashAggregationOperator method testMergeWithMemorySpill.

@Test
public void testMergeWithMemorySpill() {
    List<Integer> hashChannels = Ints.asList(0);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(BIGINT);
    int smallPagesSpillThresholdSize = 150000;
    List<Page> input = rowPagesBuilder.addSequencePage(smallPagesSpillThresholdSize, 0).addSequencePage(10, smallPagesSpillThresholdSize).build();
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), hashChannels, ImmutableList.of(), Step.SINGLE, ImmutableList.of(LONG_SUM.bind(ImmutableList.of(0), Optional.empty())), rowPagesBuilder.getHashChannel(), Optional.empty(), 1, new DataSize(16, MEGABYTE), true, new DataSize(smallPagesSpillThresholdSize, Unit.BYTE), succinctBytes(Integer.MAX_VALUE), spillerFactory, joinCompiler);
    DriverContext driverContext = createTaskContext(executor, TEST_SESSION, new DataSize(1, Unit.KILOBYTE)).addPipelineContext(0, true, true).addDriverContext();
    MaterializedResult.Builder resultBuilder = resultBuilder(driverContext.getSession(), BIGINT);
    for (int i = 0; i < smallPagesSpillThresholdSize + 10; ++i) {
        resultBuilder.row((long) i, (long) i);
    }
    assertOperatorEqualsIgnoreOrder(operatorFactory, driverContext, input, resultBuilder.build(), false, Optional.of(hashChannels.size()));
}

Example 29 with PlanNodeId

use of com.facebook.presto.sql.planner.plan.PlanNodeId in project presto by prestodb.

the class TestHashJoinOperator method testOuterJoinWithNullProbeAndFilterFunction.

@Test(dataProvider = "hashEnabledValues")
public void testOuterJoinWithNullProbeAndFilterFunction(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
    TaskContext taskContext = createTaskContext();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftBlocks, rightPosition, rightBlocks) -> VARCHAR.getSlice(rightBlocks[0], rightPosition).toStringAscii().equals("a")));
    // build
    List<Type> buildTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), buildTypes).row("a").row("b").row("c");
    LookupSourceFactory lookupSourceFactory = buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction));
    // probe
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.row("a").row((String) null).row((String) null).row("a").row("b").build();
    OperatorFactory joinOperatorFactory = LOOKUP_JOIN_OPERATORS.probeOuterJoin(0, new PlanNodeId("test"), lookupSourceFactory, probePages.getTypes(), Ints.asList(0), probePages.getHashChannel(), Optional.empty());
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row("a", "a").row(null, null).row(null, null).row("a", "a").row("b", null).build();
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}

Example 30 with PlanNodeId

use of com.facebook.presto.sql.planner.plan.PlanNodeId in project presto by prestodb.

the class TestHashJoinOperator method testInnerJoinWithNullProbe.

@Test(dataProvider = "hashEnabledValues")
public void testInnerJoinWithNullProbe(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception {
    TaskContext taskContext = createTaskContext();
    // build
    List<Type> buildTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), buildTypes).row("a").row("b").row("c");
    LookupSourceFactory lookupSourceFactory = buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty());
    // probe
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.row("a").row((String) null).row((String) null).row("a").row("b").build();
    OperatorFactory joinOperatorFactory = LOOKUP_JOIN_OPERATORS.innerJoin(0, new PlanNodeId("test"), lookupSourceFactory, probePages.getTypes(), Ints.asList(0), probePages.getHashChannel(), Optional.empty());
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildPages.getTypes())).row("a", "a").row("a", "a").row("b", "b").build();
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}