Examples with PlanNodeId com.facebook.presto.spi.plan.PlanNodeId used on opensource projects

Example 6 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestHashJoinOperator method testInnerJoinWithEmptyLookupSource.

@Test(dataProvider = "hashJoinTestValues")
public void testInnerJoinWithEmptyLookupSource(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
    TaskContext taskContext = createTaskContext();
    // build factory
    List<Type> buildTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), buildTypes);
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    OperatorFactory joinOperatorFactory = new LookupJoinOperators().innerJoin(0, new PlanNodeId("test"), lookupSourceFactoryManager, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY);
    // drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    Operator operator = joinOperatorFactory.createOperator(taskContext.addPipelineContext(0, true, true, false).addDriverContext());
    List<Page> pages = probePages.row("test").build();
    operator.addInput(pages.get(0));
    Page outputPage = operator.getOutput();
    assertNull(outputPage);
}

Example 7 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestHashJoinOperator method testYield.

@Test
public void testYield() {
    // create a filter function that yields for every probe match
    // verify we will yield #match times totally
    TaskContext taskContext = createTaskContext();
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    // force a yield for every match
    AtomicInteger filterFunctionCalls = new AtomicInteger();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> {
        filterFunctionCalls.incrementAndGet();
        driverContext.getYieldSignal().forceYieldForTesting();
        return true;
    }));
    // build with 40 entries
    int entries = 40;
    RowPagesBuilder buildPages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(BIGINT)).addSequencePage(entries, 42);
    BuildSideSetup buildSideSetup = setupBuildSide(true, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe matching the above 40 entries
    RowPagesBuilder probePages = rowPagesBuilder(false, Ints.asList(0), ImmutableList.of(BIGINT));
    List<Page> probeInput = probePages.addSequencePage(100, 0).build();
    OperatorFactory joinOperatorFactory = LOOKUP_JOIN_OPERATORS.innerJoin(0, new PlanNodeId("test"), lookupSourceFactory, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY);
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    Operator operator = joinOperatorFactory.createOperator(driverContext);
    assertTrue(operator.needsInput());
    operator.addInput(probeInput.get(0));
    operator.finish();
    // we will yield 40 times due to filterFunction
    for (int i = 0; i < entries; i++) {
        driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
        filterFunctionCalls.set(0);
        assertNull(operator.getOutput());
        assertEquals(filterFunctionCalls.get(), 1, "Expected join to stop processing (yield) after calling filter function once");
        driverContext.getYieldSignal().reset();
    }
    // delayed yield is not going to prevent operator from producing a page now (yield won't be forced because filter function won't be called anymore)
    driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
    // expect output page to be produced within few calls to getOutput(), e.g. to facilitate spill
    Page output = null;
    for (int i = 0; output == null && i < 5; i++) {
        output = operator.getOutput();
    }
    assertNotNull(output);
    driverContext.getYieldSignal().reset();
    // make sure we have all 4 entries
    assertEquals(output.getPositionCount(), entries);
}

Example 8 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestHashJoinOperator method testInnerJoinWithSpillWithEarlyTermination.

@Test(timeOut = 60000)
public void testInnerJoinWithSpillWithEarlyTermination() {
    TaskStateMachine taskStateMachine = new TaskStateMachine(new TaskId("query", 0, 0, 0), executor);
    TaskContext taskContext = TestingTaskContext.createTaskContext(executor, scheduledExecutor, TEST_SESSION, taskStateMachine);
    PipelineContext joinPipelineContext = taskContext.addPipelineContext(2, true, true, false);
    DriverContext joinDriverContext1 = joinPipelineContext.addDriverContext();
    DriverContext joinDriverContext2 = joinPipelineContext.addDriverContext();
    DriverContext joinDriverContext3 = joinPipelineContext.addDriverContext();
    // build factory
    RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(VARCHAR, BIGINT)).addSequencePage(4, 20, 200).addSequencePage(4, 20, 200).addSequencePage(4, 30, 300).addSequencePage(4, 40, 400);
    // force a yield for every match in LookupJoinOperator, set called to true after first
    AtomicBoolean called = new AtomicBoolean(false);
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction((leftPosition, leftPage, rightPosition, rightPage) -> {
        called.set(true);
        return true;
    });
    BuildSideSetup buildSideSetup = setupBuildSide(true, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), true, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    RowPagesBuilder probe1Pages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT)).row("no_match_1", 123_000L).row("no_match_2", 123_000L);
    RowPagesBuilder probe2Pages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT)).row("20", 123_000L).row("20", 123_000L).pageBreak().addSequencePage(20, 0, 123_000).addSequencePage(10, 30, 123_000);
    OperatorFactory joinOperatorFactory = innerJoinOperatorFactory(lookupSourceFactoryManager, probe2Pages, PARTITIONING_SPILLER_FACTORY, OptionalInt.of(3));
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    List<Driver> buildDrivers = buildSideSetup.getBuildDrivers();
    int buildOperatorCount = buildDrivers.size();
    LookupSourceFactory lookupSourceFactory = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide());
    Operator lookupOperator1 = joinOperatorFactory.createOperator(joinDriverContext1);
    Operator lookupOperator2 = joinOperatorFactory.createOperator(joinDriverContext2);
    Operator lookupOperator3 = joinOperatorFactory.createOperator(joinDriverContext3);
    joinOperatorFactory.noMoreOperators();
    ListenableFuture<LookupSourceProvider> lookupSourceProvider = lookupSourceFactory.createLookupSourceProvider();
    while (!lookupSourceProvider.isDone()) {
        for (Driver buildDriver : buildDrivers) {
            checkErrors(taskStateMachine);
            buildDriver.process();
        }
    }
    getFutureValue(lookupSourceProvider).close();
    for (int i = 0; i < buildOperatorCount; i++) {
        revokeMemory(buildSideSetup.getBuildOperators().get(i));
    }
    for (Driver buildDriver : buildDrivers) {
        runDriverInThread(executor, buildDriver);
    }
    ValuesOperatorFactory valuesOperatorFactory1 = new ValuesOperatorFactory(17, new PlanNodeId("values1"), probe1Pages.build());
    ValuesOperatorFactory valuesOperatorFactory2 = new ValuesOperatorFactory(18, new PlanNodeId("values2"), probe2Pages.build());
    ValuesOperatorFactory valuesOperatorFactory3 = new ValuesOperatorFactory(18, new PlanNodeId("values3"), ImmutableList.of());
    PageBuffer pageBuffer = new PageBuffer(10);
    PageBufferOperatorFactory pageBufferOperatorFactory = new PageBufferOperatorFactory(19, new PlanNodeId("pageBuffer"), pageBuffer);
    Driver joinDriver1 = Driver.createDriver(joinDriverContext1, valuesOperatorFactory1.createOperator(joinDriverContext1), lookupOperator1, pageBufferOperatorFactory.createOperator(joinDriverContext1));
    Driver joinDriver2 = Driver.createDriver(joinDriverContext2, valuesOperatorFactory2.createOperator(joinDriverContext2), lookupOperator2, pageBufferOperatorFactory.createOperator(joinDriverContext2));
    Driver joinDriver3 = Driver.createDriver(joinDriverContext3, valuesOperatorFactory3.createOperator(joinDriverContext3), lookupOperator3, pageBufferOperatorFactory.createOperator(joinDriverContext3));
    joinDriver3.close();
    joinDriver3.process();
    while (!called.get()) {
        checkErrors(taskStateMachine);
        processRow(joinDriver1, taskStateMachine);
        processRow(joinDriver2, taskStateMachine);
    }
    joinDriver1.close();
    joinDriver1.process();
    while (!joinDriver2.isFinished()) {
        processRow(joinDriver2, taskStateMachine);
    }
    checkErrors(taskStateMachine);
    List<Page> pages = getPages(pageBuffer);
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probe2Pages.getTypesWithoutHash(), buildPages.getTypesWithoutHash())).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("30", 123_000L, "30", 300L).row("31", 123_001L, "31", 301L).row("32", 123_002L, "32", 302L).row("33", 123_003L, "33", 303L).build();
    assertEqualsIgnoreOrder(getProperColumns(lookupOperator1, concat(probe2Pages.getTypes(), buildPages.getTypes()), probe2Pages, pages).getMaterializedRows(), expected.getMaterializedRows());
}

Example 9 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestDistinctLimitOperator method testDistinctLimitValuesLessThanLimit.

@Test(dataProvider = "hashEnabledValues")
public void testDistinctLimitValuesLessThanLimit(boolean hashEnabled) {
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(3, 1).addSequencePage(3, 2).build();
    OperatorFactory operatorFactory = new DistinctLimitOperator.DistinctLimitOperatorFactory(0, new PlanNodeId("test"), rowPagesBuilder.getTypes(), Ints.asList(0), 5, rowPagesBuilder.getHashChannel(), joinCompiler);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT).row(1L).row(2L).row(3L).row(4L).build();
    assertOperatorEquals(operatorFactory, driverContext, input, expected, hashEnabled, ImmutableList.of(1));
}

Example 10 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestDistinctLimitOperator method testDistinctLimit.

@Test(dataProvider = "hashEnabledValues")
public void testDistinctLimit(boolean hashEnabled) {
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(3, 1).addSequencePage(5, 2).build();
    OperatorFactory operatorFactory = new DistinctLimitOperator.DistinctLimitOperatorFactory(0, new PlanNodeId("test"), rowPagesBuilder.getTypes(), Ints.asList(0), 5, rowPagesBuilder.getHashChannel(), joinCompiler);
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT).row(1L).row(2L).row(3L).row(4L).row(5L).build();
    assertOperatorEquals(operatorFactory, driverContext, input, expected, hashEnabled, ImmutableList.of(1));
}