Examples with Page com.facebook.presto.common.Page used on opensource projects

Example 31 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestHashJoinOperator method getPages.

private static List<Page> getPages(PageBuffer pageBuffer) {
    List<Page> result = new ArrayList<>();
    Page page = pageBuffer.poll();
    while (page != null) {
        result.add(page);
        page = pageBuffer.poll();
    }
    return result;
}

Example 32 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestHashJoinOperator method testFullOuterJoinWithEmptyLookupSource.

@Test(dataProvider = "hashJoinTestValues")
public void testFullOuterJoinWithEmptyLookupSource(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
    TaskContext taskContext = createTaskContext();
    // build factory
    List<Type> buildTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), buildTypes);
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = probePages.row("a").row("b").row((String) null).row("c").build();
    OperatorFactory joinOperatorFactory = new LookupJoinOperators().fullOuterJoin(0, new PlanNodeId("test"), lookupSourceFactoryManager, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    // expected
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row("a", null).row("b", null).row(null, null).row("c", null).build();
    assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}

Example 33 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestHashJoinOperator method testInnerJoinWithEmptyLookupSource.

@Test(dataProvider = "hashJoinTestValues")
public void testInnerJoinWithEmptyLookupSource(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
    TaskContext taskContext = createTaskContext();
    // build factory
    List<Type> buildTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), buildTypes);
    BuildSideSetup buildSideSetup = setupBuildSide(parallelBuild, taskContext, Ints.asList(0), buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
    OperatorFactory joinOperatorFactory = new LookupJoinOperators().innerJoin(0, new PlanNodeId("test"), lookupSourceFactoryManager, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY);
    // drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    Operator operator = joinOperatorFactory.createOperator(taskContext.addPipelineContext(0, true, true, false).addDriverContext());
    List<Page> pages = probePages.row("test").build();
    operator.addInput(pages.get(0));
    Page outputPage = operator.getOutput();
    assertNull(outputPage);
}

Example 34 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestHashJoinOperator method testYield.

@Test
public void testYield() {
    // create a filter function that yields for every probe match
    // verify we will yield #match times totally
    TaskContext taskContext = createTaskContext();
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    // force a yield for every match
    AtomicInteger filterFunctionCalls = new AtomicInteger();
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> {
        filterFunctionCalls.incrementAndGet();
        driverContext.getYieldSignal().forceYieldForTesting();
        return true;
    }));
    // build with 40 entries
    int entries = 40;
    RowPagesBuilder buildPages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(BIGINT)).addSequencePage(entries, 42);
    BuildSideSetup buildSideSetup = setupBuildSide(true, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe matching the above 40 entries
    RowPagesBuilder probePages = rowPagesBuilder(false, Ints.asList(0), ImmutableList.of(BIGINT));
    List<Page> probeInput = probePages.addSequencePage(100, 0).build();
    OperatorFactory joinOperatorFactory = LOOKUP_JOIN_OPERATORS.innerJoin(0, new PlanNodeId("test"), lookupSourceFactory, probePages.getTypes(), Ints.asList(0), getHashChannelAsInt(probePages), Optional.empty(), OptionalInt.of(1), PARTITIONING_SPILLER_FACTORY);
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(buildSideSetup);
    Operator operator = joinOperatorFactory.createOperator(driverContext);
    assertTrue(operator.needsInput());
    operator.addInput(probeInput.get(0));
    operator.finish();
    // we will yield 40 times due to filterFunction
    for (int i = 0; i < entries; i++) {
        driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
        filterFunctionCalls.set(0);
        assertNull(operator.getOutput());
        assertEquals(filterFunctionCalls.get(), 1, "Expected join to stop processing (yield) after calling filter function once");
        driverContext.getYieldSignal().reset();
    }
    // delayed yield is not going to prevent operator from producing a page now (yield won't be forced because filter function won't be called anymore)
    driverContext.getYieldSignal().setWithDelay(5 * SECONDS.toNanos(1), driverContext.getYieldExecutor());
    // expect output page to be produced within few calls to getOutput(), e.g. to facilitate spill
    Page output = null;
    for (int i = 0; output == null && i < 5; i++) {
        output = operator.getOutput();
    }
    assertNotNull(output);
    driverContext.getYieldSignal().reset();
    // make sure we have all 4 entries
    assertEquals(output.getPositionCount(), entries);
}

Example 35 with Page

use of com.facebook.presto.common.Page in project presto by prestodb.

the class TestHashJoinOperator method testInnerJoinWithSpillWithEarlyTermination.

@Test(timeOut = 60000)
public void testInnerJoinWithSpillWithEarlyTermination() {
    TaskStateMachine taskStateMachine = new TaskStateMachine(new TaskId("query", 0, 0, 0), executor);
    TaskContext taskContext = TestingTaskContext.createTaskContext(executor, scheduledExecutor, TEST_SESSION, taskStateMachine);
    PipelineContext joinPipelineContext = taskContext.addPipelineContext(2, true, true, false);
    DriverContext joinDriverContext1 = joinPipelineContext.addDriverContext();
    DriverContext joinDriverContext2 = joinPipelineContext.addDriverContext();
    DriverContext joinDriverContext3 = joinPipelineContext.addDriverContext();
    // build factory
    RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(VARCHAR, BIGINT)).addSequencePage(4, 20, 200).addSequencePage(4, 20, 200).addSequencePage(4, 30, 300).addSequencePage(4, 40, 400);
    // force a yield for every match in LookupJoinOperator, set called to true after first
    AtomicBoolean called = new AtomicBoolean(false);
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction((leftPosition, leftPage, rightPosition, rightPage) -> {
        called.set(true);
        return true;
    });
    BuildSideSetup buildSideSetup = setupBuildSide(true, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), true, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    RowPagesBuilder probe1Pages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT)).row("no_match_1", 123_000L).row("no_match_2", 123_000L);
    RowPagesBuilder probe2Pages = rowPagesBuilder(true, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT)).row("20", 123_000L).row("20", 123_000L).pageBreak().addSequencePage(20, 0, 123_000).addSequencePage(10, 30, 123_000);
    OperatorFactory joinOperatorFactory = innerJoinOperatorFactory(lookupSourceFactoryManager, probe2Pages, PARTITIONING_SPILLER_FACTORY, OptionalInt.of(3));
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    List<Driver> buildDrivers = buildSideSetup.getBuildDrivers();
    int buildOperatorCount = buildDrivers.size();
    LookupSourceFactory lookupSourceFactory = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide());
    Operator lookupOperator1 = joinOperatorFactory.createOperator(joinDriverContext1);
    Operator lookupOperator2 = joinOperatorFactory.createOperator(joinDriverContext2);
    Operator lookupOperator3 = joinOperatorFactory.createOperator(joinDriverContext3);
    joinOperatorFactory.noMoreOperators();
    ListenableFuture<LookupSourceProvider> lookupSourceProvider = lookupSourceFactory.createLookupSourceProvider();
    while (!lookupSourceProvider.isDone()) {
        for (Driver buildDriver : buildDrivers) {
            checkErrors(taskStateMachine);
            buildDriver.process();
        }
    }
    getFutureValue(lookupSourceProvider).close();
    for (int i = 0; i < buildOperatorCount; i++) {
        revokeMemory(buildSideSetup.getBuildOperators().get(i));
    }
    for (Driver buildDriver : buildDrivers) {
        runDriverInThread(executor, buildDriver);
    }
    ValuesOperatorFactory valuesOperatorFactory1 = new ValuesOperatorFactory(17, new PlanNodeId("values1"), probe1Pages.build());
    ValuesOperatorFactory valuesOperatorFactory2 = new ValuesOperatorFactory(18, new PlanNodeId("values2"), probe2Pages.build());
    ValuesOperatorFactory valuesOperatorFactory3 = new ValuesOperatorFactory(18, new PlanNodeId("values3"), ImmutableList.of());
    PageBuffer pageBuffer = new PageBuffer(10);
    PageBufferOperatorFactory pageBufferOperatorFactory = new PageBufferOperatorFactory(19, new PlanNodeId("pageBuffer"), pageBuffer);
    Driver joinDriver1 = Driver.createDriver(joinDriverContext1, valuesOperatorFactory1.createOperator(joinDriverContext1), lookupOperator1, pageBufferOperatorFactory.createOperator(joinDriverContext1));
    Driver joinDriver2 = Driver.createDriver(joinDriverContext2, valuesOperatorFactory2.createOperator(joinDriverContext2), lookupOperator2, pageBufferOperatorFactory.createOperator(joinDriverContext2));
    Driver joinDriver3 = Driver.createDriver(joinDriverContext3, valuesOperatorFactory3.createOperator(joinDriverContext3), lookupOperator3, pageBufferOperatorFactory.createOperator(joinDriverContext3));
    joinDriver3.close();
    joinDriver3.process();
    while (!called.get()) {
        checkErrors(taskStateMachine);
        processRow(joinDriver1, taskStateMachine);
        processRow(joinDriver2, taskStateMachine);
    }
    joinDriver1.close();
    joinDriver1.process();
    while (!joinDriver2.isFinished()) {
        processRow(joinDriver2, taskStateMachine);
    }
    checkErrors(taskStateMachine);
    List<Page> pages = getPages(pageBuffer);
    MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probe2Pages.getTypesWithoutHash(), buildPages.getTypesWithoutHash())).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("30", 123_000L, "30", 300L).row("31", 123_001L, "31", 301L).row("32", 123_002L, "32", 302L).row("33", 123_003L, "33", 303L).build();
    assertEqualsIgnoreOrder(getProperColumns(lookupOperator1, concat(probe2Pages.getTypes(), buildPages.getTypes()), probe2Pages, pages).getMaterializedRows(), expected.getMaterializedRows());
}