Search in sources :

Example 1 with PageBufferOperatorFactory

use of io.prestosql.operator.index.PageBufferOperator.PageBufferOperatorFactory in project hetu-core by openlookeng.

the class IndexBuildDriverFactoryProvider method createStreaming.

public DriverFactory createStreaming(PageBuffer pageBuffer, Page indexKeyTuple) {
    ImmutableList.Builder<OperatorFactory> operatorFactories = ImmutableList.<OperatorFactory>builder().addAll(coreOperatorFactories);
    if (dynamicTupleFilterFactory.isPresent()) {
        // Bind in a dynamic tuple filter if necessary
        operatorFactories.add(dynamicTupleFilterFactory.get().filterWithTuple(indexKeyTuple));
    }
    operatorFactories.add(new PageBufferOperatorFactory(outputOperatorId, planNodeId, pageBuffer));
    return new DriverFactory(pipelineId, inputDriver, false, operatorFactories.build(), OptionalInt.empty(), UNGROUPED_EXECUTION);
}
Also used : ImmutableList(com.google.common.collect.ImmutableList) OperatorFactory(io.prestosql.operator.OperatorFactory) PagesIndexBuilderOperatorFactory(io.prestosql.operator.index.PagesIndexBuilderOperator.PagesIndexBuilderOperatorFactory) PageBufferOperatorFactory(io.prestosql.operator.index.PageBufferOperator.PageBufferOperatorFactory) DriverFactory(io.prestosql.operator.DriverFactory) PageBufferOperatorFactory(io.prestosql.operator.index.PageBufferOperator.PageBufferOperatorFactory)

Example 2 with PageBufferOperatorFactory

use of io.prestosql.operator.index.PageBufferOperator.PageBufferOperatorFactory in project hetu-core by openlookeng.

the class TestHashJoinOperator method innerJoinWithSpill.

private void innerJoinWithSpill(boolean probeHashEnabled, List<WhenSpill> whenSpill, SingleStreamSpillerFactory buildSpillerFactory, PartitioningSpillerFactory joinSpillerFactory) throws Exception {
    TaskStateMachine taskStateMachine = new TaskStateMachine(new TaskId("query", 0, 0), executor);
    TaskContext taskContext = TestingTaskContext.createTaskContext(executor, scheduledExecutor, TEST_SESSION, taskStateMachine);
    DriverContext joinDriverContext = taskContext.addPipelineContext(2, true, true, false).addDriverContext();
    // force a yield for every match in LookupJoinOperator, set called to true after first
    AtomicBoolean called = new AtomicBoolean(false);
    InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction((leftPosition, leftPage, rightPosition, rightPage) -> {
        called.set(true);
        joinDriverContext.getYieldSignal().forceYieldForTesting();
        return true;
    });
    // build factory
    RowPagesBuilder buildPages = rowPagesBuilder(ImmutableList.of(VARCHAR, BIGINT)).addSequencePage(4, 20, 200).addSequencePage(4, 20, 200).addSequencePage(4, 30, 300).addSequencePage(4, 40, 400);
    BuildSideSetup buildSideSetup = setupBuildSide(true, taskContext, Ints.asList(0), buildPages, Optional.of(filterFunction), true, buildSpillerFactory);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT)).row("20", 123_000L).row("20", 123_000L).pageBreak().addSequencePage(20, 0, 123_000).addSequencePage(10, 30, 123_000);
    OperatorFactory joinOperatorFactory = innerJoinOperatorFactory(lookupSourceFactoryManager, probePages, joinSpillerFactory);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    List<Driver> buildDrivers = buildSideSetup.getBuildDrivers();
    int buildOperatorCount = buildDrivers.size();
    checkState(buildOperatorCount == whenSpill.size());
    LookupSourceFactory lookupSourceFactory = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide());
    boolean closed = false;
    try (Operator joinOperator = joinOperatorFactory.createOperator(joinDriverContext)) {
        // build lookup source
        ListenableFuture<LookupSourceProvider> lookupSourceProvider = lookupSourceFactory.createLookupSourceProvider();
        List<Boolean> revoked = new ArrayList<>(nCopies(buildOperatorCount, false));
        while (!lookupSourceProvider.isDone()) {
            for (int i = 0; i < buildOperatorCount; i++) {
                checkErrors(taskStateMachine);
                buildDrivers.get(i).process();
                HashBuilderOperator buildOperator = buildSideSetup.getBuildOperators().get(i);
                if (whenSpill.get(i) == WhenSpill.DURING_BUILD && buildOperator.getOperatorContext().getReservedRevocableBytes() > 0) {
                    checkState(!lookupSourceProvider.isDone(), "Too late, LookupSource already done");
                    revokeMemory(buildOperator);
                    revoked.set(i, true);
                }
            }
        }
        getFutureValue(lookupSourceProvider).close();
        assertEquals(revoked, whenSpill.stream().map(WhenSpill.DURING_BUILD::equals).collect(toImmutableList()), "Some operators not spilled before LookupSource built");
        for (int i = 0; i < buildOperatorCount; i++) {
            if (whenSpill.get(i) == WhenSpill.AFTER_BUILD) {
                revokeMemory(buildSideSetup.getBuildOperators().get(i));
            }
        }
        for (Driver buildDriver : buildDrivers) {
            runDriverInThread(executor, buildDriver);
        }
        joinOperatorFactory.noMoreOperators(joinDriverContext.getLifespan());
        joinOperatorFactory.noMoreOperators();
        closed = true;
        ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(17, new PlanNodeId("values"), probePages.build());
        PageBuffer pageBuffer = new PageBuffer(10);
        PageBufferOperatorFactory pageBufferOperatorFactory = new PageBufferOperatorFactory(18, new PlanNodeId("pageBuffer"), pageBuffer);
        Driver joinDriver = Driver.createDriver(joinDriverContext, valuesOperatorFactory.createOperator(joinDriverContext), joinOperator, pageBufferOperatorFactory.createOperator(joinDriverContext));
        while (!called.get()) {
            // process first row of first page of LookupJoinOperator
            processRow(joinDriver, taskStateMachine);
        }
        for (int i = 0; i < buildOperatorCount; i++) {
            if (whenSpill.get(i) == WhenSpill.DURING_USAGE) {
                triggerMemoryRevokingAndWait(buildSideSetup.getBuildOperators().get(i), taskStateMachine);
            }
        }
        // process remaining LookupJoinOperator pages
        while (!joinDriver.isFinished()) {
            checkErrors(taskStateMachine);
            processRow(joinDriver, taskStateMachine);
        }
        checkErrors(taskStateMachine);
        List<Page> actualPages = getPages(pageBuffer);
        MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probePages.getTypesWithoutHash(), buildPages.getTypesWithoutHash())).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("20", 123_000L, "20", 200L).row("30", 123_000L, "30", 300L).row("31", 123_001L, "31", 301L).row("32", 123_002L, "32", 302L).row("33", 123_003L, "33", 303L).build();
        assertEqualsIgnoreOrder(getProperColumns(joinOperator, concat(probePages.getTypes(), buildPages.getTypes()), probePages, actualPages).getMaterializedRows(), expected.getMaterializedRows());
    } finally {
        if (!closed) {
            joinOperatorFactory.noMoreOperators(joinDriverContext.getLifespan());
            joinOperatorFactory.noMoreOperators();
        }
    }
}
Also used : LocalExchangeSourceOperator(io.prestosql.operator.exchange.LocalExchangeSourceOperator) TaskId(io.prestosql.execution.TaskId) ArrayList(java.util.ArrayList) MarkerPage(io.prestosql.spi.snapshot.MarkerPage) Page(io.prestosql.spi.Page) TaskStateMachine(io.prestosql.execution.TaskStateMachine) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) PageBuffer(io.prestosql.operator.index.PageBuffer) TestingTaskContext(io.prestosql.testing.TestingTaskContext) RowPagesBuilder(io.prestosql.RowPagesBuilder) PageBufferOperatorFactory(io.prestosql.operator.index.PageBufferOperator.PageBufferOperatorFactory) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ValuesOperatorFactory(io.prestosql.operator.ValuesOperator.ValuesOperatorFactory) LocalExchangeSinkOperatorFactory(io.prestosql.operator.exchange.LocalExchangeSinkOperator.LocalExchangeSinkOperatorFactory) HashBuilderOperatorFactory(io.prestosql.operator.HashBuilderOperator.HashBuilderOperatorFactory) LocalExchangeSourceOperatorFactory(io.prestosql.operator.exchange.LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory) PageBufferOperatorFactory(io.prestosql.operator.index.PageBufferOperator.PageBufferOperatorFactory) ValuesOperatorFactory(io.prestosql.operator.ValuesOperator.ValuesOperatorFactory) MaterializedResult(io.prestosql.testing.MaterializedResult)

Aggregations

PageBufferOperatorFactory (io.prestosql.operator.index.PageBufferOperator.PageBufferOperatorFactory)2 ImmutableList (com.google.common.collect.ImmutableList)1 RowPagesBuilder (io.prestosql.RowPagesBuilder)1 TaskId (io.prestosql.execution.TaskId)1 TaskStateMachine (io.prestosql.execution.TaskStateMachine)1 DriverFactory (io.prestosql.operator.DriverFactory)1 HashBuilderOperatorFactory (io.prestosql.operator.HashBuilderOperator.HashBuilderOperatorFactory)1 OperatorFactory (io.prestosql.operator.OperatorFactory)1 ValuesOperatorFactory (io.prestosql.operator.ValuesOperator.ValuesOperatorFactory)1 LocalExchangeSinkOperatorFactory (io.prestosql.operator.exchange.LocalExchangeSinkOperator.LocalExchangeSinkOperatorFactory)1 LocalExchangeSourceOperator (io.prestosql.operator.exchange.LocalExchangeSourceOperator)1 LocalExchangeSourceOperatorFactory (io.prestosql.operator.exchange.LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory)1 PageBuffer (io.prestosql.operator.index.PageBuffer)1 PagesIndexBuilderOperatorFactory (io.prestosql.operator.index.PagesIndexBuilderOperator.PagesIndexBuilderOperatorFactory)1 Page (io.prestosql.spi.Page)1 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)1 MarkerPage (io.prestosql.spi.snapshot.MarkerPage)1 MaterializedResult (io.prestosql.testing.MaterializedResult)1 TestingTaskContext (io.prestosql.testing.TestingTaskContext)1 ArrayList (java.util.ArrayList)1