Search in sources :

Example 1 with LocalExchangeSinkOperatorFactory

use of com.facebook.presto.operator.exchange.LocalExchangeSinkOperator.LocalExchangeSinkOperatorFactory in project presto by prestodb.

the class TestHashJoinOperator method buildHash.

private static LookupSourceFactory buildHash(boolean parallelBuild, TaskContext taskContext, List<Integer> hashChannels, RowPagesBuilder buildPages, Optional<InternalJoinFilterFunction> filterFunction) {
    Optional<JoinFilterFunctionFactory> filterFunctionFactory = filterFunction.map(function -> ((session, addresses, channels) -> new StandardJoinFilterFunction(function, addresses, channels)));
    int partitionCount = parallelBuild ? PARTITION_COUNT : 1;
    LocalExchange localExchange = new LocalExchange(FIXED_HASH_DISTRIBUTION, partitionCount, buildPages.getTypes(), hashChannels, buildPages.getHashChannel());
    LocalExchangeSinkFactory sinkFactory = localExchange.createSinkFactory();
    sinkFactory.noMoreSinkFactories();
    // collect input data into the partitioned exchange
    DriverContext collectDriverContext = taskContext.addPipelineContext(0, true, true).addDriverContext();
    ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(0, new PlanNodeId("values"), buildPages.getTypes(), buildPages.build());
    LocalExchangeSinkOperatorFactory sinkOperatorFactory = new LocalExchangeSinkOperatorFactory(1, new PlanNodeId("sink"), sinkFactory, Function.identity());
    Driver driver = new Driver(collectDriverContext, valuesOperatorFactory.createOperator(collectDriverContext), sinkOperatorFactory.createOperator(collectDriverContext));
    valuesOperatorFactory.close();
    sinkOperatorFactory.close();
    while (!driver.isFinished()) {
        driver.process();
    }
    // build hash tables
    LocalExchangeSourceOperatorFactory sourceOperatorFactory = new LocalExchangeSourceOperatorFactory(0, new PlanNodeId("source"), localExchange);
    HashBuilderOperatorFactory buildOperatorFactory = new HashBuilderOperatorFactory(1, new PlanNodeId("build"), buildPages.getTypes(), rangeList(buildPages.getTypes().size()), ImmutableMap.of(), hashChannels, buildPages.getHashChannel(), false, filterFunctionFactory, 100, partitionCount, new PagesIndex.TestingFactory());
    PipelineContext buildPipeline = taskContext.addPipelineContext(1, true, true);
    Driver[] buildDrivers = new Driver[partitionCount];
    for (int i = 0; i < partitionCount; i++) {
        DriverContext buildDriverContext = buildPipeline.addDriverContext();
        buildDrivers[i] = new Driver(buildDriverContext, sourceOperatorFactory.createOperator(buildDriverContext), buildOperatorFactory.createOperator(buildDriverContext));
    }
    while (!buildOperatorFactory.getLookupSourceFactory().createLookupSource().isDone()) {
        for (Driver buildDriver : buildDrivers) {
            buildDriver.process();
        }
    }
    return buildOperatorFactory.getLookupSourceFactory();
}
Also used : Page(com.facebook.presto.spi.Page) IntStream(java.util.stream.IntStream) Block(com.facebook.presto.spi.block.Block) DataProvider(org.testng.annotations.DataProvider) JoinProbeCompiler(com.facebook.presto.sql.gen.JoinProbeCompiler) Test(org.testng.annotations.Test) Function(java.util.function.Function) ValuesOperatorFactory(com.facebook.presto.operator.ValuesOperator.ValuesOperatorFactory) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) TEST_SESSION(com.facebook.presto.SessionTestUtils.TEST_SESSION) RowPagesBuilder(com.facebook.presto.RowPagesBuilder) ImmutableList(com.google.common.collect.ImmutableList) Type(com.facebook.presto.spi.type.Type) Iterables.concat(com.google.common.collect.Iterables.concat) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) ImmutableCollectors.toImmutableList(com.facebook.presto.util.ImmutableCollectors.toImmutableList) LocalExchange(com.facebook.presto.operator.exchange.LocalExchange) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) TestingTaskContext(com.facebook.presto.testing.TestingTaskContext) ExecutorService(java.util.concurrent.ExecutorService) AfterClass(org.testng.annotations.AfterClass) HashBuilderOperatorFactory(com.facebook.presto.operator.HashBuilderOperator.HashBuilderOperatorFactory) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) LocalExchangeSinkOperatorFactory(com.facebook.presto.operator.exchange.LocalExchangeSinkOperator.LocalExchangeSinkOperatorFactory) BeforeClass(org.testng.annotations.BeforeClass) OperatorAssertion.assertOperatorEquals(com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals) VARCHAR(com.facebook.presto.spi.type.VarcharType.VARCHAR) Ints(com.google.common.primitives.Ints) MaterializedResult(com.facebook.presto.testing.MaterializedResult) DataSize(io.airlift.units.DataSize) List(java.util.List) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) Optional(java.util.Optional) LocalExchangeSinkFactory(com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactory) LocalExchangeSourceOperatorFactory(com.facebook.presto.operator.exchange.LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory) FIXED_HASH_DISTRIBUTION(com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION) BYTE(io.airlift.units.DataSize.Unit.BYTE) RowPagesBuilder.rowPagesBuilder(com.facebook.presto.RowPagesBuilder.rowPagesBuilder) ExceededMemoryLimitException(com.facebook.presto.ExceededMemoryLimitException) JoinFilterFunctionFactory(com.facebook.presto.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory) LocalExchangeSinkOperatorFactory(com.facebook.presto.operator.exchange.LocalExchangeSinkOperator.LocalExchangeSinkOperatorFactory) HashBuilderOperatorFactory(com.facebook.presto.operator.HashBuilderOperator.HashBuilderOperatorFactory) LocalExchange(com.facebook.presto.operator.exchange.LocalExchange) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) LocalExchangeSinkFactory(com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactory) LocalExchangeSourceOperatorFactory(com.facebook.presto.operator.exchange.LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory) JoinFilterFunctionFactory(com.facebook.presto.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory) ValuesOperatorFactory(com.facebook.presto.operator.ValuesOperator.ValuesOperatorFactory)

Example 2 with LocalExchangeSinkOperatorFactory

use of com.facebook.presto.operator.exchange.LocalExchangeSinkOperator.LocalExchangeSinkOperatorFactory in project presto by prestodb.

the class TestHashJoinOperator method setupBuildSide.

private BuildSideSetup setupBuildSide(boolean parallelBuild, TaskContext taskContext, List<Integer> hashChannels, RowPagesBuilder buildPages, Optional<InternalJoinFilterFunction> filterFunction, boolean spillEnabled, SingleStreamSpillerFactory singleStreamSpillerFactory, boolean enforceBroadcastMemoryLimit) {
    Optional<JoinFilterFunctionFactory> filterFunctionFactory = filterFunction.map(function -> (session, addresses, pages) -> new StandardJoinFilterFunction(function, addresses, pages));
    int partitionCount = parallelBuild ? PARTITION_COUNT : 1;
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(partitioningProviderManager, session, FIXED_HASH_DISTRIBUTION, partitionCount, buildPages.getTypes(), hashChannels, buildPages.getHashChannel(), UNGROUPED_EXECUTION, new DataSize(32, DataSize.Unit.MEGABYTE));
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    // collect input data into the partitioned exchange
    DriverContext collectDriverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(0, new PlanNodeId("values"), buildPages.build());
    LocalExchangeSinkOperatorFactory sinkOperatorFactory = new LocalExchangeSinkOperatorFactory(localExchangeFactory, 1, new PlanNodeId("sink"), localExchangeSinkFactoryId, Function.identity());
    Driver sourceDriver = Driver.createDriver(collectDriverContext, valuesOperatorFactory.createOperator(collectDriverContext), sinkOperatorFactory.createOperator(collectDriverContext));
    valuesOperatorFactory.noMoreOperators();
    sinkOperatorFactory.noMoreOperators();
    while (!sourceDriver.isFinished()) {
        sourceDriver.process();
    }
    // build side operator factories
    LocalExchangeSourceOperatorFactory sourceOperatorFactory = new LocalExchangeSourceOperatorFactory(0, new PlanNodeId("source"), localExchangeFactory);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(buildPages.getTypes(), rangeList(buildPages.getTypes().size()).stream().map(buildPages.getTypes()::get).collect(toImmutableList()), hashChannels.stream().map(buildPages.getTypes()::get).collect(toImmutableList()), partitionCount, requireNonNull(ImmutableMap.of(), "layout is null"), false));
    HashBuilderOperatorFactory buildOperatorFactory = new HashBuilderOperatorFactory(1, new PlanNodeId("build"), lookupSourceFactoryManager, rangeList(buildPages.getTypes().size()), hashChannels, buildPages.getHashChannel().map(OptionalInt::of).orElse(OptionalInt.empty()), filterFunctionFactory, Optional.empty(), ImmutableList.of(), 100, new PagesIndex.TestingFactory(false), spillEnabled, singleStreamSpillerFactory, enforceBroadcastMemoryLimit);
    return new BuildSideSetup(lookupSourceFactoryManager, buildOperatorFactory, sourceOperatorFactory, partitionCount);
}
Also used : LocalExchangeSinkOperatorFactory(com.facebook.presto.operator.exchange.LocalExchangeSinkOperator.LocalExchangeSinkOperatorFactory) HashBuilderOperatorFactory(com.facebook.presto.operator.HashBuilderOperator.HashBuilderOperatorFactory) OptionalInt(java.util.OptionalInt) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) LocalExchangeSourceOperatorFactory(com.facebook.presto.operator.exchange.LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory) JoinFilterFunctionFactory(com.facebook.presto.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory) DataSize(io.airlift.units.DataSize) ValuesOperatorFactory(com.facebook.presto.operator.ValuesOperator.ValuesOperatorFactory) LocalExchangeFactory(com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeFactory) LocalExchangeSinkFactoryId(com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId)

Aggregations

HashBuilderOperatorFactory (com.facebook.presto.operator.HashBuilderOperator.HashBuilderOperatorFactory)2 ValuesOperatorFactory (com.facebook.presto.operator.ValuesOperator.ValuesOperatorFactory)2 LocalExchangeSinkOperatorFactory (com.facebook.presto.operator.exchange.LocalExchangeSinkOperator.LocalExchangeSinkOperatorFactory)2 LocalExchangeSourceOperatorFactory (com.facebook.presto.operator.exchange.LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory)2 JoinFilterFunctionFactory (com.facebook.presto.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory)2 DataSize (io.airlift.units.DataSize)2 ExceededMemoryLimitException (com.facebook.presto.ExceededMemoryLimitException)1 RowPagesBuilder (com.facebook.presto.RowPagesBuilder)1 RowPagesBuilder.rowPagesBuilder (com.facebook.presto.RowPagesBuilder.rowPagesBuilder)1 TEST_SESSION (com.facebook.presto.SessionTestUtils.TEST_SESSION)1 OperatorAssertion.assertOperatorEquals (com.facebook.presto.operator.OperatorAssertion.assertOperatorEquals)1 LocalExchange (com.facebook.presto.operator.exchange.LocalExchange)1 LocalExchangeFactory (com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeFactory)1 LocalExchangeSinkFactory (com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactory)1 LocalExchangeSinkFactoryId (com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId)1 Page (com.facebook.presto.spi.Page)1 Block (com.facebook.presto.spi.block.Block)1 PlanNodeId (com.facebook.presto.spi.plan.PlanNodeId)1 BIGINT (com.facebook.presto.spi.type.BigintType.BIGINT)1 Type (com.facebook.presto.spi.type.Type)1