Examples with LocalExchangeFactory com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeFactory used on opensource projects

Example 1 with LocalExchangeFactory

use of com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeFactory in project presto by prestodb.

the class TestLocalExchange method writeUnblockWhenAllReadersFinishAndPagesConsumed.

@Test(dataProvider = "executionStrategy")
public void writeUnblockWhenAllReadersFinishAndPagesConsumed(PipelineExecutionStrategy executionStrategy) {
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(partitioningProviderManager, session, FIXED_BROADCAST_DISTRIBUTION, 2, TYPES, ImmutableList.of(), Optional.empty(), executionStrategy, new DataSize(1, BYTE));
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertExchangeTotalBufferedBytes(exchange, 0);
        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sinkA = sinkFactory.createSink();
        assertSinkCanWrite(sinkA);
        LocalExchangeSink sinkB = sinkFactory.createSink();
        assertSinkCanWrite(sinkB);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();
        LocalExchangeSource sourceA = exchange.getSource(0);
        assertSource(sourceA, 0);
        LocalExchangeSource sourceB = exchange.getSource(1);
        assertSource(sourceB, 0);
        sinkA.addPage(createPage(0));
        ListenableFuture<?> sinkAFuture = assertSinkWriteBlocked(sinkA);
        ListenableFuture<?> sinkBFuture = assertSinkWriteBlocked(sinkB);
        assertSource(sourceA, 1);
        assertSource(sourceB, 1);
        assertExchangeTotalBufferedBytes(exchange, 1);
        sourceA.finish();
        assertSource(sourceA, 1);
        assertRemovePage(sourceA, createPage(0));
        assertSourceFinished(sourceA);
        assertExchangeTotalBufferedBytes(exchange, 1);
        assertSource(sourceB, 1);
        assertSinkWriteBlocked(sinkA);
        assertSinkWriteBlocked(sinkB);
        sourceB.finish();
        assertSource(sourceB, 1);
        assertRemovePage(sourceB, createPage(0));
        assertSourceFinished(sourceB);
        assertExchangeTotalBufferedBytes(exchange, 0);
        assertTrue(sinkAFuture.isDone());
        assertTrue(sinkBFuture.isDone());
        assertSinkFinished(sinkA);
        assertSinkFinished(sinkB);
    });
}

Example 2 with LocalExchangeFactory

use of com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeFactory in project presto by prestodb.

the class TestLocalExchange method testBroadcast.

@Test(dataProvider = "executionStrategy")
public void testBroadcast(PipelineExecutionStrategy executionStrategy) {
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(partitioningProviderManager, session, FIXED_BROADCAST_DISTRIBUTION, 2, TYPES, ImmutableList.of(), Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES);
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertExchangeTotalBufferedBytes(exchange, 0);
        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sinkA = sinkFactory.createSink();
        assertSinkCanWrite(sinkA);
        LocalExchangeSink sinkB = sinkFactory.createSink();
        assertSinkCanWrite(sinkB);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();
        LocalExchangeSource sourceA = exchange.getSource(0);
        assertSource(sourceA, 0);
        LocalExchangeSource sourceB = exchange.getSource(1);
        assertSource(sourceB, 0);
        sinkA.addPage(createPage(0));
        assertSource(sourceA, 1);
        assertSource(sourceB, 1);
        assertExchangeTotalBufferedBytes(exchange, 1);
        sinkA.addPage(createPage(0));
        assertSource(sourceA, 2);
        assertSource(sourceB, 2);
        assertExchangeTotalBufferedBytes(exchange, 2);
        assertRemovePage(sourceA, createPage(0));
        assertSource(sourceA, 1);
        assertSource(sourceB, 2);
        assertExchangeTotalBufferedBytes(exchange, 2);
        assertRemovePage(sourceA, createPage(0));
        assertSource(sourceA, 0);
        assertSource(sourceB, 2);
        assertExchangeTotalBufferedBytes(exchange, 2);
        sinkA.finish();
        assertSinkFinished(sinkA);
        assertExchangeTotalBufferedBytes(exchange, 2);
        sinkB.addPage(createPage(0));
        assertSource(sourceA, 1);
        assertSource(sourceB, 3);
        assertExchangeTotalBufferedBytes(exchange, 3);
        sinkB.finish();
        assertSinkFinished(sinkB);
        assertSource(sourceA, 1);
        assertSource(sourceB, 3);
        assertExchangeTotalBufferedBytes(exchange, 3);
        assertRemovePage(sourceA, createPage(0));
        assertSourceFinished(sourceA);
        assertSource(sourceB, 3);
        assertExchangeTotalBufferedBytes(exchange, 3);
        assertRemovePage(sourceB, createPage(0));
        assertRemovePage(sourceB, createPage(0));
        assertSourceFinished(sourceA);
        assertSource(sourceB, 1);
        assertExchangeTotalBufferedBytes(exchange, 1);
        assertRemovePage(sourceB, createPage(0));
        assertSourceFinished(sourceA);
        assertSourceFinished(sourceB);
        assertExchangeTotalBufferedBytes(exchange, 0);
    });
}

Example 3 with LocalExchangeFactory

use of com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeFactory in project presto by prestodb.

the class TestHashJoinOperator method setupBuildSide.

private BuildSideSetup setupBuildSide(boolean parallelBuild, TaskContext taskContext, List<Integer> hashChannels, RowPagesBuilder buildPages, Optional<InternalJoinFilterFunction> filterFunction, boolean spillEnabled, SingleStreamSpillerFactory singleStreamSpillerFactory, boolean enforceBroadcastMemoryLimit) {
    Optional<JoinFilterFunctionFactory> filterFunctionFactory = filterFunction.map(function -> (session, addresses, pages) -> new StandardJoinFilterFunction(function, addresses, pages));
    int partitionCount = parallelBuild ? PARTITION_COUNT : 1;
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(partitioningProviderManager, session, FIXED_HASH_DISTRIBUTION, partitionCount, buildPages.getTypes(), hashChannels, buildPages.getHashChannel(), UNGROUPED_EXECUTION, new DataSize(32, DataSize.Unit.MEGABYTE));
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    // collect input data into the partitioned exchange
    DriverContext collectDriverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(0, new PlanNodeId("values"), buildPages.build());
    LocalExchangeSinkOperatorFactory sinkOperatorFactory = new LocalExchangeSinkOperatorFactory(localExchangeFactory, 1, new PlanNodeId("sink"), localExchangeSinkFactoryId, Function.identity());
    Driver sourceDriver = Driver.createDriver(collectDriverContext, valuesOperatorFactory.createOperator(collectDriverContext), sinkOperatorFactory.createOperator(collectDriverContext));
    valuesOperatorFactory.noMoreOperators();
    sinkOperatorFactory.noMoreOperators();
    while (!sourceDriver.isFinished()) {
        sourceDriver.process();
    }
    // build side operator factories
    LocalExchangeSourceOperatorFactory sourceOperatorFactory = new LocalExchangeSourceOperatorFactory(0, new PlanNodeId("source"), localExchangeFactory);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(buildPages.getTypes(), rangeList(buildPages.getTypes().size()).stream().map(buildPages.getTypes()::get).collect(toImmutableList()), hashChannels.stream().map(buildPages.getTypes()::get).collect(toImmutableList()), partitionCount, requireNonNull(ImmutableMap.of(), "layout is null"), false));
    HashBuilderOperatorFactory buildOperatorFactory = new HashBuilderOperatorFactory(1, new PlanNodeId("build"), lookupSourceFactoryManager, rangeList(buildPages.getTypes().size()), hashChannels, buildPages.getHashChannel().map(OptionalInt::of).orElse(OptionalInt.empty()), filterFunctionFactory, Optional.empty(), ImmutableList.of(), 100, new PagesIndex.TestingFactory(false), spillEnabled, singleStreamSpillerFactory, enforceBroadcastMemoryLimit);
    return new BuildSideSetup(lookupSourceFactoryManager, buildOperatorFactory, sourceOperatorFactory, partitionCount);
}

Example 4 with LocalExchangeFactory

use of com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeFactory in project presto by prestodb.

the class TestLocalExchange method testPassthrough.

@Test(dataProvider = "executionStrategy")
public void testPassthrough(PipelineExecutionStrategy executionStrategy) {
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(partitioningProviderManager, session, FIXED_PASSTHROUGH_DISTRIBUTION, 2, TYPES, ImmutableList.of(), Optional.empty(), executionStrategy, new DataSize(retainedSizeOfPages(1), BYTE));
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertExchangeTotalBufferedBytes(exchange, 0);
        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sinkA = sinkFactory.createSink();
        LocalExchangeSink sinkB = sinkFactory.createSink();
        assertSinkCanWrite(sinkA);
        assertSinkCanWrite(sinkB);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();
        LocalExchangeSource sourceA = exchange.getSource(0);
        assertSource(sourceA, 0);
        LocalExchangeSource sourceB = exchange.getSource(1);
        assertSource(sourceB, 0);
        sinkA.addPage(createPage(0));
        assertSource(sourceA, 1);
        assertSource(sourceB, 0);
        assertSinkWriteBlocked(sinkA);
        assertSinkCanWrite(sinkB);
        sinkB.addPage(createPage(1));
        assertSource(sourceA, 1);
        assertSource(sourceB, 1);
        assertSinkWriteBlocked(sinkA);
        assertExchangeTotalBufferedBytes(exchange, 2);
        assertRemovePage(sourceA, createPage(0));
        assertSource(sourceA, 0);
        assertSinkCanWrite(sinkA);
        assertSinkWriteBlocked(sinkB);
        assertExchangeTotalBufferedBytes(exchange, 1);
        sinkA.finish();
        assertSinkFinished(sinkA);
        assertSource(sourceB, 1);
        sourceA.finish();
        sourceB.finish();
        assertRemovePage(sourceB, createPage(1));
        assertSourceFinished(sourceA);
        assertSourceFinished(sourceB);
        assertSinkFinished(sinkB);
        assertExchangeTotalBufferedBytes(exchange, 0);
    });
}

Example 5 with LocalExchangeFactory

use of com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeFactory in project presto by prestodb.

the class TestLocalExchange method testMismatchedExecutionStrategy.

@Test
public void testMismatchedExecutionStrategy() {
    // If sink/source didn't create a matching set of exchanges, operators will block forever,
    // waiting for the other half that will never show up.
    // The most common reason of mismatch is when one of sink/source created the wrong kind of local exchange.
    // In such case, we want to fail loudly.
    LocalExchangeFactory ungroupedLocalExchangeFactory = new LocalExchangeFactory(partitioningProviderManager, session, FIXED_HASH_DISTRIBUTION, 2, TYPES, ImmutableList.of(0), Optional.empty(), UNGROUPED_EXECUTION, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES);
    try {
        ungroupedLocalExchangeFactory.getLocalExchange(Lifespan.driverGroup(3));
        fail("expected failure");
    } catch (IllegalArgumentException e) {
        assertContains(e.getMessage(), "Driver-group exchange cannot be created.");
    }
    LocalExchangeFactory groupedLocalExchangeFactory = new LocalExchangeFactory(partitioningProviderManager, session, FIXED_HASH_DISTRIBUTION, 2, TYPES, ImmutableList.of(0), Optional.empty(), GROUPED_EXECUTION, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES);
    try {
        groupedLocalExchangeFactory.getLocalExchange(Lifespan.taskWide());
        fail("expected failure");
    } catch (IllegalArgumentException e) {
        assertContains(e.getMessage(), "Task-wide exchange cannot be created.");
    }
}