Examples with LocalExchangeSinkFactory com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactory used on opensource projects

Example 1 with LocalExchangeSinkFactory

use of com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactory in project presto by prestodb.

the class TestHashJoinOperator method buildHash.

private static LookupSourceFactory buildHash(boolean parallelBuild, TaskContext taskContext, List<Integer> hashChannels, RowPagesBuilder buildPages, Optional<InternalJoinFilterFunction> filterFunction) {
    Optional<JoinFilterFunctionFactory> filterFunctionFactory = filterFunction.map(function -> ((session, addresses, channels) -> new StandardJoinFilterFunction(function, addresses, channels)));
    int partitionCount = parallelBuild ? PARTITION_COUNT : 1;
    LocalExchange localExchange = new LocalExchange(FIXED_HASH_DISTRIBUTION, partitionCount, buildPages.getTypes(), hashChannels, buildPages.getHashChannel());
    LocalExchangeSinkFactory sinkFactory = localExchange.createSinkFactory();
    sinkFactory.noMoreSinkFactories();
    // collect input data into the partitioned exchange
    DriverContext collectDriverContext = taskContext.addPipelineContext(0, true, true).addDriverContext();
    ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(0, new PlanNodeId("values"), buildPages.getTypes(), buildPages.build());
    LocalExchangeSinkOperatorFactory sinkOperatorFactory = new LocalExchangeSinkOperatorFactory(1, new PlanNodeId("sink"), sinkFactory, Function.identity());
    Driver driver = new Driver(collectDriverContext, valuesOperatorFactory.createOperator(collectDriverContext), sinkOperatorFactory.createOperator(collectDriverContext));
    valuesOperatorFactory.close();
    sinkOperatorFactory.close();
    while (!driver.isFinished()) {
        driver.process();
    }
    // build hash tables
    LocalExchangeSourceOperatorFactory sourceOperatorFactory = new LocalExchangeSourceOperatorFactory(0, new PlanNodeId("source"), localExchange);
    HashBuilderOperatorFactory buildOperatorFactory = new HashBuilderOperatorFactory(1, new PlanNodeId("build"), buildPages.getTypes(), rangeList(buildPages.getTypes().size()), ImmutableMap.of(), hashChannels, buildPages.getHashChannel(), false, filterFunctionFactory, 100, partitionCount, new PagesIndex.TestingFactory());
    PipelineContext buildPipeline = taskContext.addPipelineContext(1, true, true);
    Driver[] buildDrivers = new Driver[partitionCount];
    for (int i = 0; i < partitionCount; i++) {
        DriverContext buildDriverContext = buildPipeline.addDriverContext();
        buildDrivers[i] = new Driver(buildDriverContext, sourceOperatorFactory.createOperator(buildDriverContext), buildOperatorFactory.createOperator(buildDriverContext));
    }
    while (!buildOperatorFactory.getLookupSourceFactory().createLookupSource().isDone()) {
        for (Driver buildDriver : buildDrivers) {
            buildDriver.process();
        }
    }
    return buildOperatorFactory.getLookupSourceFactory();
}

Example 2 with LocalExchangeSinkFactory

use of com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactory in project presto by prestodb.

the class TestLocalExchange method testBroadcast.

@Test
public void testBroadcast() {
    LocalExchange exchange = new LocalExchange(FIXED_BROADCAST_DISTRIBUTION, 2, TYPES, ImmutableList.of(), Optional.empty());
    assertEquals(exchange.getBufferCount(), 2);
    assertExchangeTotalBufferedBytes(exchange, 0);
    LocalExchangeSinkFactory sinkFactory = exchange.createSinkFactory();
    LocalExchangeSink sinkA = sinkFactory.createSink();
    assertSinkCanWrite(sinkA);
    LocalExchangeSink sinkB = sinkFactory.createSink();
    assertSinkCanWrite(sinkB);
    sinkFactory.close();
    sinkFactory.noMoreSinkFactories();
    LocalExchangeSource sourceA = exchange.getSource(0);
    assertSource(sourceA, 0);
    LocalExchangeSource sourceB = exchange.getSource(1);
    assertSource(sourceB, 0);
    sinkA.addPage(createPage(0));
    assertSource(sourceA, 1);
    assertSource(sourceB, 1);
    assertExchangeTotalBufferedBytes(exchange, 1);
    sinkA.addPage(createPage(0));
    assertSource(sourceA, 2);
    assertSource(sourceB, 2);
    assertExchangeTotalBufferedBytes(exchange, 2);
    assertRemovePage(sourceA, createPage(0));
    assertSource(sourceA, 1);
    assertSource(sourceB, 2);
    assertExchangeTotalBufferedBytes(exchange, 2);
    assertRemovePage(sourceA, createPage(0));
    assertSource(sourceA, 0);
    assertSource(sourceB, 2);
    assertExchangeTotalBufferedBytes(exchange, 2);
    sinkA.finish();
    assertSinkFinished(sinkA);
    assertExchangeTotalBufferedBytes(exchange, 2);
    sinkB.addPage(createPage(0));
    assertSource(sourceA, 1);
    assertSource(sourceB, 3);
    assertExchangeTotalBufferedBytes(exchange, 3);
    sinkB.finish();
    assertSinkFinished(sinkB);
    assertSource(sourceA, 1);
    assertSource(sourceB, 3);
    assertExchangeTotalBufferedBytes(exchange, 3);
    assertRemovePage(sourceA, createPage(0));
    assertSourceFinished(sourceA);
    assertSource(sourceB, 3);
    assertExchangeTotalBufferedBytes(exchange, 3);
    assertRemovePage(sourceB, createPage(0));
    assertRemovePage(sourceB, createPage(0));
    assertSourceFinished(sourceA);
    assertSource(sourceB, 1);
    assertExchangeTotalBufferedBytes(exchange, 1);
    assertRemovePage(sourceB, createPage(0));
    assertSourceFinished(sourceA);
    assertSourceFinished(sourceB);
    assertExchangeTotalBufferedBytes(exchange, 0);
}

Example 3 with LocalExchangeSinkFactory

use of com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactory in project presto by prestodb.

the class TestLocalExchange method writeUnblockWhenAllReadersFinishAndPagesConsumed.

@Test
public void writeUnblockWhenAllReadersFinishAndPagesConsumed() {
    LocalExchange exchange = new LocalExchange(FIXED_BROADCAST_DISTRIBUTION, 2, TYPES, ImmutableList.of(), Optional.empty(), new DataSize(1, BYTE));
    assertEquals(exchange.getBufferCount(), 2);
    assertExchangeTotalBufferedBytes(exchange, 0);
    LocalExchangeSinkFactory sinkFactory = exchange.createSinkFactory();
    LocalExchangeSink sinkA = sinkFactory.createSink();
    assertSinkCanWrite(sinkA);
    LocalExchangeSink sinkB = sinkFactory.createSink();
    assertSinkCanWrite(sinkB);
    sinkFactory.close();
    sinkFactory.noMoreSinkFactories();
    LocalExchangeSource sourceA = exchange.getSource(0);
    assertSource(sourceA, 0);
    LocalExchangeSource sourceB = exchange.getSource(1);
    assertSource(sourceB, 0);
    sinkA.addPage(createPage(0));
    ListenableFuture<?> sinkAFuture = assertSinkWriteBlocked(sinkA);
    ListenableFuture<?> sinkBFuture = assertSinkWriteBlocked(sinkB);
    assertSource(sourceA, 1);
    assertSource(sourceB, 1);
    assertExchangeTotalBufferedBytes(exchange, 1);
    sourceA.finish();
    assertSource(sourceA, 1);
    assertRemovePage(sourceA, createPage(0));
    assertSourceFinished(sourceA);
    assertExchangeTotalBufferedBytes(exchange, 1);
    assertSource(sourceB, 1);
    assertSinkWriteBlocked(sinkA);
    assertSinkWriteBlocked(sinkB);
    sourceB.finish();
    assertSource(sourceB, 1);
    assertRemovePage(sourceB, createPage(0));
    assertSourceFinished(sourceB);
    assertExchangeTotalBufferedBytes(exchange, 0);
    assertTrue(sinkAFuture.isDone());
    assertTrue(sinkBFuture.isDone());
    assertSinkFinished(sinkA);
    assertSinkFinished(sinkB);
}

Example 4 with LocalExchangeSinkFactory

use of com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactory in project presto by prestodb.

the class TestLocalExchange method testPartition.

@Test
public void testPartition() {
    LocalExchange exchange = new LocalExchange(FIXED_HASH_DISTRIBUTION, 2, TYPES, ImmutableList.of(0), Optional.empty());
    assertEquals(exchange.getBufferCount(), 2);
    assertExchangeTotalBufferedBytes(exchange, 0);
    LocalExchangeSinkFactory sinkFactory = exchange.createSinkFactory();
    LocalExchangeSink sink = sinkFactory.createSink();
    assertSinkCanWrite(sink);
    sinkFactory.close();
    sinkFactory.noMoreSinkFactories();
    LocalExchangeSource sourceA = exchange.getSource(0);
    assertSource(sourceA, 0);
    LocalExchangeSource sourceB = exchange.getSource(1);
    assertSource(sourceB, 0);
    sink.addPage(createPage(0));
    assertSource(sourceA, 1);
    assertSource(sourceB, 1);
    assertTrue(exchange.getBufferedBytes() >= retainedSizeOfPages(1));
    sink.addPage(createPage(0));
    assertSource(sourceA, 2);
    assertSource(sourceB, 2);
    assertTrue(exchange.getBufferedBytes() >= retainedSizeOfPages(2));
    assertPartitionedRemovePage(sourceA, 0, 2);
    assertSource(sourceA, 1);
    assertSource(sourceB, 2);
    assertPartitionedRemovePage(sourceA, 0, 2);
    assertSource(sourceA, 0);
    assertSource(sourceB, 2);
    sink.finish();
    assertSinkFinished(sink);
    assertSourceFinished(sourceA);
    assertSource(sourceB, 2);
    assertPartitionedRemovePage(sourceB, 1, 2);
    assertSourceFinished(sourceA);
    assertSource(sourceB, 1);
    assertPartitionedRemovePage(sourceB, 1, 2);
    assertSourceFinished(sourceA);
    assertSourceFinished(sourceB);
    assertExchangeTotalBufferedBytes(exchange, 0);
}

Example 5 with LocalExchangeSinkFactory

use of com.facebook.presto.operator.exchange.LocalExchange.LocalExchangeSinkFactory in project presto by prestodb.

the class TestLocalExchange method testRandom.

@Test
public void testRandom() {
    LocalExchange exchange = new LocalExchange(FIXED_ARBITRARY_DISTRIBUTION, 2, TYPES, ImmutableList.of(), Optional.empty());
    assertEquals(exchange.getBufferCount(), 2);
    assertExchangeTotalBufferedBytes(exchange, 0);
    LocalExchangeSinkFactory sinkFactory = exchange.createSinkFactory();
    LocalExchangeSink sink = sinkFactory.createSink();
    assertSinkCanWrite(sink);
    sinkFactory.close();
    sinkFactory.noMoreSinkFactories();
    LocalExchangeSource sourceA = exchange.getSource(0);
    assertSource(sourceA, 0);
    LocalExchangeSource sourceB = exchange.getSource(1);
    assertSource(sourceB, 0);
    for (int i = 0; i < 100; i++) {
        Page page = createPage(0);
        sink.addPage(page);
        assertExchangeTotalBufferedBytes(exchange, i + 1);
        LocalExchangeBufferInfo bufferInfoA = sourceA.getBufferInfo();
        LocalExchangeBufferInfo bufferInfoB = sourceB.getBufferInfo();
        assertEquals(bufferInfoA.getBufferedBytes() + bufferInfoB.getBufferedBytes(), retainedSizeOfPages(i + 1));
        assertEquals(bufferInfoA.getBufferedPages() + bufferInfoB.getBufferedPages(), i + 1);
    }
    // we should get ~50 pages per source, but we should get at least some pages in each buffer
    assertTrue(sourceA.getBufferInfo().getBufferedPages() > 0);
    assertTrue(sourceB.getBufferInfo().getBufferedPages() > 0);
    assertExchangeTotalBufferedBytes(exchange, 100);
}