Search in sources :

Example 1 with LocalExchangeFactory

use of io.trino.operator.exchange.LocalExchange.LocalExchangeFactory in project trino by trinodb.

the class TestLocalExchange method testMismatchedExecutionStrategy.

@Test
public void testMismatchedExecutionStrategy() {
    // If sink/source didn't create a matching set of exchanges, operators will block forever,
    // waiting for the other half that will never show up.
    // The most common reason of mismatch is when one of sink/source created the wrong kind of local exchange.
    // In such case, we want to fail loudly.
    LocalExchangeFactory ungroupedLocalExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_HASH_DISTRIBUTION, 2, TYPES, ImmutableList.of(0), Optional.empty(), UNGROUPED_EXECUTION, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
    assertThatThrownBy(() -> ungroupedLocalExchangeFactory.getLocalExchange(Lifespan.driverGroup(3))).isInstanceOf(IllegalArgumentException.class).hasMessage("LocalExchangeFactory is declared as UNGROUPED_EXECUTION. Driver-group exchange cannot be created.");
    LocalExchangeFactory groupedLocalExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_HASH_DISTRIBUTION, 2, TYPES, ImmutableList.of(0), Optional.empty(), GROUPED_EXECUTION, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
    assertThatThrownBy(() -> groupedLocalExchangeFactory.getLocalExchange(Lifespan.taskWide())).isInstanceOf(IllegalArgumentException.class).hasMessage("LocalExchangeFactory is declared as GROUPED_EXECUTION. Task-wide exchange cannot be created.");
}
Also used : LocalExchangeFactory(io.trino.operator.exchange.LocalExchange.LocalExchangeFactory) Test(org.testng.annotations.Test)

Example 2 with LocalExchangeFactory

use of io.trino.operator.exchange.LocalExchange.LocalExchangeFactory in project trino by trinodb.

the class TestLocalExchange method testPartition.

@Test(dataProvider = "executionStrategy")
public void testPartition(PipelineExecutionStrategy executionStrategy) {
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_HASH_DISTRIBUTION, 2, TYPES, ImmutableList.of(0), Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertExchangeTotalBufferedBytes(exchange, 0);
        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sink = sinkFactory.createSink();
        assertSinkCanWrite(sink);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();
        LocalExchangeSource sourceA = exchange.getSource(0);
        assertSource(sourceA, 0);
        LocalExchangeSource sourceB = exchange.getSource(1);
        assertSource(sourceB, 0);
        sink.addPage(createPage(0));
        assertSource(sourceA, 1);
        assertSource(sourceB, 1);
        assertTrue(exchange.getBufferedBytes() >= retainedSizeOfPages(1));
        sink.addPage(createPage(0));
        assertSource(sourceA, 2);
        assertSource(sourceB, 2);
        assertTrue(exchange.getBufferedBytes() >= retainedSizeOfPages(2));
        assertPartitionedRemovePage(sourceA, 0, 2);
        assertSource(sourceA, 1);
        assertSource(sourceB, 2);
        assertPartitionedRemovePage(sourceA, 0, 2);
        assertSource(sourceA, 0);
        assertSource(sourceB, 2);
        sink.finish();
        assertSinkFinished(sink);
        assertSourceFinished(sourceA);
        assertSource(sourceB, 2);
        assertPartitionedRemovePage(sourceB, 1, 2);
        assertSourceFinished(sourceA);
        assertSource(sourceB, 1);
        assertPartitionedRemovePage(sourceB, 1, 2);
        assertSourceFinished(sourceA);
        assertSourceFinished(sourceB);
        assertExchangeTotalBufferedBytes(exchange, 0);
    });
}
Also used : LocalExchangeSinkFactory(io.trino.operator.exchange.LocalExchange.LocalExchangeSinkFactory) LocalExchangeFactory(io.trino.operator.exchange.LocalExchange.LocalExchangeFactory) LocalExchangeSinkFactoryId(io.trino.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId) Test(org.testng.annotations.Test)

Example 3 with LocalExchangeFactory

use of io.trino.operator.exchange.LocalExchange.LocalExchangeFactory in project trino by trinodb.

the class TestLocalExchange method writeUnblockWhenAllReadersFinish.

@Test(dataProvider = "executionStrategy")
public void writeUnblockWhenAllReadersFinish(PipelineExecutionStrategy executionStrategy) {
    ImmutableList<Type> types = ImmutableList.of(BIGINT);
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_BROADCAST_DISTRIBUTION, 2, types, ImmutableList.of(), Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertExchangeTotalBufferedBytes(exchange, 0);
        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sinkA = sinkFactory.createSink();
        assertSinkCanWrite(sinkA);
        LocalExchangeSink sinkB = sinkFactory.createSink();
        assertSinkCanWrite(sinkB);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();
        LocalExchangeSource sourceA = exchange.getSource(0);
        assertSource(sourceA, 0);
        LocalExchangeSource sourceB = exchange.getSource(1);
        assertSource(sourceB, 0);
        sourceA.finish();
        assertSourceFinished(sourceA);
        assertSinkCanWrite(sinkA);
        assertSinkCanWrite(sinkB);
        sourceB.finish();
        assertSourceFinished(sourceB);
        assertSinkFinished(sinkA);
        assertSinkFinished(sinkB);
    });
}
Also used : LocalExchangeSinkFactory(io.trino.operator.exchange.LocalExchange.LocalExchangeSinkFactory) Type(io.trino.spi.type.Type) LocalExchangeFactory(io.trino.operator.exchange.LocalExchange.LocalExchangeFactory) LocalExchangeSinkFactoryId(io.trino.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId) Test(org.testng.annotations.Test)

Example 4 with LocalExchangeFactory

use of io.trino.operator.exchange.LocalExchange.LocalExchangeFactory in project trino by trinodb.

the class TestLocalExchange method testRandom.

@Test(dataProvider = "executionStrategy")
public void testRandom(PipelineExecutionStrategy executionStrategy) {
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_ARBITRARY_DISTRIBUTION, 2, TYPES, ImmutableList.of(), Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertExchangeTotalBufferedBytes(exchange, 0);
        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sink = sinkFactory.createSink();
        assertSinkCanWrite(sink);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();
        LocalExchangeSource sourceA = exchange.getSource(0);
        assertSource(sourceA, 0);
        LocalExchangeSource sourceB = exchange.getSource(1);
        assertSource(sourceB, 0);
        for (int i = 0; i < 100; i++) {
            Page page = createPage(0);
            sink.addPage(page);
            assertExchangeTotalBufferedBytes(exchange, i + 1);
            LocalExchangeBufferInfo bufferInfoA = sourceA.getBufferInfo();
            LocalExchangeBufferInfo bufferInfoB = sourceB.getBufferInfo();
            assertEquals(bufferInfoA.getBufferedBytes() + bufferInfoB.getBufferedBytes(), retainedSizeOfPages(i + 1));
            assertEquals(bufferInfoA.getBufferedPages() + bufferInfoB.getBufferedPages(), i + 1);
        }
        // we should get ~50 pages per source, but we should get at least some pages in each buffer
        assertTrue(sourceA.getBufferInfo().getBufferedPages() > 0);
        assertTrue(sourceB.getBufferInfo().getBufferedPages() > 0);
        assertExchangeTotalBufferedBytes(exchange, 100);
    });
}
Also used : LocalExchangeSinkFactory(io.trino.operator.exchange.LocalExchange.LocalExchangeSinkFactory) Page(io.trino.spi.Page) LocalExchangeFactory(io.trino.operator.exchange.LocalExchange.LocalExchangeFactory) LocalExchangeSinkFactoryId(io.trino.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId) Test(org.testng.annotations.Test)

Example 5 with LocalExchangeFactory

use of io.trino.operator.exchange.LocalExchange.LocalExchangeFactory in project trino by trinodb.

the class TestLocalExchange method testPassthrough.

@Test(dataProvider = "executionStrategy")
public void testPassthrough(PipelineExecutionStrategy executionStrategy) {
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_PASSTHROUGH_DISTRIBUTION, 2, TYPES, ImmutableList.of(), Optional.empty(), executionStrategy, DataSize.ofBytes(retainedSizeOfPages(1)), TYPE_OPERATOR_FACTORY);
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertExchangeTotalBufferedBytes(exchange, 0);
        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sinkA = sinkFactory.createSink();
        LocalExchangeSink sinkB = sinkFactory.createSink();
        assertSinkCanWrite(sinkA);
        assertSinkCanWrite(sinkB);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();
        LocalExchangeSource sourceA = exchange.getSource(0);
        assertSource(sourceA, 0);
        LocalExchangeSource sourceB = exchange.getSource(1);
        assertSource(sourceB, 0);
        sinkA.addPage(createPage(0));
        assertSource(sourceA, 1);
        assertSource(sourceB, 0);
        assertSinkWriteBlocked(sinkA);
        assertSinkCanWrite(sinkB);
        sinkB.addPage(createPage(1));
        assertSource(sourceA, 1);
        assertSource(sourceB, 1);
        assertSinkWriteBlocked(sinkA);
        assertExchangeTotalBufferedBytes(exchange, 2);
        assertRemovePage(sourceA, createPage(0));
        assertSource(sourceA, 0);
        assertSinkCanWrite(sinkA);
        assertSinkWriteBlocked(sinkB);
        assertExchangeTotalBufferedBytes(exchange, 1);
        sinkA.finish();
        assertSinkFinished(sinkA);
        assertSource(sourceB, 1);
        sourceA.finish();
        sourceB.finish();
        assertRemovePage(sourceB, createPage(1));
        assertSourceFinished(sourceA);
        assertSourceFinished(sourceB);
        assertSinkFinished(sinkB);
        assertExchangeTotalBufferedBytes(exchange, 0);
    });
}
Also used : LocalExchangeSinkFactory(io.trino.operator.exchange.LocalExchange.LocalExchangeSinkFactory) LocalExchangeFactory(io.trino.operator.exchange.LocalExchange.LocalExchangeFactory) LocalExchangeSinkFactoryId(io.trino.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId) Test(org.testng.annotations.Test)

Aggregations

LocalExchangeFactory (io.trino.operator.exchange.LocalExchange.LocalExchangeFactory)9 Test (org.testng.annotations.Test)9 LocalExchangeSinkFactory (io.trino.operator.exchange.LocalExchange.LocalExchangeSinkFactory)8 LocalExchangeSinkFactoryId (io.trino.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId)8 Page (io.trino.spi.Page)2 Type (io.trino.spi.type.Type)2 ImmutableList (com.google.common.collect.ImmutableList)1 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)1 DataSize (io.airlift.units.DataSize)1 SequencePageBuilder (io.trino.SequencePageBuilder)1 Session (io.trino.Session)1 CatalogName (io.trino.connector.CatalogName)1 Lifespan (io.trino.execution.Lifespan)1 NodeTaskMap (io.trino.execution.NodeTaskMap)1 NodeScheduler (io.trino.execution.scheduler.NodeScheduler)1 NodeSchedulerConfig (io.trino.execution.scheduler.NodeSchedulerConfig)1 UniformNodeSelectorFactory (io.trino.execution.scheduler.UniformNodeSelectorFactory)1 InMemoryNodeManager (io.trino.metadata.InMemoryNodeManager)1 InterpretedHashGenerator (io.trino.operator.InterpretedHashGenerator)1 PageAssertions (io.trino.operator.PageAssertions)1