use of io.trino.operator.exchange.LocalExchange.LocalExchangeFactory in project trino by trinodb.
the class TestLocalExchange method testMismatchedExecutionStrategy.
@Test
public void testMismatchedExecutionStrategy() {
// If sink/source didn't create a matching set of exchanges, operators will block forever,
// waiting for the other half that will never show up.
// The most common reason of mismatch is when one of sink/source created the wrong kind of local exchange.
// In such case, we want to fail loudly.
LocalExchangeFactory ungroupedLocalExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_HASH_DISTRIBUTION, 2, TYPES, ImmutableList.of(0), Optional.empty(), UNGROUPED_EXECUTION, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
assertThatThrownBy(() -> ungroupedLocalExchangeFactory.getLocalExchange(Lifespan.driverGroup(3))).isInstanceOf(IllegalArgumentException.class).hasMessage("LocalExchangeFactory is declared as UNGROUPED_EXECUTION. Driver-group exchange cannot be created.");
LocalExchangeFactory groupedLocalExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_HASH_DISTRIBUTION, 2, TYPES, ImmutableList.of(0), Optional.empty(), GROUPED_EXECUTION, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
assertThatThrownBy(() -> groupedLocalExchangeFactory.getLocalExchange(Lifespan.taskWide())).isInstanceOf(IllegalArgumentException.class).hasMessage("LocalExchangeFactory is declared as GROUPED_EXECUTION. Task-wide exchange cannot be created.");
}
use of io.trino.operator.exchange.LocalExchange.LocalExchangeFactory in project trino by trinodb.
the class TestLocalExchange method testPartition.
@Test(dataProvider = "executionStrategy")
public void testPartition(PipelineExecutionStrategy executionStrategy) {
LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_HASH_DISTRIBUTION, 2, TYPES, ImmutableList.of(0), Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
localExchangeFactory.noMoreSinkFactories();
run(localExchangeFactory, executionStrategy, exchange -> {
assertEquals(exchange.getBufferCount(), 2);
assertExchangeTotalBufferedBytes(exchange, 0);
LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
LocalExchangeSink sink = sinkFactory.createSink();
assertSinkCanWrite(sink);
sinkFactory.close();
sinkFactory.noMoreSinkFactories();
LocalExchangeSource sourceA = exchange.getSource(0);
assertSource(sourceA, 0);
LocalExchangeSource sourceB = exchange.getSource(1);
assertSource(sourceB, 0);
sink.addPage(createPage(0));
assertSource(sourceA, 1);
assertSource(sourceB, 1);
assertTrue(exchange.getBufferedBytes() >= retainedSizeOfPages(1));
sink.addPage(createPage(0));
assertSource(sourceA, 2);
assertSource(sourceB, 2);
assertTrue(exchange.getBufferedBytes() >= retainedSizeOfPages(2));
assertPartitionedRemovePage(sourceA, 0, 2);
assertSource(sourceA, 1);
assertSource(sourceB, 2);
assertPartitionedRemovePage(sourceA, 0, 2);
assertSource(sourceA, 0);
assertSource(sourceB, 2);
sink.finish();
assertSinkFinished(sink);
assertSourceFinished(sourceA);
assertSource(sourceB, 2);
assertPartitionedRemovePage(sourceB, 1, 2);
assertSourceFinished(sourceA);
assertSource(sourceB, 1);
assertPartitionedRemovePage(sourceB, 1, 2);
assertSourceFinished(sourceA);
assertSourceFinished(sourceB);
assertExchangeTotalBufferedBytes(exchange, 0);
});
}
use of io.trino.operator.exchange.LocalExchange.LocalExchangeFactory in project trino by trinodb.
the class TestLocalExchange method writeUnblockWhenAllReadersFinish.
@Test(dataProvider = "executionStrategy")
public void writeUnblockWhenAllReadersFinish(PipelineExecutionStrategy executionStrategy) {
ImmutableList<Type> types = ImmutableList.of(BIGINT);
LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_BROADCAST_DISTRIBUTION, 2, types, ImmutableList.of(), Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
localExchangeFactory.noMoreSinkFactories();
run(localExchangeFactory, executionStrategy, exchange -> {
assertEquals(exchange.getBufferCount(), 2);
assertExchangeTotalBufferedBytes(exchange, 0);
LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
LocalExchangeSink sinkA = sinkFactory.createSink();
assertSinkCanWrite(sinkA);
LocalExchangeSink sinkB = sinkFactory.createSink();
assertSinkCanWrite(sinkB);
sinkFactory.close();
sinkFactory.noMoreSinkFactories();
LocalExchangeSource sourceA = exchange.getSource(0);
assertSource(sourceA, 0);
LocalExchangeSource sourceB = exchange.getSource(1);
assertSource(sourceB, 0);
sourceA.finish();
assertSourceFinished(sourceA);
assertSinkCanWrite(sinkA);
assertSinkCanWrite(sinkB);
sourceB.finish();
assertSourceFinished(sourceB);
assertSinkFinished(sinkA);
assertSinkFinished(sinkB);
});
}
use of io.trino.operator.exchange.LocalExchange.LocalExchangeFactory in project trino by trinodb.
the class TestLocalExchange method testRandom.
@Test(dataProvider = "executionStrategy")
public void testRandom(PipelineExecutionStrategy executionStrategy) {
LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_ARBITRARY_DISTRIBUTION, 2, TYPES, ImmutableList.of(), Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
localExchangeFactory.noMoreSinkFactories();
run(localExchangeFactory, executionStrategy, exchange -> {
assertEquals(exchange.getBufferCount(), 2);
assertExchangeTotalBufferedBytes(exchange, 0);
LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
LocalExchangeSink sink = sinkFactory.createSink();
assertSinkCanWrite(sink);
sinkFactory.close();
sinkFactory.noMoreSinkFactories();
LocalExchangeSource sourceA = exchange.getSource(0);
assertSource(sourceA, 0);
LocalExchangeSource sourceB = exchange.getSource(1);
assertSource(sourceB, 0);
for (int i = 0; i < 100; i++) {
Page page = createPage(0);
sink.addPage(page);
assertExchangeTotalBufferedBytes(exchange, i + 1);
LocalExchangeBufferInfo bufferInfoA = sourceA.getBufferInfo();
LocalExchangeBufferInfo bufferInfoB = sourceB.getBufferInfo();
assertEquals(bufferInfoA.getBufferedBytes() + bufferInfoB.getBufferedBytes(), retainedSizeOfPages(i + 1));
assertEquals(bufferInfoA.getBufferedPages() + bufferInfoB.getBufferedPages(), i + 1);
}
// we should get ~50 pages per source, but we should get at least some pages in each buffer
assertTrue(sourceA.getBufferInfo().getBufferedPages() > 0);
assertTrue(sourceB.getBufferInfo().getBufferedPages() > 0);
assertExchangeTotalBufferedBytes(exchange, 100);
});
}
use of io.trino.operator.exchange.LocalExchange.LocalExchangeFactory in project trino by trinodb.
the class TestLocalExchange method testPassthrough.
@Test(dataProvider = "executionStrategy")
public void testPassthrough(PipelineExecutionStrategy executionStrategy) {
LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, FIXED_PASSTHROUGH_DISTRIBUTION, 2, TYPES, ImmutableList.of(), Optional.empty(), executionStrategy, DataSize.ofBytes(retainedSizeOfPages(1)), TYPE_OPERATOR_FACTORY);
LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
localExchangeFactory.noMoreSinkFactories();
run(localExchangeFactory, executionStrategy, exchange -> {
assertEquals(exchange.getBufferCount(), 2);
assertExchangeTotalBufferedBytes(exchange, 0);
LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
LocalExchangeSink sinkA = sinkFactory.createSink();
LocalExchangeSink sinkB = sinkFactory.createSink();
assertSinkCanWrite(sinkA);
assertSinkCanWrite(sinkB);
sinkFactory.close();
sinkFactory.noMoreSinkFactories();
LocalExchangeSource sourceA = exchange.getSource(0);
assertSource(sourceA, 0);
LocalExchangeSource sourceB = exchange.getSource(1);
assertSource(sourceB, 0);
sinkA.addPage(createPage(0));
assertSource(sourceA, 1);
assertSource(sourceB, 0);
assertSinkWriteBlocked(sinkA);
assertSinkCanWrite(sinkB);
sinkB.addPage(createPage(1));
assertSource(sourceA, 1);
assertSource(sourceB, 1);
assertSinkWriteBlocked(sinkA);
assertExchangeTotalBufferedBytes(exchange, 2);
assertRemovePage(sourceA, createPage(0));
assertSource(sourceA, 0);
assertSinkCanWrite(sinkA);
assertSinkWriteBlocked(sinkB);
assertExchangeTotalBufferedBytes(exchange, 1);
sinkA.finish();
assertSinkFinished(sinkA);
assertSource(sourceB, 1);
sourceA.finish();
sourceB.finish();
assertRemovePage(sourceB, createPage(1));
assertSourceFinished(sourceA);
assertSourceFinished(sourceB);
assertSinkFinished(sinkB);
assertExchangeTotalBufferedBytes(exchange, 0);
});
}
Aggregations