use of io.prestosql.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId in project hetu-core by openlookeng.
the class TestLocalExchange method testPartition.
@Test(dataProvider = "executionStrategy")
public void testPartition(PipelineExecutionStrategy executionStrategy) {
LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(FIXED_HASH_DISTRIBUTION, 2, TYPES, ImmutableList.of(0), Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES);
LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
localExchangeFactory.noMoreSinkFactories();
run(localExchangeFactory, executionStrategy, exchange -> {
assertEquals(exchange.getBufferCount(), 2);
assertExchangeTotalBufferedBytes(exchange, 0);
LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
LocalExchangeSink sink = sinkFactory.createSink("");
assertSinkCanWrite(sink);
sinkFactory.close();
sinkFactory.noMoreSinkFactories();
LocalExchangeSource sourceA = exchange.getSource(0);
assertSource(sourceA, 0);
LocalExchangeSource sourceB = exchange.getSource(1);
assertSource(sourceB, 0);
sink.addPage(createPage(0), null);
assertSource(sourceA, 1);
assertSource(sourceB, 1);
assertTrue(exchange.getBufferedBytes() >= retainedSizeOfPages(1));
sink.addPage(createPage(0), null);
assertSource(sourceA, 2);
assertSource(sourceB, 2);
assertTrue(exchange.getBufferedBytes() >= retainedSizeOfPages(2));
assertPartitionedRemovePage(sourceA, 0, 2);
assertSource(sourceA, 1);
assertSource(sourceB, 2);
assertPartitionedRemovePage(sourceA, 0, 2);
assertSource(sourceA, 0);
assertSource(sourceB, 2);
sink.finish();
assertSinkFinished(sink);
assertSourceFinished(sourceA);
assertSource(sourceB, 2);
assertPartitionedRemovePage(sourceB, 1, 2);
assertSourceFinished(sourceA);
assertSource(sourceB, 1);
assertPartitionedRemovePage(sourceB, 1, 2);
assertSourceFinished(sourceA);
assertSourceFinished(sourceB);
assertExchangeTotalBufferedBytes(exchange, 0);
});
}
use of io.prestosql.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId in project hetu-core by openlookeng.
the class TestLocalExchange method testGatherSingleWriter.
@Test(dataProvider = "executionStrategy")
public void testGatherSingleWriter(PipelineExecutionStrategy executionStrategy) {
LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(SINGLE_DISTRIBUTION, 8, TYPES, ImmutableList.of(), Optional.empty(), executionStrategy, new DataSize(retainedSizeOfPages(99), BYTE));
LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
localExchangeFactory.noMoreSinkFactories();
run(localExchangeFactory, executionStrategy, exchange -> {
assertEquals(exchange.getBufferCount(), 1);
assertExchangeTotalBufferedBytes(exchange, 0);
LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
LocalExchangeSource source = exchange.getSource(0);
assertSource(source, 0);
LocalExchangeSink sink = sinkFactory.createSink("");
sinkFactory.close();
sinkFactory.noMoreSinkFactories();
assertSinkCanWrite(sink);
assertSource(source, 0);
// add the first page which should cause the reader to unblock
ListenableFuture<?> readFuture = source.waitForReading();
assertFalse(readFuture.isDone());
sink.addPage(createPage(0), null);
assertTrue(readFuture.isDone());
assertExchangeTotalBufferedBytes(exchange, 1);
assertSource(source, 1);
sink.addPage(createPage(1), null);
assertSource(source, 2);
assertExchangeTotalBufferedBytes(exchange, 2);
assertRemovePage(source, createPage(0));
assertSource(source, 1);
assertExchangeTotalBufferedBytes(exchange, 1);
assertRemovePage(source, createPage(1));
assertSource(source, 0);
assertExchangeTotalBufferedBytes(exchange, 0);
sink.addPage(createPage(2), null);
sink.addPage(createPage(3), null);
assertSource(source, 2);
assertExchangeTotalBufferedBytes(exchange, 2);
sink.finish();
assertSinkFinished(sink);
assertSource(source, 2);
assertRemovePage(source, createPage(2));
assertSource(source, 1);
assertSinkFinished(sink);
assertExchangeTotalBufferedBytes(exchange, 1);
assertRemovePage(source, createPage(3));
assertSourceFinished(source);
assertExchangeTotalBufferedBytes(exchange, 0);
});
}
use of io.prestosql.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId in project hetu-core by openlookeng.
the class TestLocalExchange method writeUnblockWhenAllReadersFinish.
@Test(dataProvider = "executionStrategy")
public void writeUnblockWhenAllReadersFinish(PipelineExecutionStrategy executionStrategy) {
ImmutableList<Type> types = ImmutableList.of(BIGINT);
LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(FIXED_BROADCAST_DISTRIBUTION, 2, types, ImmutableList.of(), Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES);
LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
localExchangeFactory.noMoreSinkFactories();
run(localExchangeFactory, executionStrategy, exchange -> {
assertEquals(exchange.getBufferCount(), 2);
assertExchangeTotalBufferedBytes(exchange, 0);
LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
LocalExchangeSink sinkA = sinkFactory.createSink("sinkA");
assertSinkCanWrite(sinkA);
LocalExchangeSink sinkB = sinkFactory.createSink("sinkB");
assertSinkCanWrite(sinkB);
sinkFactory.close();
sinkFactory.noMoreSinkFactories();
LocalExchangeSource sourceA = exchange.getSource(0);
assertSource(sourceA, 0);
LocalExchangeSource sourceB = exchange.getSource(1);
assertSource(sourceB, 0);
sourceA.finish();
assertSourceFinished(sourceA);
assertSinkCanWrite(sinkA);
assertSinkCanWrite(sinkB);
sourceB.finish();
assertSourceFinished(sourceB);
assertSinkFinished(sinkA);
assertSinkFinished(sinkB);
});
}
use of io.prestosql.operator.exchange.LocalExchange.LocalExchangeSinkFactoryId in project hetu-core by openlookeng.
the class TestHashJoinOperator method setupBuildSide.
private BuildSideSetup setupBuildSide(boolean parallelBuild, TaskContext taskContext, List<Integer> hashChannels, RowPagesBuilder buildPages, Optional<InternalJoinFilterFunction> filterFunction, boolean spillEnabled, SingleStreamSpillerFactory singleStreamSpillerFactory, boolean outer) {
Optional<JoinFilterFunctionFactory> filterFunctionFactory = filterFunction.map(function -> (session, addresses, pages) -> new StandardJoinFilterFunction(function, addresses, pages));
int partitionCount = parallelBuild ? PARTITION_COUNT : 1;
LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(FIXED_HASH_DISTRIBUTION, partitionCount, buildPages.getTypes(), hashChannels, buildPages.getHashChannel(), UNGROUPED_EXECUTION, new DataSize(32, DataSize.Unit.MEGABYTE));
LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
localExchangeFactory.noMoreSinkFactories();
// collect input data into the partitioned exchange
DriverContext collectDriverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory(0, new PlanNodeId("values"), buildPages.build());
LocalExchangeSinkOperatorFactory sinkOperatorFactory = new LocalExchangeSinkOperatorFactory(localExchangeFactory, 1, new PlanNodeId("sink"), localExchangeSinkFactoryId, Function.identity());
Driver sourceDriver = Driver.createDriver(collectDriverContext, valuesOperatorFactory.createOperator(collectDriverContext), sinkOperatorFactory.createOperator(collectDriverContext));
valuesOperatorFactory.noMoreOperators();
sinkOperatorFactory.noMoreOperators();
while (!sourceDriver.isFinished()) {
sourceDriver.process();
}
// build side operator factories
LocalExchangeSourceOperatorFactory sourceOperatorFactory = new LocalExchangeSourceOperatorFactory(0, new PlanNodeId("source"), localExchangeFactory, 1);
PartitionedLookupSourceFactory lookupSourceFactory = new PartitionedLookupSourceFactory(buildPages.getTypes(), rangeList(buildPages.getTypes().size()).stream().map(buildPages.getTypes()::get).collect(toImmutableList()), hashChannels.stream().map(buildPages.getTypes()::get).collect(toImmutableList()), partitionCount, requireNonNull(ImmutableMap.of(), "layout is null"), outer, false);
JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = new JoinBridgeManager<>(outer, UNGROUPED_EXECUTION, UNGROUPED_EXECUTION, ignored -> lookupSourceFactory, lookupSourceFactory.getOutputTypes());
HashBuilderOperatorFactory buildOperatorFactory = new HashBuilderOperatorFactory(1, new PlanNodeId("build"), lookupSourceFactoryManager, rangeList(buildPages.getTypes().size()), hashChannels, buildPages.getHashChannel().map(OptionalInt::of).orElse(OptionalInt.empty()), filterFunctionFactory, Optional.empty(), ImmutableList.of(), 100, new PagesIndex.TestingFactory(false), spillEnabled, singleStreamSpillerFactory);
return new BuildSideSetup(lookupSourceFactoryManager, buildOperatorFactory, sourceOperatorFactory, partitionCount);
}
Aggregations