use of io.prestosql.operator.WorkProcessor in project hetu-core by openlookeng.
the class TestMergeSortedPages method mergeSortedPages.
private static MaterializedResult mergeSortedPages(List<Type> types, List<Integer> sortChannels, List<SortOrder> sortOrder, List<List<Page>> sortedPages) {
List<WorkProcessor<Page>> pageProducers = sortedPages.stream().map(WorkProcessor::fromIterable).collect(toImmutableList());
PageWithPositionComparator comparator = new SimplePageWithPositionComparator(types, sortChannels, sortOrder);
AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newAggregatedMemoryContext();
WorkProcessor<Page> mergedPages = MergeSortedPages.mergeSortedPages(pageProducers, comparator, types, memoryContext, new DriverYieldSignal());
assertTrue(mergedPages.process());
if (mergedPages.isFinished()) {
return toMaterializedResult(TEST_SESSION, types, ImmutableList.of());
}
Page page = mergedPages.getResult();
assertTrue(mergedPages.process());
assertTrue(mergedPages.isFinished());
assertEquals(memoryContext.getBytes(), 0L);
return toMaterializedResult(TEST_SESSION, types, ImmutableList.of(page));
}
use of io.prestosql.operator.WorkProcessor in project hetu-core by openlookeng.
the class MergeSortedPages method buildPage.
private static WorkProcessor<Page> buildPage(WorkProcessor<PageWithPosition> pageWithPositions, List<Integer> outputChannels, List<Type> outputTypes, BiPredicate<PageBuilder, PageWithPosition> pageBreakPredicate, boolean updateMemoryAfterEveryPosition, AggregatedMemoryContext aggregatedMemoryContext, DriverYieldSignal yieldSignal) {
LocalMemoryContext memoryContext = aggregatedMemoryContext.newLocalMemoryContext(MergeSortedPages.class.getSimpleName());
PageBuilder pageBuilder = new PageBuilder(outputTypes);
return pageWithPositions.yielding(yieldSignal::isSet).transform(new WorkProcessor.Transformation<PageWithPosition, Page>() {
@RestorableConfig(stateClassName = "BuildPageState", uncapturedFields = { "val$outputTypes", "val$pageBreakPredicate", "val$outputChannels" })
private final RestorableConfig restorableConfig = null;
@Override
public TransformationState<Page> process(@Nullable PageWithPosition pageWithPosition) {
boolean finished = pageWithPosition == null;
if (finished && pageBuilder.isEmpty()) {
memoryContext.close();
return TransformationState.finished();
}
if (finished || pageBreakPredicate.test(pageBuilder, pageWithPosition)) {
if (!updateMemoryAfterEveryPosition) {
// update memory usage just before producing page to cap from top
memoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
}
Page page = pageBuilder.build();
pageBuilder.reset();
if (!finished) {
pageWithPosition.appendTo(pageBuilder, outputChannels, outputTypes);
}
if (updateMemoryAfterEveryPosition) {
memoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
}
return TransformationState.ofResult(page, !finished);
}
pageWithPosition.appendTo(pageBuilder, outputChannels, outputTypes);
if (updateMemoryAfterEveryPosition) {
memoryContext.setBytes(pageBuilder.getRetainedSizeInBytes());
}
return TransformationState.needsMoreData();
}
@Override
public Object captureResult(Page result, BlockEncodingSerdeProvider serdeProvider) {
if (result != null) {
SerializedPage serializedPage = ((PagesSerde) serdeProvider).serialize(result);
return serializedPage.capture(serdeProvider);
}
return null;
}
@Override
public Page restoreResult(Object resultState, BlockEncodingSerdeProvider serdeProvider) {
if (resultState != null) {
return ((PagesSerde) serdeProvider).deserialize(SerializedPage.restoreSerializedPage(resultState));
}
return null;
}
@Override
public Object captureInput(PageWithPosition input, BlockEncodingSerdeProvider serdeProvider) {
if (input != null) {
return input.capture(serdeProvider);
}
return null;
}
@Override
public PageWithPosition restoreInput(Object inputState, PageWithPosition input, BlockEncodingSerdeProvider serdeProvider) {
if (inputState != null) {
return PageWithPosition.restorePageWithPosition(inputState, serdeProvider);
}
return null;
}
@Override
public Object capture(BlockEncodingSerdeProvider serdeProvider) {
BuildPageState myState = new BuildPageState();
myState.pageBuilder = pageBuilder.capture(serdeProvider);
myState.memoryContext = memoryContext.getBytes();
return myState;
}
@Override
public void restore(Object state, BlockEncodingSerdeProvider serdeProvider) {
BuildPageState myState = (BuildPageState) state;
pageBuilder.restore(myState.pageBuilder, serdeProvider);
memoryContext.setBytes(myState.memoryContext);
}
});
}
use of io.prestosql.operator.WorkProcessor in project hetu-core by openlookeng.
the class PagesSortBenchmark method runPagesMergeSortBenchmark.
@Benchmark
public List<Page> runPagesMergeSortBenchmark(MergeSortedBenchmarkData data) {
WorkProcessor<Page> sortedPagesWork = mergeSortedPages(data.getSplitPages().stream().map(WorkProcessor::fromIterable).collect(toImmutableList()), ORDERING_COMPILER.compilePageWithPositionComparator(data.getSortTypes(), data.getSortChannels(), data.getSortOrders()), data.getOutputChannels(), data.getTypes(), (pageBuilder, pageWithPosition) -> pageBuilder.isFull(), false, newSimpleAggregatedMemoryContext(), new DriverYieldSignal());
ImmutableList.Builder<Page> sortedPages = ImmutableList.builder();
while (true) {
sortedPagesWork.process();
if (sortedPagesWork.isFinished()) {
return sortedPages.build();
}
sortedPages.add(sortedPagesWork.getResult());
}
}
Aggregations