Examples with WorkProcessor io.trino.operator.WorkProcessor used on opensource projects

Example 1 with WorkProcessor

use of io.trino.operator.WorkProcessor in project trino by trinodb.

the class MergeSortedPages method pageWithPositions.

private static WorkProcessor<PageWithPosition> pageWithPositions(WorkProcessor<Page> pages, AggregatedMemoryContext aggregatedMemoryContext) {
    return pages.flatMap(page -> {
        LocalMemoryContext memoryContext = aggregatedMemoryContext.newLocalMemoryContext(MergeSortedPages.class.getSimpleName());
        memoryContext.setBytes(page.getRetainedSizeInBytes());
        return WorkProcessor.create(new WorkProcessor.Process<PageWithPosition>() {

            int position;

            @Override
            public ProcessState<PageWithPosition> process() {
                if (position >= page.getPositionCount()) {
                    memoryContext.close();
                    return ProcessState.finished();
                }
                return ProcessState.ofResult(new PageWithPosition(page, position++));
            }
        });
    });
}

Example 2 with WorkProcessor

use of io.trino.operator.WorkProcessor in project trino by trinodb.

the class BenchmarkPagesSort method runPagesMergeSortBenchmark.

@Benchmark
public List<Page> runPagesMergeSortBenchmark(MergeSortedBenchmarkData data) {
    WorkProcessor<Page> sortedPagesWork = mergeSortedPages(data.getSplitPages().stream().map(WorkProcessor::fromIterable).collect(toImmutableList()), ORDERING_COMPILER.compilePageWithPositionComparator(data.getSortTypes(), data.getSortChannels(), data.getSortOrders()), data.getOutputChannels(), data.getTypes(), (pageBuilder, pageWithPosition) -> pageBuilder.isFull(), false, newSimpleAggregatedMemoryContext(), new DriverYieldSignal());
    ImmutableList.Builder<Page> sortedPages = ImmutableList.builder();
    while (true) {
        sortedPagesWork.process();
        if (sortedPagesWork.isFinished()) {
            return sortedPages.build();
        }
        sortedPages.add(sortedPagesWork.getResult());
    }
}

Example 3 with WorkProcessor

use of io.trino.operator.WorkProcessor in project trino by trinodb.

the class TestMergeSortedPages method mergeSortedPages.

private static MaterializedResult mergeSortedPages(List<Type> types, List<Integer> sortChannels, List<SortOrder> sortOrder, List<List<Page>> sortedPages) {
    List<WorkProcessor<Page>> pageProducers = sortedPages.stream().map(WorkProcessor::fromIterable).collect(toImmutableList());
    PageWithPositionComparator comparator = new SimplePageWithPositionComparator(types, sortChannels, sortOrder, TYPE_OPERATORS_CACHE);
    AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newAggregatedMemoryContext();
    WorkProcessor<Page> mergedPages = MergeSortedPages.mergeSortedPages(pageProducers, comparator, types, memoryContext, new DriverYieldSignal());
    assertTrue(mergedPages.process());
    if (mergedPages.isFinished()) {
        return toMaterializedResult(TEST_SESSION, types, ImmutableList.of());
    }
    Page page = mergedPages.getResult();
    assertTrue(mergedPages.process());
    assertTrue(mergedPages.isFinished());
    assertEquals(memoryContext.getBytes(), 0L);
    return toMaterializedResult(TEST_SESSION, types, ImmutableList.of(page));
}