Search in sources :

Example 1 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestMergeSortedPages method testMergeSortYieldingProgresses.

@Test
public void testMergeSortYieldingProgresses() {
    DriverYieldSignal yieldSignal = new DriverYieldSignal();
    yieldSignal.forceYieldForTesting();
    List<Type> types = ImmutableList.of(INTEGER);
    WorkProcessor<Page> mergedPages = MergeSortedPages.mergeSortedPages(ImmutableList.of(WorkProcessor.fromIterable(rowPagesBuilder(types).build())), new SimplePageWithPositionComparator(types, ImmutableList.of(0), ImmutableList.of(DESC_NULLS_LAST)), ImmutableList.of(0), types, (pageBuilder, pageWithPosition) -> pageBuilder.isFull(), false, newSimpleAggregatedMemoryContext().newAggregatedMemoryContext(), yieldSignal);
    // yield signal is on
    assertFalse(mergedPages.process());
    // processor finishes computations (yield signal is still on, but previous process() call yielded)
    assertTrue(mergedPages.process());
    assertTrue(mergedPages.isFinished());
}
Also used : Type(io.prestosql.spi.type.Type) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) Page(io.prestosql.spi.Page) SimplePageWithPositionComparator(io.prestosql.operator.SimplePageWithPositionComparator) Test(org.testng.annotations.Test)

Example 2 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestMergeSortedPages method mergeSortedPages.

private static MaterializedResult mergeSortedPages(List<Type> types, List<Integer> sortChannels, List<SortOrder> sortOrder, List<List<Page>> sortedPages) {
    List<WorkProcessor<Page>> pageProducers = sortedPages.stream().map(WorkProcessor::fromIterable).collect(toImmutableList());
    PageWithPositionComparator comparator = new SimplePageWithPositionComparator(types, sortChannels, sortOrder);
    AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newAggregatedMemoryContext();
    WorkProcessor<Page> mergedPages = MergeSortedPages.mergeSortedPages(pageProducers, comparator, types, memoryContext, new DriverYieldSignal());
    assertTrue(mergedPages.process());
    if (mergedPages.isFinished()) {
        return toMaterializedResult(TEST_SESSION, types, ImmutableList.of());
    }
    Page page = mergedPages.getResult();
    assertTrue(mergedPages.process());
    assertTrue(mergedPages.isFinished());
    assertEquals(memoryContext.getBytes(), 0L);
    return toMaterializedResult(TEST_SESSION, types, ImmutableList.of(page));
}
Also used : WorkProcessor(io.prestosql.operator.WorkProcessor) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) SimplePageWithPositionComparator(io.prestosql.operator.SimplePageWithPositionComparator) Page(io.prestosql.spi.Page) SimplePageWithPositionComparator(io.prestosql.operator.SimplePageWithPositionComparator) PageWithPositionComparator(io.prestosql.operator.PageWithPositionComparator) AggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext)

Example 3 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageFunctionCompiler method project.

private Block project(PageProjection projection, Page page, SelectedPositions selectedPositions) {
    Work<Block> work = projection.project(SESSION, new DriverYieldSignal(), page, selectedPositions);
    assertTrue(work.process());
    return work.getResult();
}
Also used : Block(io.prestosql.spi.block.Block) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal)

Example 4 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageProcessor method testSelectNoneFilter.

@Test
public void testSelectNoneFilter() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectNoneFilter()), ImmutableList.of(new InputPageProjection(0, BIGINT)));
    Page inputPage = new Page(createLongSequenceBlock(0, 100));
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName());
    Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage);
    assertEquals(memoryContext.getBytes(), 0);
    List<Optional<Page>> outputPages = ImmutableList.copyOf(output);
    assertEquals(outputPages.size(), 0);
}
Also used : LocalMemoryContext(io.prestosql.memory.context.LocalMemoryContext) Optional(java.util.Optional) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Example 5 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageProcessor method testRetainedSize.

@Test
public void testRetainedSize() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new InputPageProjection(0, VARCHAR), new InputPageProjection(1, VARCHAR)), OptionalInt.of(MAX_BATCH_SIZE));
    // create 2 columns X 800 rows of strings with each string's size = 10KB
    // this can force previouslyComputedResults to be saved given the page is 16MB in size
    String value = join("", nCopies(10_000, "a"));
    List<String> values = nCopies(800, value);
    Page inputPage = new Page(createStringsBlock(values), createStringsBlock(values));
    AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext();
    Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, new DriverYieldSignal(), memoryContext, inputPage);
    // force a compute
    // one block of previouslyComputedResults will be saved given the first column is with 8MB
    output.hasNext();
    // verify we do not count block sizes twice
    // comparing with the input page, the output page also contains an extra instance size for previouslyComputedResults
    assertEquals(memoryContext.getBytes() - ClassLayout.parseClass(VariableWidthBlock.class).instanceSize(), inputPage.getRetainedSizeInBytes());
}
Also used : Optional(java.util.Optional) DriverYieldSignal(io.prestosql.operator.DriverYieldSignal) Page(io.prestosql.spi.Page) AggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext) VariableWidthBlock(io.prestosql.spi.block.VariableWidthBlock) Test(org.testng.annotations.Test)

Aggregations

DriverYieldSignal (io.prestosql.operator.DriverYieldSignal)25 Page (io.prestosql.spi.Page)22 Test (org.testng.annotations.Test)18 Block (io.prestosql.spi.block.Block)9 Optional (java.util.Optional)9 BlockAssertions.createLongSequenceBlock (io.prestosql.block.BlockAssertions.createLongSequenceBlock)7 PageProcessor (io.prestosql.operator.project.PageProcessor)7 LazyBlock (io.prestosql.spi.block.LazyBlock)7 DictionaryBlock (io.prestosql.spi.block.DictionaryBlock)6 BuiltInFunctionHandle (io.prestosql.spi.function.BuiltInFunctionHandle)6 CallExpression (io.prestosql.spi.relation.CallExpression)6 Slice (io.airlift.slice.Slice)5 LocalMemoryContext (io.prestosql.memory.context.LocalMemoryContext)5 RunLengthEncodedBlock (io.prestosql.spi.block.RunLengthEncodedBlock)5 VariableWidthBlock (io.prestosql.spi.block.VariableWidthBlock)5 Type (io.prestosql.spi.type.Type)5 BlockAssertions.createSlicesBlock (io.prestosql.block.BlockAssertions.createSlicesBlock)4 BlockAssertions.createStringsBlock (io.prestosql.block.BlockAssertions.createStringsBlock)4 AggregatedMemoryContext (io.prestosql.memory.context.AggregatedMemoryContext)4 AggregatedMemoryContext.newSimpleAggregatedMemoryContext (io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext)4