Examples with DriverYieldSignal io.trino.operator.DriverYieldSignal used on opensource projects

Example 26 with DriverYieldSignal

use of io.trino.operator.DriverYieldSignal in project trino by trinodb.

the class TestDictionaryAwarePageProjection method testProjectList.

private static void testProjectList(Block block, Class<? extends Block> expectedResultType, DictionaryAwarePageProjection projection, boolean forceYield, boolean produceLazyBlock) {
    if (produceLazyBlock) {
        block = lazyWrapper(block);
    }
    DriverYieldSignal yieldSignal = new DriverYieldSignal();
    int[] positions = { 0, 2, 4, 6, 8, 10 };
    Work<Block> work = projection.project(null, yieldSignal, new Page(block), SelectedPositions.positionsList(positions, 0, positions.length));
    Block result;
    if (forceYield) {
        result = projectWithYield(work, yieldSignal);
    } else {
        assertTrue(work.process());
        result = work.getResult();
    }
    if (produceLazyBlock) {
        assertInstanceOf(result, LazyBlock.class);
        assertFalse(result.isLoaded());
        assertFalse(block.isLoaded());
        result = result.getLoadedBlock();
    }
    assertBlockEquals(BIGINT, result, block.copyPositions(positions, 0, positions.length));
    assertInstanceOf(result, expectedResultType);
}

Example 27 with DriverYieldSignal

use of io.trino.operator.DriverYieldSignal in project trino by trinodb.

the class TestDictionaryAwarePageProjection method testPreservesDictionaryInstance.

@Test
public void testPreservesDictionaryInstance() {
    DictionaryAwarePageProjection projection = new DictionaryAwarePageProjection(new InputPageProjection(0, BIGINT), block -> randomDictionaryId(), false);
    Block dictionary = createLongsBlock(0, 1);
    DictionaryBlock firstDictionaryBlock = new DictionaryBlock(dictionary, new int[] { 0, 1, 2, 3 });
    DictionaryBlock secondDictionaryBlock = new DictionaryBlock(dictionary, new int[] { 3, 2, 1, 0 });
    DriverYieldSignal yieldSignal = new DriverYieldSignal();
    Work<Block> firstWork = projection.project(null, yieldSignal, new Page(firstDictionaryBlock), SelectedPositions.positionsList(new int[] { 0 }, 0, 1));
    assertTrue(firstWork.process());
    Block firstOutputBlock = firstWork.getResult();
    assertInstanceOf(firstOutputBlock, DictionaryBlock.class);
    Work<Block> secondWork = projection.project(null, yieldSignal, new Page(secondDictionaryBlock), SelectedPositions.positionsList(new int[] { 0 }, 0, 1));
    assertTrue(secondWork.process());
    Block secondOutputBlock = secondWork.getResult();
    assertInstanceOf(secondOutputBlock, DictionaryBlock.class);
    assertNotSame(firstOutputBlock, secondOutputBlock);
    Block firstDictionary = ((DictionaryBlock) firstOutputBlock).getDictionary();
    Block secondDictionary = ((DictionaryBlock) secondOutputBlock).getDictionary();
    assertSame(firstDictionary, secondDictionary);
    assertSame(firstDictionary, dictionary);
}

Example 28 with DriverYieldSignal

use of io.trino.operator.DriverYieldSignal in project trino by trinodb.

the class TestDictionaryAwarePageProjection method testProjectRange.

private static void testProjectRange(Block block, Class<? extends Block> expectedResultType, DictionaryAwarePageProjection projection, boolean forceYield, boolean produceLazyBlock) {
    if (produceLazyBlock) {
        block = lazyWrapper(block);
    }
    DriverYieldSignal yieldSignal = new DriverYieldSignal();
    Work<Block> work = projection.project(null, yieldSignal, new Page(block), SelectedPositions.positionsRange(5, 10));
    Block result;
    if (forceYield) {
        result = projectWithYield(work, yieldSignal);
    } else {
        assertTrue(work.process());
        result = work.getResult();
    }
    if (produceLazyBlock) {
        assertInstanceOf(result, LazyBlock.class);
        assertFalse(result.isLoaded());
        assertFalse(block.isLoaded());
        result = result.getLoadedBlock();
    }
    assertBlockEquals(BIGINT, result, block.getRegion(5, 10));
    assertInstanceOf(result, expectedResultType);
}

Example 29 with DriverYieldSignal

use of io.trino.operator.DriverYieldSignal in project trino by trinodb.

the class FunctionAssertions method assertCachedInstanceHasBoundedRetainedSizeInTx.

private void assertCachedInstanceHasBoundedRetainedSizeInTx(String projection, Session session) {
    requireNonNull(projection, "projection is null");
    Expression projectionExpression = createExpression(session, projection, getPlannerContext(), INPUT_TYPES);
    RowExpression projectionRowExpression = toRowExpression(session, projectionExpression);
    PageProcessor processor = runner.getExpressionCompiler().compilePageProcessor(Optional.empty(), ImmutableList.of(projectionRowExpression)).get();
    // This is a heuristic to detect whether the retained size of cachedInstance is bounded.
    // * The test runs at least 1000 iterations.
    // * The test passes if max retained size doesn't refresh after
    // 4x the number of iterations when max was last updated.
    // * The test fails if retained size reaches 1MB.
    // Note that 1MB is arbitrarily chosen and may be increased if a function implementation
    // legitimately needs more.
    long maxRetainedSize = 0;
    int maxIterationCount = 0;
    for (int iterationCount = 0; iterationCount < Math.max(1000, maxIterationCount * 4); iterationCount++) {
        Iterator<Optional<Page>> output = processor.process(session.toConnectorSession(), new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), SOURCE_PAGE);
        // consume the iterator
        @SuppressWarnings("unused") Optional<Page> ignored = Iterators.getOnlyElement(output);
        long retainedSize = processor.getProjections().stream().mapToLong(this::getRetainedSizeOfCachedInstance).sum();
        if (retainedSize > maxRetainedSize) {
            maxRetainedSize = retainedSize;
            maxIterationCount = iterationCount;
        }
        if (maxRetainedSize >= 1048576) {
            fail(format("The retained size of cached instance of function invocation is likely unbounded: %s", projection));
        }
    }
}

Example 30 with DriverYieldSignal

use of io.trino.operator.DriverYieldSignal in project trino by trinodb.

the class TestPageProcessor method testRetainedSize.

@Test
public void testRetainedSize() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new InputPageProjection(0, VARCHAR), new InputPageProjection(1, VARCHAR)), OptionalInt.of(MAX_BATCH_SIZE));
    // create 2 columns X 800 rows of strings with each string's size = 10KB
    // this can force previouslyComputedResults to be saved given the page is 16MB in size
    String value = join("", nCopies(10_000, "a"));
    List<String> values = nCopies(800, value);
    Page inputPage = new Page(createStringsBlock(values), createStringsBlock(values));
    AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext();
    Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, new DriverYieldSignal(), memoryContext, inputPage);
    // force a compute
    // one block of previouslyComputedResults will be saved given the first column is with 8MB
    assertTrue(output.hasNext());
    // verify we do not count block sizes twice
    // comparing with the input page, the output page also contains an extra instance size for previouslyComputedResults
    assertEquals(memoryContext.getBytes() - ClassLayout.parseClass(VariableWidthBlock.class).instanceSize(), inputPage.getRetainedSizeInBytes());
}