Examples with DriverYieldSignal io.prestosql.operator.DriverYieldSignal used on opensource projects

Example 11 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class FunctionAssertions method assertCachedInstanceHasBoundedRetainedSize.

public void assertCachedInstanceHasBoundedRetainedSize(String projection) {
    requireNonNull(projection, "projection is null");
    Expression projectionExpression = createExpression(session, projection, metadata, TypeProvider.copyOf(INPUT_TYPES));
    RowExpression projectionRowExpression = toRowExpression(session, projectionExpression);
    PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(projectionRowExpression)).get();
    // This is a heuristic to detect whether the retained size of cachedInstance is bounded.
    // * The test runs at least 1000 iterations.
    // * The test passes if max retained size doesn't refresh after
    // 4x the number of iterations when max was last updated.
    // * The test fails if retained size reaches 1MB.
    // Note that 1MB is arbitrarily chosen and may be increased if a function implementation
    // legitimately needs more.
    long maxRetainedSize = 0;
    int maxIterationCount = 0;
    for (int iterationCount = 0; iterationCount < Math.max(1000, maxIterationCount * 4); iterationCount++) {
        Iterator<Optional<Page>> output = processor.process(session.toConnectorSession(), new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), SOURCE_PAGE);
        // consume the iterator
        Iterators.getOnlyElement(output);
        long retainedSize = processor.getProjections().stream().mapToLong(this::getRetainedSizeOfCachedInstance).sum();
        if (retainedSize > maxRetainedSize) {
            maxRetainedSize = retainedSize;
            maxIterationCount = iterationCount;
        }
        if (maxRetainedSize >= 1048576) {
            fail(format("The retained size of cached instance of function invocation is likely unbounded: %s", projection));
        }
    }
}

Example 12 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageProcessorCompiler method testNonDeterministicProject.

@Test
public void testNonDeterministicProject() {
    Signature lessThan = internalOperator(LESS_THAN, BOOLEAN, ImmutableList.of(BIGINT, BIGINT));
    CallExpression random = new CallExpression(QualifiedObjectName.valueOfDefaultFunction("random").getObjectName(), new BuiltInFunctionHandle(new Signature(QualifiedObjectName.valueOfDefaultFunction("random"), SCALAR, parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.BIGINT))), BIGINT, singletonList(constant(10L, BIGINT)), Optional.empty());
    InputReferenceExpression col0 = field(0, BIGINT);
    CallExpression lessThanRandomExpression = new CallExpression(lessThan.getName().getObjectName(), new BuiltInFunctionHandle(lessThan), BOOLEAN, ImmutableList.of(col0, random), Optional.empty());
    PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(lessThanRandomExpression), MAX_BATCH_SIZE).get();
    assertFalse(new RowExpressionDeterminismEvaluator(metadata).isDeterministic(lessThanRandomExpression));
    Page page = new Page(createLongDictionaryBlock(1, 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertFalse(outputPage.getBlock(0) instanceof DictionaryBlock);
}

Example 13 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageProcessor method testProjectEmptyPage.

@Test
public void testProjectEmptyPage() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new InputPageProjection(0, BIGINT)));
    Page inputPage = new Page(createLongSequenceBlock(0, 0));
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName());
    Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage);
    assertEquals(memoryContext.getBytes(), 0);
    // output should be one page containing no columns (only a count)
    List<Optional<Page>> outputPages = ImmutableList.copyOf(output);
    assertEquals(outputPages.size(), 0);
}

Example 14 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageProcessor method testSelectNoneFilterLazyLoad.

@Test
public void testSelectNoneFilterLazyLoad() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectNoneFilter()), ImmutableList.of(new InputPageProjection(1, BIGINT)));
    // if channel 1 is loaded, test will fail
    Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> {
        throw new AssertionError("Lazy block should not be loaded");
    }));
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName());
    Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage);
    assertEquals(memoryContext.getBytes(), 0);
    List<Optional<Page>> outputPages = ImmutableList.copyOf(output);
    assertEquals(outputPages.size(), 0);
}

Example 15 with DriverYieldSignal

use of io.prestosql.operator.DriverYieldSignal in project hetu-core by openlookeng.

the class TestPageProcessor method testExpressionProfiler.

@Test
public void testExpressionProfiler() {
    CallExpression add10Expression = call(ADD.getFunctionName().toString(), new BuiltInFunctionHandle(internalOperator(ADD, BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), BIGINT.getTypeSignature()))), BIGINT, field(0, BIGINT), constant(10L, BIGINT));
    TestingTicker testingTicker = new TestingTicker();
    PageFunctionCompiler functionCompiler = new PageFunctionCompiler(createTestMetadataManager(), 0);
    Supplier<PageProjection> projectionSupplier = functionCompiler.compileProjection(add10Expression, Optional.empty());
    PageProjection projection = projectionSupplier.get();
    Page page = new Page(createLongSequenceBlock(1, 11));
    ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, SPLIT_RUN_QUANTA);
    for (int i = 0; i < 100; i++) {
        profiler.start();
        Work<Block> work = projection.project(SESSION, new DriverYieldSignal(), page, SelectedPositions.positionsRange(0, page.getPositionCount()));
        if (i < 10) {
            // increment the ticker with a large value to mark the expression as expensive
            testingTicker.increment(10, SECONDS);
            profiler.stop(page.getPositionCount());
            assertTrue(profiler.isExpressionExpensive());
        } else {
            testingTicker.increment(0, NANOSECONDS);
            profiler.stop(page.getPositionCount());
            assertFalse(profiler.isExpressionExpensive());
        }
        work.process();
    }
}