use of io.prestosql.operator.project.PageProcessor in project hetu-core by openlookeng.
the class TestScanFilterAndProjectOperator method testRecordCursorYield.
@Test
public void testRecordCursorYield() {
// create a generic long function that yields for projection on every row
// verify we will yield #row times totally
// create a table with 15 rows
int length = 15;
Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), length, 0);
DriverContext driverContext = newDriverContext();
// set up generic long function with a callback to force yield
Metadata localMetadata = functionAssertions.getMetadata();
localMetadata.getFunctionAndTypeManager().registerBuiltInFunctions(ImmutableList.of(new GenericLongFunction("record_cursor", value -> {
driverContext.getYieldSignal().forceYieldForTesting();
return value;
})));
ExpressionCompiler compiler = new ExpressionCompiler(localMetadata, new PageFunctionCompiler(localMetadata, 0));
List<RowExpression> projections = ImmutableList.of(call(QualifiedObjectName.valueOfDefaultFunction("generic_long_record_cursor").toString(), new BuiltInFunctionHandle(internalScalarFunction(QualifiedObjectName.valueOfDefaultFunction("generic_long_record_cursor"), BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature()))), BIGINT, field(0, BIGINT)));
Supplier<CursorProcessor> cursorProcessor = compiler.compileCursorProcessor(Optional.empty(), projections, "key");
Supplier<PageProcessor> pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections);
ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new RecordPageSource(new PageRecordSet(ImmutableList.of(BIGINT), input)), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), null, ImmutableList.of(BIGINT), new DataSize(0, BYTE), 0, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), false, Optional.empty(), 0, 0);
SourceOperator operator = factory.createOperator(driverContext);
operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
operator.noMoreSplits();
// start driver; get null value due to yield for the first 15 times
for (int i = 0; i < length; i++) {
driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
assertNull(operator.getOutput());
driverContext.getYieldSignal().reset();
}
// the 16th yield is not going to prevent the operator from producing a page
driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
Page output = operator.getOutput();
driverContext.getYieldSignal().reset();
assertNotNull(output);
assertEquals(toValues(BIGINT, output.getBlock(0)), toValues(BIGINT, input.getBlock(0)));
}
use of io.prestosql.operator.project.PageProcessor in project hetu-core by openlookeng.
the class TestColumnarPageProcessor method testProcess.
@Test
public void testProcess() {
PageProcessor processor = newPageProcessor();
Page page = createPage(types, false);
Page outputPage = getOnlyElement(processor.process(SESSION, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
assertPageEquals(types, outputPage, page);
}
use of io.prestosql.operator.project.PageProcessor in project hetu-core by openlookeng.
the class TestTupleFilterProcessor method testFilter.
@Test
public void testFilter() {
Page tuplePage = Iterables.getOnlyElement(rowPagesBuilder(BIGINT, VARCHAR, DOUBLE).row(1L, "a", 0.1).build());
List<Type> outputTypes = ImmutableList.of(VARCHAR, BIGINT, BOOLEAN, DOUBLE, DOUBLE);
Page inputPage = Iterables.getOnlyElement(rowPagesBuilder(outputTypes).row("a", 1L, true, 0.1, 0.0).row("b", 1L, true, 0.1, 2.0).row("a", 1L, false, 0.1, 2.0).row("a", 0L, false, 0.2, 0.2).build());
DynamicTupleFilterFactory filterFactory = new DynamicTupleFilterFactory(42, new PlanNodeId("42"), new int[] { 0, 1, 2 }, new int[] { 1, 0, 3 }, outputTypes, new PageFunctionCompiler(createTestMetadataManager(), 0));
PageProcessor tupleFilterProcessor = filterFactory.createPageProcessor(tuplePage, OptionalInt.of(MAX_BATCH_SIZE)).get();
Page actualPage = getOnlyElement(tupleFilterProcessor.process(SESSION, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), inputPage)).orElseThrow(() -> new AssertionError("page is not present"));
Page expectedPage = Iterables.getOnlyElement(rowPagesBuilder(outputTypes).row("a", 1L, true, 0.1, 0.0).row("a", 1L, false, 0.1, 2.0).build());
assertPageEquals(outputTypes, actualPage, expectedPage);
}
use of io.prestosql.operator.project.PageProcessor in project hetu-core by openlookeng.
the class FunctionAssertions method assertCachedInstanceHasBoundedRetainedSize.
public void assertCachedInstanceHasBoundedRetainedSize(String projection) {
requireNonNull(projection, "projection is null");
Expression projectionExpression = createExpression(session, projection, metadata, TypeProvider.copyOf(INPUT_TYPES));
RowExpression projectionRowExpression = toRowExpression(session, projectionExpression);
PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(projectionRowExpression)).get();
// This is a heuristic to detect whether the retained size of cachedInstance is bounded.
// * The test runs at least 1000 iterations.
// * The test passes if max retained size doesn't refresh after
// 4x the number of iterations when max was last updated.
// * The test fails if retained size reaches 1MB.
// Note that 1MB is arbitrarily chosen and may be increased if a function implementation
// legitimately needs more.
long maxRetainedSize = 0;
int maxIterationCount = 0;
for (int iterationCount = 0; iterationCount < Math.max(1000, maxIterationCount * 4); iterationCount++) {
Iterator<Optional<Page>> output = processor.process(session.toConnectorSession(), new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), SOURCE_PAGE);
// consume the iterator
Iterators.getOnlyElement(output);
long retainedSize = processor.getProjections().stream().mapToLong(this::getRetainedSizeOfCachedInstance).sum();
if (retainedSize > maxRetainedSize) {
maxRetainedSize = retainedSize;
maxIterationCount = iterationCount;
}
if (maxRetainedSize >= 1048576) {
fail(format("The retained size of cached instance of function invocation is likely unbounded: %s", projection));
}
}
}
use of io.prestosql.operator.project.PageProcessor in project hetu-core by openlookeng.
the class HandTpchQuery6 method createOperatorFactories.
@Override
protected List<? extends OperatorFactory> createOperatorFactories() {
// select sum(extendedprice * discount) as revenue
// from lineitem
// where shipdate >= '1994-01-01'
// and shipdate < '1995-01-01'
// and discount >= 0.05
// and discount <= 0.07
// and quantity < 24;
OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "extendedprice", "discount", "shipdate", "quantity");
Supplier<PageProjection> projection = new PageFunctionCompiler(localQueryRunner.getMetadata(), 0).compileProjection(field(0, BIGINT), Optional.empty());
FilterAndProjectOperator.FilterAndProjectOperatorFactory tpchQuery6Operator = new FilterAndProjectOperator.FilterAndProjectOperatorFactory(1, new PlanNodeId("test"), () -> new PageProcessor(Optional.of(new TpchQuery6Filter()), ImmutableList.of(projection.get())), ImmutableList.of(DOUBLE), new DataSize(0, BYTE), 0);
AggregationOperatorFactory aggregationOperator = new AggregationOperatorFactory(2, new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(doubleSum.bind(ImmutableList.of(0), Optional.empty())), false);
return ImmutableList.of(tableScanOperator, tpchQuery6Operator, aggregationOperator);
}
Aggregations