Search in sources :

Example 6 with CursorProcessor

use of com.facebook.presto.operator.project.CursorProcessor in project presto by prestodb.

the class FunctionAssertions method compileScanFilterProject.

private static SourceOperatorFactory compileScanFilterProject(SqlFunctionProperties sqlFunctionProperties, Optional<RowExpression> filter, RowExpression projection, ExpressionCompiler compiler) {
    try {
        Supplier<CursorProcessor> cursorProcessor = compiler.compileCursorProcessor(sqlFunctionProperties, filter, ImmutableList.of(projection), SOURCE_ID);
        Supplier<PageProcessor> pageProcessor = compiler.compilePageProcessor(sqlFunctionProperties, filter, ImmutableList.of(projection));
        return new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), SOURCE_ID, PAGE_SOURCE_PROVIDER, cursorProcessor, pageProcessor, new TableHandle(new ConnectorId("test"), new ConnectorTableHandle() {
        }, new ConnectorTransactionHandle() {
        }, Optional.empty()), ImmutableList.of(), ImmutableList.of(projection.getType()), Optional.empty(), new DataSize(0, BYTE), 0);
    } catch (Throwable e) {
        if (e instanceof UncheckedExecutionException) {
            e = e.getCause();
        }
        throw new RuntimeException("Error compiling filter " + filter + ": " + e.getMessage(), e);
    }
}
Also used : CursorProcessor(com.facebook.presto.operator.project.CursorProcessor) UncheckedExecutionException(com.google.common.util.concurrent.UncheckedExecutionException) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) PageProcessor(com.facebook.presto.operator.project.PageProcessor) DataSize(io.airlift.units.DataSize) TableHandle(com.facebook.presto.spi.TableHandle) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) ConnectorId(com.facebook.presto.spi.ConnectorId)

Example 7 with CursorProcessor

use of com.facebook.presto.operator.project.CursorProcessor in project presto by prestodb.

the class TestScanFilterAndProjectOperator method testPageYield.

@Test
public void testPageYield() {
    int totalRows = 1000;
    Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), totalRows, 1);
    DriverContext driverContext = newDriverContext();
    // 20 columns; each column is associated with a function that will force yield per projection
    int totalColumns = 20;
    ImmutableList.Builder<SqlScalarFunction> functions = ImmutableList.builder();
    for (int i = 0; i < totalColumns; i++) {
        functions.add(new GenericLongFunction("page_col" + i, value -> {
            driverContext.getYieldSignal().forceYieldForTesting();
            return value;
        }));
    }
    Metadata metadata = functionAssertions.getMetadata();
    FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager();
    functionAndTypeManager.registerBuiltInFunctions(functions.build());
    // match each column with a projection
    ExpressionCompiler expressionCompiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0));
    ImmutableList.Builder<RowExpression> projections = ImmutableList.builder();
    for (int i = 0; i < totalColumns; i++) {
        projections.add(call("generic_long_page_col", functionAndTypeManager.lookupFunction("generic_long_page_col" + i, fromTypes(BIGINT)), BIGINT, field(0, BIGINT)));
    }
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections.build(), "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections.build(), false, MAX_BATCH_SIZE);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns) -> new FixedPageSource(ImmutableList.of(input)), cursorProcessor, pageProcessor, TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), Optional.empty(), new DataSize(0, BYTE), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit()));
    operator.noMoreSplits();
    // exactly 20 blocks (one for each column) and the PageProcessor will be able to create a Page out of it.
    for (int i = 1; i <= totalRows * totalColumns; i++) {
        driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
        Page page = operator.getOutput();
        if (i == totalColumns) {
            assertNotNull(page);
            assertEquals(page.getPositionCount(), totalRows);
            assertEquals(page.getChannelCount(), totalColumns);
            for (int j = 0; j < totalColumns; j++) {
                assertEquals(toValues(BIGINT, page.getBlock(j)), toValues(BIGINT, input.getBlock(0)));
            }
        } else {
            assertNull(page);
        }
        driverContext.getYieldSignal().reset();
    }
}
Also used : FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) RecordPageSource(com.facebook.presto.spi.RecordPageSource) Page(com.facebook.presto.common.Page) MetadataManager(com.facebook.presto.metadata.MetadataManager) Test(org.testng.annotations.Test) FixedPageSource(com.facebook.presto.spi.FixedPageSource) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) Expressions.constant(com.facebook.presto.sql.relational.Expressions.constant) BlockAssertions(com.facebook.presto.block.BlockAssertions) Expressions.field(com.facebook.presto.sql.relational.Expressions.field) CursorProcessor(com.facebook.presto.operator.project.CursorProcessor) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) EQUAL(com.facebook.presto.common.function.OperatorType.EQUAL) KILOBYTE(io.airlift.units.DataSize.Unit.KILOBYTE) PageFunctionCompiler(com.facebook.presto.sql.gen.PageFunctionCompiler) PageProcessor(com.facebook.presto.operator.project.PageProcessor) PageProjectionWithOutputs(com.facebook.presto.operator.project.PageProjectionWithOutputs) Assert.assertNotNull(org.testng.Assert.assertNotNull) Preconditions.checkState(com.google.common.base.Preconditions.checkState) Threads.daemonThreadsNamed(com.facebook.airlift.concurrent.Threads.daemonThreadsNamed) DataSize(io.airlift.units.DataSize) List(java.util.List) LazyBlockLoader(com.facebook.presto.common.block.LazyBlockLoader) Optional(java.util.Optional) LazyPagePageProjection(com.facebook.presto.operator.project.TestPageProcessor.LazyPagePageProjection) ConnectorId(com.facebook.presto.spi.ConnectorId) RowPagesBuilder.rowPagesBuilder(com.facebook.presto.RowPagesBuilder.rowPagesBuilder) LazyBlock(com.facebook.presto.common.block.LazyBlock) ExpressionCompiler(com.facebook.presto.sql.gen.ExpressionCompiler) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) SequencePageBuilder(com.facebook.presto.SequencePageBuilder) Assert.assertNull(org.testng.Assert.assertNull) Assert.assertEquals(com.facebook.presto.testing.assertions.Assert.assertEquals) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) MAX_BATCH_SIZE(com.facebook.presto.operator.project.PageProcessor.MAX_BATCH_SIZE) PageAssertions.assertPageEquals(com.facebook.presto.operator.PageAssertions.assertPageEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PageRecordSet(com.facebook.presto.operator.index.PageRecordSet) Supplier(java.util.function.Supplier) TypeSignatureProvider.fromTypes(com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes) Expressions.call(com.facebook.presto.sql.relational.Expressions.call) Iterators(com.google.common.collect.Iterators) TestingSplit(com.facebook.presto.testing.TestingSplit) TEST_SESSION(com.facebook.presto.SessionTestUtils.TEST_SESSION) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) TableHandle(com.facebook.presto.spi.TableHandle) ExecutorService(java.util.concurrent.ExecutorService) RowExpression(com.facebook.presto.spi.relation.RowExpression) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) SelectAllFilter(com.facebook.presto.operator.project.TestPageProcessor.SelectAllFilter) Iterator(java.util.Iterator) TestingTaskContext.createTaskContext(com.facebook.presto.testing.TestingTaskContext.createTaskContext) SqlScalarFunction(com.facebook.presto.metadata.SqlScalarFunction) AbstractTestFunctions(com.facebook.presto.operator.scalar.AbstractTestFunctions) OperatorAssertion.toMaterializedResult(com.facebook.presto.operator.OperatorAssertion.toMaterializedResult) TestingTransactionHandle(com.facebook.presto.testing.TestingTransactionHandle) MaterializedResult(com.facebook.presto.testing.MaterializedResult) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) MetadataManager.createTestMetadataManager(com.facebook.presto.metadata.MetadataManager.createTestMetadataManager) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) BlockAssertions.toValues(com.facebook.presto.block.BlockAssertions.toValues) Split(com.facebook.presto.metadata.Split) Assert.assertTrue(org.testng.Assert.assertTrue) Block(com.facebook.presto.common.block.Block) BYTE(io.airlift.units.DataSize.Unit.BYTE) SECONDS(java.util.concurrent.TimeUnit.SECONDS) Metadata(com.facebook.presto.metadata.Metadata) PageFunctionCompiler(com.facebook.presto.sql.gen.PageFunctionCompiler) CursorProcessor(com.facebook.presto.operator.project.CursorProcessor) ImmutableList(com.google.common.collect.ImmutableList) Metadata(com.facebook.presto.metadata.Metadata) Page(com.facebook.presto.common.Page) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) PageProcessor(com.facebook.presto.operator.project.PageProcessor) FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) DataSize(io.airlift.units.DataSize) ConnectorId(com.facebook.presto.spi.ConnectorId) RowExpression(com.facebook.presto.spi.relation.RowExpression) FixedPageSource(com.facebook.presto.spi.FixedPageSource) SqlScalarFunction(com.facebook.presto.metadata.SqlScalarFunction) ExpressionCompiler(com.facebook.presto.sql.gen.ExpressionCompiler) TestingSplit(com.facebook.presto.testing.TestingSplit) Split(com.facebook.presto.metadata.Split) Test(org.testng.annotations.Test)

Example 8 with CursorProcessor

use of com.facebook.presto.operator.project.CursorProcessor in project presto by prestodb.

the class TestScanFilterAndProjectOperator method testRecordCursorYield.

@Test
public void testRecordCursorYield() {
    // create a generic long function that yields for projection on every row
    // verify we will yield #row times totally
    // create a table with 15 rows
    int length = 15;
    Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), length, 0);
    DriverContext driverContext = newDriverContext();
    // set up generic long function with a callback to force yield
    Metadata metadata = functionAssertions.getMetadata();
    FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager();
    functionAndTypeManager.registerBuiltInFunctions(ImmutableList.of(new GenericLongFunction("record_cursor", value -> {
        driverContext.getYieldSignal().forceYieldForTesting();
        return value;
    })));
    ExpressionCompiler expressionCompiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0));
    List<RowExpression> projections = ImmutableList.of(call("generic_long_record_cursor", functionAndTypeManager.lookupFunction("generic_long_record_cursor", fromTypes(BIGINT)), BIGINT, field(0, BIGINT)));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections, "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(driverContext.getSession().getSqlFunctionProperties(), Optional.empty(), projections);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns) -> new RecordPageSource(new PageRecordSet(ImmutableList.of(BIGINT), input)), cursorProcessor, pageProcessor, TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), Optional.empty(), new DataSize(0, BYTE), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit()));
    operator.noMoreSplits();
    // start driver; get null value due to yield for the first 15 times
    for (int i = 0; i < length; i++) {
        driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
        assertNull(operator.getOutput());
        driverContext.getYieldSignal().reset();
    }
    // the 16th yield is not going to prevent the operator from producing a page
    driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
    Page output = operator.getOutput();
    driverContext.getYieldSignal().reset();
    assertNotNull(output);
    assertEquals(toValues(BIGINT, output.getBlock(0)), toValues(BIGINT, input.getBlock(0)));
}
Also used : PageFunctionCompiler(com.facebook.presto.sql.gen.PageFunctionCompiler) CursorProcessor(com.facebook.presto.operator.project.CursorProcessor) Metadata(com.facebook.presto.metadata.Metadata) RowExpression(com.facebook.presto.spi.relation.RowExpression) Page(com.facebook.presto.common.Page) PageRecordSet(com.facebook.presto.operator.index.PageRecordSet) RecordPageSource(com.facebook.presto.spi.RecordPageSource) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) PageProcessor(com.facebook.presto.operator.project.PageProcessor) FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) DataSize(io.airlift.units.DataSize) ExpressionCompiler(com.facebook.presto.sql.gen.ExpressionCompiler) TestingSplit(com.facebook.presto.testing.TestingSplit) Split(com.facebook.presto.metadata.Split) ConnectorId(com.facebook.presto.spi.ConnectorId) Test(org.testng.annotations.Test)

Example 9 with CursorProcessor

use of com.facebook.presto.operator.project.CursorProcessor in project presto by prestodb.

the class TestScanFilterAndProjectOperator method testPageSourceMergeOutput.

@Test
public void testPageSourceMergeOutput() {
    List<Page> input = rowPagesBuilder(BIGINT).addSequencePage(100, 0).addSequencePage(100, 0).addSequencePage(100, 0).addSequencePage(100, 0).build();
    RowExpression filter = call(EQUAL.name(), createTestMetadataManager().getFunctionAndTypeManager().resolveOperator(EQUAL, fromTypes(BIGINT, BIGINT)), BOOLEAN, field(0, BIGINT), constant(10L, BIGINT));
    List<RowExpression> projections = ImmutableList.of(field(0, BIGINT));
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), projections, "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(TEST_SESSION.getSqlFunctionProperties(), Optional.of(filter), projections);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns) -> new FixedPageSource(input), cursorProcessor, pageProcessor, TESTING_TABLE_HANDLE, ImmutableList.of(), ImmutableList.of(BIGINT), Optional.empty(), new DataSize(64, KILOBYTE), 2);
    SourceOperator operator = factory.createOperator(newDriverContext());
    operator.addSplit(new Split(new ConnectorId("test"), TestingTransactionHandle.create(), TestingSplit.createLocalSplit()));
    operator.noMoreSplits();
    List<Page> actual = toPages(operator);
    assertEquals(actual.size(), 1);
    List<Page> expected = rowPagesBuilder(BIGINT).row(10L).row(10L).row(10L).row(10L).build();
    assertPageEquals(ImmutableList.of(BIGINT), actual.get(0), expected.get(0));
}
Also used : CursorProcessor(com.facebook.presto.operator.project.CursorProcessor) RowExpression(com.facebook.presto.spi.relation.RowExpression) Page(com.facebook.presto.common.Page) FixedPageSource(com.facebook.presto.spi.FixedPageSource) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) PageProcessor(com.facebook.presto.operator.project.PageProcessor) DataSize(io.airlift.units.DataSize) TestingSplit(com.facebook.presto.testing.TestingSplit) Split(com.facebook.presto.metadata.Split) ConnectorId(com.facebook.presto.spi.ConnectorId) Test(org.testng.annotations.Test)

Aggregations

CursorProcessor (com.facebook.presto.operator.project.CursorProcessor)9 Page (com.facebook.presto.common.Page)8 PageProcessor (com.facebook.presto.operator.project.PageProcessor)8 ConnectorId (com.facebook.presto.spi.ConnectorId)8 PlanNodeId (com.facebook.presto.spi.plan.PlanNodeId)8 RowExpression (com.facebook.presto.spi.relation.RowExpression)8 DataSize (io.airlift.units.DataSize)8 Test (org.testng.annotations.Test)8 Split (com.facebook.presto.metadata.Split)7 TestingSplit (com.facebook.presto.testing.TestingSplit)7 OperatorAssertion.toMaterializedResult (com.facebook.presto.operator.OperatorAssertion.toMaterializedResult)5 PageRecordSet (com.facebook.presto.operator.index.PageRecordSet)5 FixedPageSource (com.facebook.presto.spi.FixedPageSource)4 RecordPageSource (com.facebook.presto.spi.RecordPageSource)4 MaterializedResult (com.facebook.presto.testing.MaterializedResult)4 Block (com.facebook.presto.common.block.Block)3 LazyBlock (com.facebook.presto.common.block.LazyBlock)3 FunctionAndTypeManager (com.facebook.presto.metadata.FunctionAndTypeManager)3 Metadata (com.facebook.presto.metadata.Metadata)3 Threads.daemonThreadsNamed (com.facebook.airlift.concurrent.Threads.daemonThreadsNamed)2