Examples with FixedPageSource io.trino.spi.connector.FixedPageSource used on opensource projects

Example 6 with FixedPageSource

use of io.trino.spi.connector.FixedPageSource in project trino by trinodb.

the class MemoryPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table, List<ColumnHandle> columns, DynamicFilter dynamicFilter) {
    MemorySplit memorySplit = (MemorySplit) split;
    long tableId = memorySplit.getTable();
    int partNumber = memorySplit.getPartNumber();
    int totalParts = memorySplit.getTotalPartsPerWorker();
    long expectedRows = memorySplit.getExpectedRows();
    MemoryTableHandle memoryTable = (MemoryTableHandle) table;
    OptionalDouble sampleRatio = memoryTable.getSampleRatio();
    List<Integer> columnIndexes = columns.stream().map(MemoryColumnHandle.class::cast).map(MemoryColumnHandle::getColumnIndex).collect(toList());
    List<Page> pages = pagesStore.getPages(tableId, partNumber, totalParts, columnIndexes, expectedRows, memorySplit.getLimit(), sampleRatio);
    return new DynamicFilteringPageSource(new FixedPageSource(pages), columns, dynamicFilter, enableLazyDynamicFiltering);
}

Example 7 with FixedPageSource

use of io.trino.spi.connector.FixedPageSource in project trino by trinodb.

the class TestScanFilterAndProjectOperator method testPageYield.

@Test
public void testPageYield() {
    int totalRows = 1000;
    Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), totalRows, 1);
    DriverContext driverContext = newDriverContext();
    // 20 columns; each column is associated with a function that will force yield per projection
    int totalColumns = 20;
    ImmutableList.Builder<SqlScalarFunction> functions = ImmutableList.builder();
    for (int i = 0; i < totalColumns; i++) {
        functions.add(new GenericLongFunction("page_col" + i, value -> {
            driverContext.getYieldSignal().forceYieldForTesting();
            return value;
        }));
    }
    functionAssertions.addFunctions(new InternalFunctionBundle(functions.build()));
    // match each column with a projection
    ExpressionCompiler expressionCompiler = new ExpressionCompiler(functionAssertions.getFunctionManager(), new PageFunctionCompiler(functionAssertions.getFunctionManager(), 0));
    ImmutableList.Builder<RowExpression> projections = ImmutableList.builder();
    for (int i = 0; i < totalColumns; i++) {
        projections.add(call(functionAssertions.getMetadata().resolveFunction(session, QualifiedName.of("generic_long_page_col" + i), fromTypes(BIGINT)), field(0, BIGINT)));
    }
    Supplier<CursorProcessor> cursorProcessor = expressionCompiler.compileCursorProcessor(Optional.empty(), projections.build(), "key");
    Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(Optional.empty(), projections.build(), MAX_BATCH_SIZE);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(ImmutableList.of(input)), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), DynamicFilter.EMPTY, ImmutableList.of(BIGINT), DataSize.ofBytes(0), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    // exactly 20 blocks (one for each column) and the PageProcessor will be able to create a Page out of it.
    for (int i = 1; i <= totalRows * totalColumns; i++) {
        driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
        Page page = operator.getOutput();
        if (i == totalColumns) {
            assertNotNull(page);
            assertEquals(page.getPositionCount(), totalRows);
            assertEquals(page.getChannelCount(), totalColumns);
            for (int j = 0; j < totalColumns; j++) {
                assertEquals(toValues(BIGINT, page.getBlock(j)), toValues(BIGINT, input.getBlock(0)));
            }
        } else {
            assertNull(page);
        }
        driverContext.getYieldSignal().reset();
    }
}

Example 8 with FixedPageSource

use of io.trino.spi.connector.FixedPageSource in project trino by trinodb.

the class TestDriver method testMemoryRevocationRace.

@Test
public void testMemoryRevocationRace() {
    List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    TableScanOperator source = new AlwaysBlockedMemoryRevokingTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "scan"), new PlanNodeId("source"), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of());
    Driver driver = Driver.createDriver(driverContext, source, createSinkOperator(types));
    // the table scan operator will request memory revocation with requestMemoryRevoking()
    // while the driver is still not done with the processFor() method and before it moves to
    // updateDriverBlockedFuture() method.
    assertTrue(driver.processFor(new Duration(100, TimeUnit.MILLISECONDS)).isDone());
}

Example 9 with FixedPageSource

use of io.trino.spi.connector.FixedPageSource in project trino by trinodb.

the class TestDriver method testBrokenOperatorAddSource.

@Test
public void testBrokenOperatorAddSource() throws Exception {
    PlanNodeId sourceId = new PlanNodeId("source");
    List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    // create a table scan operator that does not block, which will cause the driver loop to busy wait
    TableScanOperator source = new NotBlockedTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of());
    BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"));
    Driver driver = Driver.createDriver(driverContext, source, brokenOperator);
    // block thread in operator processing
    Future<Boolean> driverProcessFor = executor.submit(() -> driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    brokenOperator.waitForLocked();
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.updateSplitAssignment(new SplitAssignment(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.close();
    assertTrue(driver.isFinished());
    assertThatThrownBy(() -> driverProcessFor.get(1, TimeUnit.SECONDS)).isInstanceOf(ExecutionException.class).hasCause(new TrinoException(GENERIC_INTERNAL_ERROR, "Driver was interrupted"));
}

Example 10 with FixedPageSource

use of io.trino.spi.connector.FixedPageSource in project trino by trinodb.

the class TestDriver method testAddSourceFinish.

@Test
public void testAddSourceFinish() {
    PlanNodeId sourceId = new PlanNodeId("source");
    List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of(), DynamicFilter.EMPTY);
    PageConsumerOperator sink = createSinkOperator(types);
    Driver driver = Driver.createDriver(driverContext, source, sink);
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    assertFalse(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.updateSplitAssignment(new SplitAssignment(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
    assertFalse(driver.isFinished());
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertTrue(driver.isFinished());
    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
}