Examples with FixedPageSource io.trino.spi.connector.FixedPageSource used on opensource projects

Example 1 with FixedPageSource

use of io.trino.spi.connector.FixedPageSource in project trino by trinodb.

the class RuleStatsSystemTable method pageSource.

@Override
public ConnectorPageSource pageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) {
    checkState(ruleStatsRecorder.isPresent(), "Rule stats system table can return results only on coordinator");
    Map<Class<?>, RuleStats> ruleStats = ruleStatsRecorder.get().getStats();
    int positionCount = ruleStats.size();
    Map<String, BlockBuilder> blockBuilders = ruleStatsTable.getColumns().stream().collect(toImmutableMap(ColumnMetadata::getName, column -> column.getType().createBlockBuilder(null, positionCount)));
    for (Map.Entry<Class<?>, RuleStats> entry : ruleStats.entrySet()) {
        RuleStats stats = entry.getValue();
        VARCHAR.writeString(blockBuilders.get("rule_name"), entry.getKey().getSimpleName());
        BIGINT.writeLong(blockBuilders.get("invocations"), stats.getInvocations());
        BIGINT.writeLong(blockBuilders.get("matches"), stats.getHits());
        BIGINT.writeLong(blockBuilders.get("failures"), stats.getFailures());
        DOUBLE.writeDouble(blockBuilders.get("average_time"), stats.getTime().getAvg());
        BlockBuilder mapWriter = blockBuilders.get("time_distribution_percentiles").beginBlockEntry();
        for (Map.Entry<Double, Double> percentile : stats.getTime().getPercentiles().entrySet()) {
            DOUBLE.writeDouble(mapWriter, percentile.getKey());
            DOUBLE.writeDouble(mapWriter, percentile.getValue());
        }
        blockBuilders.get("time_distribution_percentiles").closeEntry();
    }
    Block[] blocks = ruleStatsTable.getColumns().stream().map(column -> blockBuilders.get(column.getName()).build()).toArray(Block[]::new);
    return new FixedPageSource(ImmutableList.of(new Page(positionCount, blocks)));
}

Example 2 with FixedPageSource

use of io.trino.spi.connector.FixedPageSource in project trino by trinodb.

the class TestScanFilterAndProjectOperator method testPageSource.

@Test
public void testPageSource() {
    Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(VARCHAR), 10_000, 0);
    DriverContext driverContext = newDriverContext();
    List<RowExpression> projections = ImmutableList.of(field(0, VARCHAR));
    Supplier<CursorProcessor> cursorProcessor = functionAssertions.getExpressionCompiler().compileCursorProcessor(Optional.empty(), projections, "key");
    Supplier<PageProcessor> pageProcessor = functionAssertions.getExpressionCompiler().compilePageProcessor(Optional.empty(), projections);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(ImmutableList.of(input)), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), DynamicFilter.EMPTY, ImmutableList.of(VARCHAR), DataSize.ofBytes(0), 0);
    SourceOperator operator = factory.createOperator(driverContext);
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    MaterializedResult expected = toMaterializedResult(driverContext.getSession(), ImmutableList.of(VARCHAR), ImmutableList.of(input));
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), ImmutableList.of(VARCHAR), toPages(operator));
    assertEquals(actual.getRowCount(), expected.getRowCount());
    assertEquals(actual, expected);
}

Example 3 with FixedPageSource

use of io.trino.spi.connector.FixedPageSource in project trino by trinodb.

the class TestScanFilterAndProjectOperator method testPageSourceMergeOutput.

@Test
public void testPageSourceMergeOutput() {
    List<Page> input = rowPagesBuilder(BIGINT).addSequencePage(100, 0).addSequencePage(100, 0).addSequencePage(100, 0).addSequencePage(100, 0).build();
    RowExpression filter = call(functionAssertions.getTestingFunctionResolution().resolveOperator(EQUAL, ImmutableList.of(BIGINT, BIGINT)), field(0, BIGINT), constant(10L, BIGINT));
    List<RowExpression> projections = ImmutableList.of(field(0, BIGINT));
    Supplier<CursorProcessor> cursorProcessor = functionAssertions.getExpressionCompiler().compileCursorProcessor(Optional.of(filter), projections, "key");
    Supplier<PageProcessor> pageProcessor = functionAssertions.getExpressionCompiler().compilePageProcessor(Optional.of(filter), projections);
    ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(input), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), DynamicFilter.EMPTY, ImmutableList.of(BIGINT), DataSize.of(64, KILOBYTE), 2);
    SourceOperator operator = factory.createOperator(newDriverContext());
    operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
    operator.noMoreSplits();
    List<Page> actual = toPages(operator);
    assertEquals(actual.size(), 1);
    List<Page> expected = rowPagesBuilder(BIGINT).row(10L).row(10L).row(10L).row(10L).build();
    assertPageEquals(ImmutableList.of(BIGINT), actual.get(0), expected.get(0));
}

Example 4 with FixedPageSource

use of io.trino.spi.connector.FixedPageSource in project trino by trinodb.

the class TestMemoryBlocking method testTableScanMemoryBlocking.

@Test
public void testTableScanMemoryBlocking() {
    PlanNodeId sourceId = new PlanNodeId("source");
    List<Type> types = ImmutableList.of(VARCHAR);
    TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(1, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).build()), TEST_TABLE_HANDLE, ImmutableList.of(), DynamicFilter.EMPTY);
    PageConsumerOperator sink = createSinkOperator(types);
    Driver driver = Driver.createDriver(driverContext, source, sink);
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    Split testSplit = new Split(new CatalogName("test"), new TestSplit(), Lifespan.taskWide());
    driver.updateSplitAssignment(new SplitAssignment(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, testSplit)), true));
    ListenableFuture<Void> blocked = driver.processFor(new Duration(1, NANOSECONDS));
    // the driver shouldn't block in the first call as it will be able to move a page between source and the sink operator
    // but the operator should be blocked
    assertTrue(blocked.isDone());
    assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
    // and they should stay blocked until more memory becomes available
    for (int i = 0; i < 10; i++) {
        blocked = driver.processFor(new Duration(1, NANOSECONDS));
        assertFalse(blocked.isDone());
        assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
    }
    // free up some memory
    memoryPool.free(QUERY_ID, "test", memoryPool.getReservedBytes());
    // the operator should be unblocked
    assertTrue(source.getOperatorContext().isWaitingForMemory().isDone());
    // the driver shouldn't be blocked
    blocked = driver.processFor(new Duration(1, NANOSECONDS));
    assertTrue(blocked.isDone());
}

Example 5 with FixedPageSource

use of io.trino.spi.connector.FixedPageSource in project trino by trinodb.

the class NodesSystemTable method pageSource.

@Override
public ConnectorPageSource pageSource(ConnectorTransactionHandle transaction, ConnectorSession session, TupleDomain<Integer> constraint) {
    Set<ElasticsearchNode> nodes = client.getNodes();
    BlockBuilder nodeId = VARCHAR.createBlockBuilder(null, nodes.size());
    BlockBuilder prestoAddress = VARCHAR.createBlockBuilder(null, nodes.size());
    BlockBuilder elasticsearchNodeId = VARCHAR.createBlockBuilder(null, nodes.size());
    BlockBuilder elasticsearchAddress = VARCHAR.createBlockBuilder(null, nodes.size());
    for (ElasticsearchNode node : nodes) {
        VARCHAR.writeString(nodeId, currentNode.getNodeIdentifier());
        VARCHAR.writeString(prestoAddress, currentNode.getHostAndPort().toString());
        VARCHAR.writeString(elasticsearchNodeId, node.getId());
        if (node.getAddress().isPresent()) {
            VARCHAR.writeString(elasticsearchAddress, node.getAddress().get());
        } else {
            elasticsearchAddress.appendNull();
        }
    }
    return new FixedPageSource(ImmutableList.of(new Page(nodeId.build(), prestoAddress.build(), elasticsearchNodeId.build(), elasticsearchAddress.build())));
}