Examples with LazyBlock io.trino.spi.block.LazyBlock used on opensource projects

Example 1 with LazyBlock

use of io.trino.spi.block.LazyBlock in project trino by trinodb.

the class ParquetPageSource method getNextPage.

@Override
public Page getNextPage() {
    try {
        batchId++;
        int batchSize = parquetReader.nextBatch();
        if (closed || batchSize <= 0) {
            close();
            return null;
        }
        completedPositions += batchSize;
        Block[] blocks = new Block[fields.size()];
        for (int column = 0; column < blocks.length; column++) {
            if (isIndexColumn(column)) {
                blocks[column] = getRowIndexColumn(parquetReader.lastBatchStartRow(), batchSize);
            } else {
                Type type = types.get(column);
                blocks[column] = fields.get(column).<Block>map(field -> new LazyBlock(batchSize, new ParquetBlockLoader(field))).orElseGet(() -> RunLengthEncodedBlock.create(type, null, batchSize));
            }
        }
        return new Page(batchSize, blocks);
    } catch (TrinoException e) {
        closeAllSuppress(e, this);
        throw e;
    } catch (RuntimeException e) {
        closeAllSuppress(e, this);
        throw new TrinoException(HIVE_CURSOR_ERROR, e);
    }
}

Example 2 with LazyBlock

use of io.trino.spi.block.LazyBlock in project trino by trinodb.

the class TestPageUtils method testRecordMaterializedBytes.

@Test
public void testRecordMaterializedBytes() {
    Block first = createIntsBlock(1, 2, 3);
    LazyBlock second = lazyWrapper(first);
    LazyBlock third = lazyWrapper(first);
    Page page = new Page(3, first, second, third);
    second.getLoadedBlock();
    AtomicLong sizeInBytes = new AtomicLong();
    recordMaterializedBytes(page, sizeInBytes::getAndAdd);
    assertEquals(sizeInBytes.get(), first.getSizeInBytes() * 2);
    page.getBlock(2).getLoadedBlock();
    assertEquals(sizeInBytes.get(), first.getSizeInBytes() * 3);
}

Example 3 with LazyBlock

use of io.trino.spi.block.LazyBlock in project trino by trinodb.

the class TestPageProcessor method testSelectAllFilterLazyBlock.

@Test
public void testSelectAllFilterLazyBlock() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new InputPageProjection(0, BIGINT), new InputPageProjection(1, BIGINT)), OptionalInt.of(100));
    LazyBlock inputFilterBlock = lazyWrapper(createLongSequenceBlock(0, 100));
    LazyBlock inputProjectionBlock = lazyWrapper(createLongSequenceBlock(100, 200));
    Page inputPage = new Page(inputFilterBlock, inputProjectionBlock);
    Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, new DriverYieldSignal(), newSimpleAggregatedMemoryContext(), inputPage, true);
    List<Optional<Page>> outputPages = ImmutableList.copyOf(output);
    assertTrue(inputFilterBlock.isLoaded());
    assertFalse(inputProjectionBlock.isLoaded());
    assertEquals(outputPages.size(), 1);
    inputFilterBlock = lazyWrapper(createLongSequenceBlock(0, 200));
    inputProjectionBlock = lazyWrapper(createLongSequenceBlock(100, 300));
    inputPage = new Page(inputFilterBlock, inputProjectionBlock);
    // batch size should increase because filter block was materialized
    output = processAndAssertRetainedPageSize(pageProcessor, new DriverYieldSignal(), newSimpleAggregatedMemoryContext(), inputPage, true);
    outputPages = ImmutableList.copyOf(output);
    assertEquals(outputPages.size(), 1);
    assertTrue(inputFilterBlock.isLoaded());
    assertFalse(inputProjectionBlock.isLoaded());
    assertPageEquals(ImmutableList.of(BIGINT, BIGINT), outputPages.get(0).get(), new Page(createLongSequenceBlock(0, 200), createLongSequenceBlock(100, 300)));
    assertTrue(inputProjectionBlock.isLoaded());
}

Example 4 with LazyBlock

use of io.trino.spi.block.LazyBlock in project trino by trinodb.

the class TestPage method testGetLoadedPage.

@Test
public void testGetLoadedPage() {
    int entries = 10;
    BlockBuilder blockBuilder = BIGINT.createBlockBuilder(null, entries);
    for (int i = 0; i < entries; i++) {
        BIGINT.writeLong(blockBuilder, i);
    }
    Block block = blockBuilder.build();
    LazyBlock lazyBlock = lazyWrapper(block);
    Page page = new Page(lazyBlock);
    long lazyPageRetainedSize = Page.INSTANCE_SIZE + sizeOf(new Block[] { block }) + lazyBlock.getRetainedSizeInBytes();
    assertEquals(page.getRetainedSizeInBytes(), lazyPageRetainedSize);
    Page loadedPage = page.getLoadedPage();
    // Retained size of page remains the same
    assertEquals(page.getRetainedSizeInBytes(), lazyPageRetainedSize);
    long loadedPageRetainedSize = Page.INSTANCE_SIZE + sizeOf(new Block[] { block }) + block.getRetainedSizeInBytes();
    // Retained size of loaded page depends on the loaded block
    assertEquals(loadedPage.getRetainedSizeInBytes(), loadedPageRetainedSize);
    lazyBlock = lazyWrapper(block);
    page = new Page(lazyBlock);
    assertEquals(page.getRetainedSizeInBytes(), lazyPageRetainedSize);
    loadedPage = page.getLoadedPage(new int[] { 0 }, new int[] { 0 });
    // Retained size of page is updated based on loaded block
    assertEquals(page.getRetainedSizeInBytes(), loadedPageRetainedSize);
    assertEquals(loadedPage.getRetainedSizeInBytes(), loadedPageRetainedSize);
}

Example 5 with LazyBlock

use of io.trino.spi.block.LazyBlock in project trino by trinodb.

the class TestHashJoinOperator method testInnerJoinLoadsPagesInOrder.

@Test
public void testInnerJoinLoadsPagesInOrder() {
    TaskContext taskContext = createTaskContext();
    // build factory
    List<Type> buildTypes = ImmutableList.of(VARCHAR);
    RowPagesBuilder buildPages = rowPagesBuilder(false, Ints.asList(0), buildTypes);
    for (int i = 0; i < 100_000; ++i) {
        buildPages.row("a");
    }
    BuildSideSetup buildSideSetup = setupBuildSide(nodePartitioningManager, false, taskContext, buildPages, Optional.empty(), false, SINGLE_STREAM_SPILLER_FACTORY);
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
    // probe factory
    List<Type> probeTypes = ImmutableList.of(VARCHAR, INTEGER, INTEGER);
    RowPagesBuilder probePages = rowPagesBuilder(false, Ints.asList(0), probeTypes);
    probePages.row("a", 1L, 2L);
    WorkProcessorOperatorFactory joinOperatorFactory = (WorkProcessorOperatorFactory) innerJoinOperatorFactory(operatorFactories, lookupSourceFactory, probePages, PARTITIONING_SPILLER_FACTORY, false);
    // build drivers and operators
    instantiateBuildDrivers(buildSideSetup, taskContext);
    buildLookupSource(executor, buildSideSetup);
    Page probePage = getOnlyElement(probePages.build());
    AtomicInteger totalProbePages = new AtomicInteger();
    WorkProcessor<Page> inputPages = WorkProcessor.create(() -> {
        int probePageNumber = totalProbePages.incrementAndGet();
        if (probePageNumber == 5) {
            return finished();
        }
        return ofResult(new Page(1, probePage.getBlock(0), // this block should not be loaded by join operator as it's not being used by join
        new LazyBlock(1, () -> probePage.getBlock(1)), // and outputting current probe page
        new LazyBlock(1, () -> {
            // when loaded this block should be the latest one
            assertEquals(probePageNumber, totalProbePages.get());
            return probePage.getBlock(2);
        })));
    });
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    OperatorContext operatorContext = driverContext.addOperatorContext(joinOperatorFactory.getOperatorId(), joinOperatorFactory.getPlanNodeId(), joinOperatorFactory.getOperatorType());
    WorkProcessorOperator joinOperator = joinOperatorFactory.create(new ProcessorContext(taskContext.getSession(), taskContext.getTaskMemoryContext(), operatorContext), inputPages);
    WorkProcessor<Page> outputPages = joinOperator.getOutputPages();
    int totalOutputPages = 0;
    for (int i = 0; i < 1_000_000; ++i) {
        if (!outputPages.process()) {
            // allow join to progress
            driverContext.getYieldSignal().resetYieldForTesting();
            continue;
        }
        if (outputPages.isFinished()) {
            break;
        }
        Page page = outputPages.getResult();
        totalOutputPages++;
        assertFalse(page.getBlock(1).isLoaded());
        page.getBlock(2).getLoadedBlock();
        // yield to enforce more complex execution
        driverContext.getYieldSignal().forceYieldForTesting();
    }
    // make sure that multiple pages were produced for some probe pages
    assertTrue(totalOutputPages > totalProbePages.get());
    assertTrue(outputPages.isFinished());
}