Examples with LazyBlock com.facebook.presto.common.block.LazyBlock used on opensource projects

Example 6 with LazyBlock

use of com.facebook.presto.common.block.LazyBlock in project presto by prestodb.

the class TestPageProcessor method testProjectLazyLoad.

@Test
public void testProjectLazyLoad() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new PageProjectionWithOutputs(new LazyPagePageProjection(), new int[] { 0 })), OptionalInt.of(MAX_BATCH_SIZE));
    // if channel 1 is loaded, test will fail
    Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> {
        throw new AssertionError("Lazy block should not be loaded");
    }));
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName());
    Iterator<Optional<Page>> output = pageProcessor.process(SESSION.getSqlFunctionProperties(), new DriverYieldSignal(), memoryContext, inputPage);
    List<Optional<Page>> outputPages = ImmutableList.copyOf(output);
    assertEquals(outputPages.size(), 1);
    assertPageEquals(ImmutableList.of(BIGINT), outputPages.get(0).orElse(null), new Page(createLongSequenceBlock(0, 100)));
}

Example 7 with LazyBlock

use of com.facebook.presto.common.block.LazyBlock in project presto by prestodb.

the class TestPageProcessor method testSelectNoneFilterLazyLoad.

@Test
public void testSelectNoneFilterLazyLoad() {
    PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectNoneFilter()), ImmutableList.of(createInputPageProjectionWithOutputs(1, BIGINT, 0)));
    // if channel 1 is loaded, test will fail
    Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> {
        throw new AssertionError("Lazy block should not be loaded");
    }));
    LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName());
    Iterator<Optional<Page>> output = pageProcessor.process(SESSION.getSqlFunctionProperties(), new DriverYieldSignal(), memoryContext, inputPage);
    assertEquals(memoryContext.getBytes(), 0);
    List<Optional<Page>> outputPages = ImmutableList.copyOf(output);
    assertEquals(outputPages.size(), 0);
}

Example 8 with LazyBlock

use of com.facebook.presto.common.block.LazyBlock in project presto by prestodb.

the class ParquetPageSource method getNextPage.

@Override
public Page getNextPage() {
    try {
        batchId++;
        int batchSize = parquetReader.nextBatch();
        if (closed || batchSize <= 0) {
            close();
            return null;
        }
        completedPositions += batchSize;
        Block[] blocks = new Block[fields.size()];
        for (int fieldId = 0; fieldId < blocks.length; fieldId++) {
            Optional<Field> field = fields.get(fieldId);
            if (field.isPresent()) {
                blocks[fieldId] = new LazyBlock(batchSize, new ParquetBlockLoader(field.get()));
            } else {
                blocks[fieldId] = RunLengthEncodedBlock.create(types.get(fieldId), null, batchSize);
            }
        }
        return new Page(batchSize, blocks);
    } catch (PrestoException e) {
        closeWithSuppression(e);
        throw e;
    } catch (RuntimeException e) {
        closeWithSuppression(e);
        throw new PrestoException(HIVE_CURSOR_ERROR, e);
    }
}

Example 9 with LazyBlock

use of com.facebook.presto.common.block.LazyBlock in project presto by prestodb.

the class OrcSelectiveRecordReader method getNextPage.

public Page getNextPage() throws IOException {
    if (constantFilterIsFalse) {
        return null;
    }
    int batchSize = prepareNextBatch();
    if (batchSize < 0) {
        return null;
    }
    readPositions += batchSize;
    initializePositions(batchSize);
    int[] positionsToRead = this.positions;
    int positionCount = batchSize;
    if (filterFunctionWithoutInput.isPresent()) {
        positionCount = applyFilterFunctionWithNoInputs(positionCount);
        if (positionCount == 0) {
            batchRead(batchSize);
            return EMPTY_PAGE;
        }
        positionsToRead = outputPositions;
    }
    if (!filterFunctionsWithConstantInputs.isEmpty()) {
        positionCount = applyFilterFunctions(filterFunctionsWithConstantInputs, filterFunctionConstantInputs, positionsToRead, positionCount);
        if (positionCount == 0) {
            batchRead(batchSize);
            return EMPTY_PAGE;
        }
        positionsToRead = outputPositions;
    }
    int offset = getNextRowInGroup();
    if (reorderFilters && offset >= MAX_BATCH_SIZE) {
        reorderFiltersIfNeeded();
    }
    for (int i = 0; i < streamReaderOrder.length; i++) {
        int columnIndex = streamReaderOrder[i];
        if (!hasAnyFilter(columnIndex)) {
            break;
        }
        SelectiveStreamReader streamReader = getStreamReader(columnIndex);
        positionCount = streamReader.read(offset, positionsToRead, positionCount);
        if (positionCount == 0) {
            break;
        }
        positionsToRead = streamReader.getReadPositions();
        verify(positionCount == 1 || positionsToRead[positionCount - 1] - positionsToRead[0] >= positionCount - 1, "positions must monotonically increase");
        if (filterFunctionsOrder[i] != null) {
            positionCount = applyFilterFunctions(filterFunctionsOrder[i], filterFunctionInputs[i], positionsToRead, positionCount);
            if (positionCount == 0) {
                break;
            }
            positionsToRead = outputPositions;
        }
    }
    localMemoryContext.setBytes(getSelfRetainedSizeInBytes());
    batchRead(batchSize);
    if (positionCount == 0) {
        return EMPTY_PAGE;
    }
    if (constantFilterError != null) {
        throw constantFilterError;
    }
    for (int i = 0; i < positionCount; i++) {
        if (errors[positionsToRead[i]] != null) {
            throw errors[positionsToRead[i]];
        }
    }
    for (SelectiveStreamReader reader : getStreamReaders()) {
        if (reader != null) {
            reader.throwAnyError(positionsToRead, positionCount);
        }
    }
    Block[] blocks = new Block[appendRowNumber ? outputColumns.size() + 1 : outputColumns.size()];
    for (int i = 0; i < outputColumns.size(); i++) {
        int columnIndex = outputColumns.get(i);
        if (constantValues[columnIndex] != null) {
            blocks[i] = RunLengthEncodedBlock.create(columnTypes.get(columnIndex), constantValues[columnIndex] == NULL_MARKER ? null : constantValues[columnIndex], positionCount);
        } else if (!hasAnyFilter(columnIndex)) {
            blocks[i] = new LazyBlock(positionCount, new OrcBlockLoader(columnIndex, offset, positionsToRead, positionCount));
        } else {
            Block block = getStreamReader(columnIndex).getBlock(positionsToRead, positionCount);
            updateMaxCombinedBytesPerRow(hiveColumnIndices[columnIndex], block);
            if (coercers[columnIndex] != null) {
                block = coercers[columnIndex].apply(block);
            }
            blocks[i] = block;
        }
    }
    if (appendRowNumber) {
        blocks[outputColumns.size()] = createRowNumbersBlock(positionsToRead, positionCount, this.getFilePosition());
    }
    Page page = new Page(positionCount, blocks);
    validateWritePageChecksum(page);
    return page;
}

Example 10 with LazyBlock

use of com.facebook.presto.common.block.LazyBlock in project presto by prestodb.

the class OrcBatchPageSource method getNextPage.

@Override
public Page getNextPage() {
    try {
        batchId++;
        int batchSize = recordReader.nextBatch();
        if (batchSize <= 0) {
            close();
            return null;
        }
        completedPositions += batchSize;
        Block[] blocks = new Block[hiveColumnIndexes.length];
        for (int fieldId = 0; fieldId < blocks.length; fieldId++) {
            if (constantBlocks[fieldId] != null) {
                blocks[fieldId] = constantBlocks[fieldId].getRegion(0, batchSize);
            } else {
                blocks[fieldId] = new LazyBlock(batchSize, new OrcBlockLoader(hiveColumnIndexes[fieldId]));
            }
        }
        return new Page(batchSize, blocks);
    } catch (PrestoException e) {
        closeWithSuppression(e);
        throw e;
    } catch (OrcCorruptionException e) {
        closeWithSuppression(e);
        throw new PrestoException(HIVE_BAD_DATA, e);
    } catch (IOException | RuntimeException e) {
        closeWithSuppression(e);
        throw new PrestoException(HIVE_CURSOR_ERROR, format("Failed to read ORC file: %s", orcDataSource.getId()), e);
    }
}