Examples with RunLengthEncodedBlock io.prestosql.spi.block.RunLengthEncodedBlock used on opensource projects

Example 11 with RunLengthEncodedBlock

use of io.prestosql.spi.block.RunLengthEncodedBlock in project hetu-core by openlookeng.

the class TestRunLengthEncodedBlock method testEstimatedDataSizeForStats.

@Test
public void testEstimatedDataSizeForStats() {
    int positionCount = 10;
    Slice expectedValue = createExpectedValue(5);
    Block block = new RunLengthEncodedBlock(createSingleValueBlock(expectedValue), positionCount);
    for (int postition = 0; postition < positionCount; postition++) {
        assertEquals(block.getEstimatedDataSizeForStats(postition), expectedValue.length());
    }
}

Example 12 with RunLengthEncodedBlock

use of io.prestosql.spi.block.RunLengthEncodedBlock in project hetu-core by openlookeng.

the class TestPageSplitterUtil method testSplitPageNonDecreasingPageSize.

@Test
private void testSplitPageNonDecreasingPageSize() {
    int positionCount = 100;
    int maxPageSizeInBytes = 1;
    List<Type> types = ImmutableList.of(VARCHAR);
    Slice expectedValue = wrappedBuffer("test".getBytes(StandardCharsets.UTF_8));
    BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 1, expectedValue.length());
    blockBuilder.writeBytes(expectedValue, 0, expectedValue.length()).closeEntry();
    Block rleBlock = new RunLengthEncodedBlock(blockBuilder.build(), positionCount);
    Page initialPage = new Page(rleBlock);
    List<Page> pages = splitPage(initialPage, maxPageSizeInBytes);
    // the page should only be split in half as the recursion should terminate
    // after seeing that the size of the Page doesn't decrease
    assertEquals(pages.size(), 2);
    Page first = pages.get(0);
    Page second = pages.get(1);
    // the size of the pages will remain the same and should be greater than the maxPageSizeInBytes
    assertGreaterThan((int) first.getSizeInBytes(), maxPageSizeInBytes);
    assertGreaterThan((int) second.getSizeInBytes(), maxPageSizeInBytes);
    assertPositionCount(pages, positionCount);
    MaterializedResult actual = toMaterializedResult(TEST_SESSION, types, pages);
    MaterializedResult expected = toMaterializedResult(TEST_SESSION, types, ImmutableList.of(initialPage));
    assertEquals(actual, expected);
}

Example 13 with RunLengthEncodedBlock

use of io.prestosql.spi.block.RunLengthEncodedBlock in project hetu-core by openlookeng.

the class TestApproximatePercentileAggregation method createRLEBlock.

private static RunLengthEncodedBlock createRLEBlock(double percentile, int positionCount) {
    BlockBuilder blockBuilder = DOUBLE.createBlockBuilder(null, 1);
    DOUBLE.writeDouble(blockBuilder, percentile);
    return new RunLengthEncodedBlock(blockBuilder.build(), positionCount);
}

Example 14 with RunLengthEncodedBlock

use of io.prestosql.spi.block.RunLengthEncodedBlock in project boostkit-bigdata by kunpengcompute.

the class OrcFileWriter method appendRows.

@Override
public void appendRows(Page dataPage) {
    if (deleteDeltaFileWriter.isPresent()) {
        // Forward to delete writer
        deleteDeltaFileWriter.get().appendRows(dataPage);
    }
    Block[] dataBlocks = new Block[fileInputColumnIndexes.length];
    for (int i = 0; i < fileInputColumnIndexes.length; i++) {
        int inputColumnIndex = fileInputColumnIndexes[i];
        if (inputColumnIndex < 0) {
            dataBlocks[i] = new RunLengthEncodedBlock(dataNullBlocks.get(i), dataPage.getPositionCount());
        } else {
            dataBlocks[i] = dataPage.getBlock(inputColumnIndex);
        }
    }
    Block[] blocks = null;
    int i = 0;
    int totalColumns;
    if (isFullAcid()) {
        Block rowIdBlock = null;
        if (HiveACIDWriteType.isRowIdNeeded(acidWriteType.get())) {
            Block block = dataPage.getBlock(dataPage.getChannelCount() - 1);
            rowIdBlock = block.getLoadedBlock();
        }
        totalColumns = 6;
        blocks = new Block[totalColumns];
        // operation
        blocks[i++] = insertOperationId(dataPage, rowIdBlock, acidWriteType.get().getOperationId());
        // originalTransactionId
        blocks[i++] = insertOriginalTransaction(dataPage, rowIdBlock, writeId);
        // bucketId
        // Bucket Id is encoded to include some extra information from options.
        blocks[i++] = insertBucketIdBlock(dataPage, rowIdBlock, encodedBucketId);
        // rowId
        // rowId is incremental within a delta file./
        blocks[i++] = insertRowIdBlock(dataPage, rowIdBlock);
        // currentTransactionId
        blocks[i++] = insertCurrentTransaction(dataPage, rowIdBlock, writeId);
        boolean isDelete = acidWriteType.get() == HiveACIDWriteType.DELETE || (acidWriteType.get() == HiveACIDWriteType.VACUUM && acidOptions.map(o -> o.isWritingDeleteDelta()).orElse(false));
        blocks[i] = !isDelete ? RowBlock.fromFieldBlocks(dataPage.getPositionCount(), Optional.empty(), dataBlocks) : new RunLengthEncodedBlock(nullBlocks.get(nullBlocks.size() - 1), dataPage.getPositionCount());
        // statistics required to read from hive-cli for historical reasons.
        if (isDelete) {
            acidStats.deletes += dataPage.getPositionCount();
        } else {
            acidStats.inserts += dataPage.getPositionCount();
        }
    } else {
        blocks = dataBlocks;
    }
    Page page = new Page(dataPage.getPositionCount(), blocks);
    try {
        orcWriter.write(page);
    } catch (IOException | UncheckedIOException e) {
        throw new PrestoException(HIVE_WRITER_DATA_ERROR, e);
    }
}

Example 15 with RunLengthEncodedBlock

use of io.prestosql.spi.block.RunLengthEncodedBlock in project boostkit-bigdata by kunpengcompute.

the class RcFileFileWriter method appendRows.

@Override
public void appendRows(Page dataPage) {
    Block[] blocks = new Block[fileInputColumnIndexes.length];
    for (int i = 0; i < fileInputColumnIndexes.length; i++) {
        int inputColumnIndex = fileInputColumnIndexes[i];
        if (inputColumnIndex < 0) {
            blocks[i] = new RunLengthEncodedBlock(nullBlocks.get(i), dataPage.getPositionCount());
        } else {
            blocks[i] = dataPage.getBlock(inputColumnIndex);
        }
    }
    Page page = new Page(dataPage.getPositionCount(), blocks);
    try {
        rcFileWriter.write(page);
    } catch (IOException | UncheckedIOException e) {
        throw new PrestoException(HiveErrorCode.HIVE_WRITER_DATA_ERROR, e);
    }
}