Examples with RunLengthEncodedBlock io.prestosql.spi.block.RunLengthEncodedBlock used on opensource projects

Example 26 with RunLengthEncodedBlock

use of io.prestosql.spi.block.RunLengthEncodedBlock in project hetu-core by openlookeng.

the class TestDictionaryAwarePageProjection method testRleBlock.

@Test(dataProvider = "forceYield")
public void testRleBlock(boolean forceYield) {
    Block value = createLongSequenceBlock(42, 43);
    RunLengthEncodedBlock block = new RunLengthEncodedBlock(value, 100);
    testProject(block, RunLengthEncodedBlock.class, forceYield);
}

Example 27 with RunLengthEncodedBlock

use of io.prestosql.spi.block.RunLengthEncodedBlock in project hetu-core by openlookeng.

the class OrcFileWriter method appendRows.

@Override
public void appendRows(Page dataPage) {
    if (deleteDeltaFileWriter.isPresent()) {
        // Forward to delete writer
        deleteDeltaFileWriter.get().appendRows(dataPage);
    }
    Block[] dataBlocks = new Block[fileInputColumnIndexes.length];
    for (int i = 0; i < fileInputColumnIndexes.length; i++) {
        int inputColumnIndex = fileInputColumnIndexes[i];
        if (inputColumnIndex < 0) {
            dataBlocks[i] = new RunLengthEncodedBlock(dataNullBlocks.get(i), dataPage.getPositionCount());
        } else {
            dataBlocks[i] = dataPage.getBlock(inputColumnIndex);
        }
    }
    Block[] blocks = null;
    int i = 0;
    int totalColumns;
    if (isFullAcid()) {
        Block rowIdBlock = null;
        if (HiveACIDWriteType.isRowIdNeeded(acidWriteType.get())) {
            Block block = dataPage.getBlock(dataPage.getChannelCount() - 1);
            rowIdBlock = block.getLoadedBlock();
        }
        totalColumns = 6;
        blocks = new Block[totalColumns];
        // operation
        blocks[i++] = insertOperationId(dataPage, rowIdBlock, acidWriteType.get().getOperationId());
        // originalTransactionId
        blocks[i++] = insertOriginalTransaction(dataPage, rowIdBlock, writeId);
        // bucketId
        // Bucket Id is encoded to include some extra information from options.
        blocks[i++] = insertBucketIdBlock(dataPage, rowIdBlock, encodedBucketId);
        // rowId
        // rowId is incremental within a delta file./
        blocks[i++] = insertRowIdBlock(dataPage, rowIdBlock);
        // currentTransactionId
        blocks[i++] = insertCurrentTransaction(dataPage, rowIdBlock, writeId);
        boolean isDelete = acidWriteType.get() == HiveACIDWriteType.DELETE || (acidWriteType.get() == HiveACIDWriteType.VACUUM && acidOptions.map(o -> o.isWritingDeleteDelta()).orElse(false));
        blocks[i] = !isDelete ? RowBlock.fromFieldBlocks(dataPage.getPositionCount(), Optional.empty(), dataBlocks) : new RunLengthEncodedBlock(nullBlocks.get(nullBlocks.size() - 1), dataPage.getPositionCount());
        // statistics required to read from hive-cli for historical reasons.
        if (isDelete) {
            acidStats.deletes += dataPage.getPositionCount();
        } else {
            acidStats.inserts += dataPage.getPositionCount();
        }
    } else {
        blocks = dataBlocks;
    }
    Page page = new Page(dataPage.getPositionCount(), blocks);
    try {
        orcWriter.write(page);
    } catch (IOException | UncheckedIOException e) {
        throw new PrestoException(HIVE_WRITER_DATA_ERROR, e);
    }
}

Example 28 with RunLengthEncodedBlock

use of io.prestosql.spi.block.RunLengthEncodedBlock in project hetu-core by openlookeng.

the class RcFileFileWriter method appendRows.

@Override
public void appendRows(Page dataPage) {
    Block[] blocks = new Block[fileInputColumnIndexes.length];
    for (int i = 0; i < fileInputColumnIndexes.length; i++) {
        int inputColumnIndex = fileInputColumnIndexes[i];
        if (inputColumnIndex < 0) {
            blocks[i] = new RunLengthEncodedBlock(nullBlocks.get(i), dataPage.getPositionCount());
        } else {
            blocks[i] = dataPage.getBlock(inputColumnIndex);
        }
    }
    Page page = new Page(dataPage.getPositionCount(), blocks);
    try {
        rcFileWriter.write(page);
    } catch (IOException | UncheckedIOException e) {
        throw new PrestoException(HiveErrorCode.HIVE_WRITER_DATA_ERROR, e);
    }
}

Example 29 with RunLengthEncodedBlock

use of io.prestosql.spi.block.RunLengthEncodedBlock in project hetu-core by openlookeng.

the class DictionaryAwarePageFilter method filter.

@Override
public SelectedPositions filter(ConnectorSession session, Page page) {
    Block block = page.getBlock(0).getLoadedBlock();
    if (block instanceof RunLengthEncodedBlock) {
        Block value = ((RunLengthEncodedBlock) block).getValue();
        Optional<boolean[]> selectedPosition = processDictionary(session, value);
        // in that case we fallback and process again so the correct error message sent
        if (selectedPosition.isPresent()) {
            return SelectedPositions.positionsRange(0, selectedPosition.get()[0] ? page.getPositionCount() : 0);
        }
    }
    if (block instanceof DictionaryBlock) {
        DictionaryBlock dictionaryBlock = (DictionaryBlock) block;
        // Attempt to process the dictionary.  If dictionary is processing has not been considered effective, an empty response will be returned
        Optional<boolean[]> selectedDictionaryPositions = processDictionary(session, dictionaryBlock.getDictionary());
        // record the usage count regardless of dictionary processing choice, so we have stats for next time
        lastDictionaryUsageCount += page.getPositionCount();
        // if dictionary was processed, produce a dictionary block; otherwise do normal processing
        if (selectedDictionaryPositions.isPresent()) {
            return selectDictionaryPositions(dictionaryBlock, selectedDictionaryPositions.get());
        }
    }
    return filter.filter(session, new Page(block));
}

Example 30 with RunLengthEncodedBlock

use of io.prestosql.spi.block.RunLengthEncodedBlock in project hetu-core by openlookeng.

the class TestPageProcessorCompiler method testSanityRLE.

@Test
public void testSanityRLE() {
    PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get();
    Slice varcharValue = Slices.utf8Slice("hello");
    Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100));
    Page outputPage = getOnlyElement(processor.process(null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)).orElseThrow(() -> new AssertionError("page is not present"));
    assertEquals(outputPage.getPositionCount(), 100);
    assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock);
    assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock);
    RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0);
    assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L);
    RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1);
    assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue);
}