Examples with RunLengthEncodedBlock com.facebook.presto.common.block.RunLengthEncodedBlock used on opensource projects

Example 26 with RunLengthEncodedBlock

use of com.facebook.presto.common.block.RunLengthEncodedBlock in project presto by prestodb.

the class TestPageSplitterUtil method testSplitPageNonDecreasingPageSize.

@Test
private void testSplitPageNonDecreasingPageSize() {
    int positionCount = 100;
    int maxPageSizeInBytes = 1;
    List<Type> types = ImmutableList.of(VARCHAR);
    Slice expectedValue = wrappedBuffer("test".getBytes());
    BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 1, expectedValue.length());
    blockBuilder.writeBytes(expectedValue, 0, expectedValue.length()).closeEntry();
    Block rleBlock = new RunLengthEncodedBlock(blockBuilder.build(), positionCount);
    Page initialPage = new Page(rleBlock);
    List<Page> pages = splitPage(initialPage, maxPageSizeInBytes);
    // the page should only be split in half as the recursion should terminate
    // after seeing that the size of the Page doesn't decrease
    assertEquals(pages.size(), 2);
    Page first = pages.get(0);
    Page second = pages.get(1);
    // the size of the pages will remain the same and should be greater than the maxPageSizeInBytes
    assertGreaterThan((int) first.getSizeInBytes(), maxPageSizeInBytes);
    assertGreaterThan((int) second.getSizeInBytes(), maxPageSizeInBytes);
    assertPositionCount(pages, positionCount);
    MaterializedResult actual = toMaterializedResult(TEST_SESSION, types, pages);
    MaterializedResult expected = toMaterializedResult(TEST_SESSION, types, ImmutableList.of(initialPage));
    assertEquals(actual, expected);
}

Example 27 with RunLengthEncodedBlock

use of com.facebook.presto.common.block.RunLengthEncodedBlock in project presto by prestodb.

the class TestMarkDistinctOperator method testRleDistinctMask.

@Test(dataProvider = "hashEnabledValues")
public void testRleDistinctMask(boolean hashEnabled) {
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT);
    List<Page> inputs = rowPagesBuilder.addSequencePage(100, 0).addSequencePage(100, 50).addSequencePage(1, 200).addSequencePage(1, 100).build();
    Page firstInput = inputs.get(0);
    Page secondInput = inputs.get(1);
    Page singleDistinctPage = inputs.get(2);
    Page singleNotDistinctPage = inputs.get(3);
    OperatorFactory operatorFactory = new MarkDistinctOperatorFactory(0, new PlanNodeId("test"), rowPagesBuilder.getTypes(), ImmutableList.of(0), rowPagesBuilder.getHashChannel(), joinCompiler);
    // mask channel is appended to the input
    int maskChannel = firstInput.getChannelCount();
    try (Operator operator = operatorFactory.createOperator(driverContext)) {
        operator.addInput(firstInput);
        Block allDistinctOutput = operator.getOutput().getBlock(maskChannel);
        operator.addInput(firstInput);
        Block noDistinctOutput = operator.getOutput().getBlock(maskChannel);
        // all distinct and no distinct conditions produce RLE blocks
        assertInstanceOf(allDistinctOutput, RunLengthEncodedBlock.class);
        assertTrue(BOOLEAN.getBoolean(allDistinctOutput, 0));
        assertInstanceOf(noDistinctOutput, RunLengthEncodedBlock.class);
        assertFalse(BOOLEAN.getBoolean(noDistinctOutput, 0));
        operator.addInput(secondInput);
        Block halfDistinctOutput = operator.getOutput().getBlock(maskChannel);
        // [0,50) is not distinct
        for (int position = 0; position < 50; position++) {
            assertFalse(BOOLEAN.getBoolean(halfDistinctOutput, position));
        }
        for (int position = 50; position < 100; position++) {
            assertTrue(BOOLEAN.getBoolean(halfDistinctOutput, position));
        }
        operator.addInput(singleDistinctPage);
        Block singleDistinctBlock = operator.getOutput().getBlock(maskChannel);
        assertFalse(singleDistinctBlock instanceof RunLengthEncodedBlock, "single position inputs should not be RLE");
        assertTrue(BOOLEAN.getBoolean(singleDistinctBlock, 0));
        operator.addInput(singleNotDistinctPage);
        Block singleNotDistinctBlock = operator.getOutput().getBlock(maskChannel);
        assertFalse(singleNotDistinctBlock instanceof RunLengthEncodedBlock, "single position inputs should not be RLE");
        assertFalse(BOOLEAN.getBoolean(singleNotDistinctBlock, 0));
    } catch (Exception e) {
        throwIfUnchecked(e);
        throw new RuntimeException(e);
    }
}

Example 28 with RunLengthEncodedBlock

use of com.facebook.presto.common.block.RunLengthEncodedBlock in project presto by prestodb.

the class RcFileFileWriter method appendRows.

@Override
public void appendRows(Page dataPage) {
    Block[] blocks = new Block[fileInputColumnIndexes.length];
    for (int i = 0; i < fileInputColumnIndexes.length; i++) {
        int inputColumnIndex = fileInputColumnIndexes[i];
        if (inputColumnIndex < 0) {
            blocks[i] = new RunLengthEncodedBlock(nullBlocks.get(i), dataPage.getPositionCount());
        } else {
            blocks[i] = dataPage.getBlock(inputColumnIndex);
        }
    }
    Page page = new Page(dataPage.getPositionCount(), blocks);
    try {
        rcFileWriter.write(page);
    } catch (IOException | UncheckedIOException e) {
        throw new PrestoException(HIVE_WRITER_DATA_ERROR, e);
    }
}

Example 29 with RunLengthEncodedBlock

use of com.facebook.presto.common.block.RunLengthEncodedBlock in project presto by prestodb.

the class TestAggregationOperator method testDistinctMaskWithNull.

@Test
public void testDistinctMaskWithNull() {
    AccumulatorFactory distinctFactory = COUNT.bind(ImmutableList.of(0), Optional.of(1), ImmutableList.of(BIGINT, BOOLEAN), ImmutableList.of(), ImmutableList.of(), null, // distinct
    true, new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()), ImmutableList.of(), false, TEST_SESSION, new TempStorageStandaloneSpillerFactory(new TestingTempStorageManager(), new BlockEncodingManager(), new NodeSpillConfig(), new FeaturesConfig(), new SpillerStats()));
    OperatorFactory operatorFactory = new AggregationOperatorFactory(0, new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(distinctFactory), false);
    DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION).addPipelineContext(0, true, true, false).addDriverContext();
    ByteArrayBlock trueMaskAllNull = new ByteArrayBlock(4, Optional.of(new boolean[] { true, true, true, true }), /* all positions are null */
    new byte[] { 1, 1, 1, 1 });
    /* non-zero value is true, all masks are true */
    Block trueNullRleMask = new RunLengthEncodedBlock(trueMaskAllNull.getSingleValueBlock(0), 4);
    List<Page> input = ImmutableList.of(new Page(4, createLongsBlock(1, 2, 3, 4), trueMaskAllNull), new Page(4, createLongsBlock(5, 6, 7, 8), trueNullRleMask));
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT).row(// all rows should be filtered by nulls
    0L).build();
    assertOperatorEquals(operatorFactory, driverContext, input, expected);
}

Example 30 with RunLengthEncodedBlock

use of com.facebook.presto.common.block.RunLengthEncodedBlock in project presto by prestodb.

the class AbstractDecimalSelectiveStreamReader method getBlockView.

@Override
public BlockLease getBlockView(int[] positions, int positionCount) {
    checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero");
    checkState(outputRequired, "This stream reader doesn't produce output");
    checkState(positionCount <= outputPositionCount, "Not enough values");
    checkState(!valuesInUse, "BlockLease hasn't been closed yet");
    if (allNulls) {
        return newLease(new RunLengthEncodedBlock(nullBlock, positionCount));
    }
    boolean includeNulls = nullsAllowed && presentStream != null;
    if (positionCount != outputPositionCount) {
        compactValues(positions, positionCount, includeNulls);
    }
    return newLease(makeBlock(positionCount, includeNulls, nulls, values));
}