Search in sources :

Example 1 with ByteArrayBlock

use of com.facebook.presto.common.block.ByteArrayBlock in project presto by prestodb.

the class ByteSelectiveStreamReader method getBlockView.

@Override
public BlockLease getBlockView(int[] positions, int positionCount) {
    checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero");
    checkState(outputRequired, "This stream reader doesn't produce output");
    checkState(positionCount <= outputPositionCount, "Not enough values");
    checkState(!valuesInUse, "BlockLease hasn't been closed yet");
    if (allNulls) {
        return newLease(new RunLengthEncodedBlock(NULL_BLOCK, positionCount));
    }
    boolean includeNulls = nullsAllowed && presentStream != null;
    if (positionCount != outputPositionCount) {
        compactValues(positions, positionCount, includeNulls);
    }
    return newLease(new ByteArrayBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), values));
}
Also used : ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) RunLengthEncodedBlock(com.facebook.presto.common.block.RunLengthEncodedBlock)

Example 2 with ByteArrayBlock

use of com.facebook.presto.common.block.ByteArrayBlock in project presto by prestodb.

the class TestHashAggregationOperator method testMask.

@Test
public void testMask() {
    int positions = 4;
    Block groupingBlock = RunLengthEncodedBlock.create(BIGINT, 1L, positions);
    Block countBlock = RunLengthEncodedBlock.create(BIGINT, 1L, positions);
    Block maskBlock = new ByteArrayBlock(positions, Optional.of(new boolean[] { false, false, true, true }), new byte[] { (byte) 0, (byte) 1, (byte) 0, (byte) 1 });
    Page page = new Page(groupingBlock, countBlock, maskBlock);
    HashAggregationOperatorFactory operatorFactory = new HashAggregationOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT), ImmutableList.of(0), ImmutableList.of(), Step.SINGLE, false, ImmutableList.of(COUNT.bind(ImmutableList.of(1), Optional.of(2))), Optional.empty(), Optional.empty(), 1, Optional.of(new DataSize(16, MEGABYTE)), false, new DataSize(16, MEGABYTE), new DataSize(16, MEGABYTE), new FailingSpillerFactory(), joinCompiler, false);
    List<Page> outputPages = toPages(operatorFactory, createDriverContext(), ImmutableList.of(page)).stream().filter(p -> p.getPositionCount() > 0).collect(toImmutableList());
    assertEquals(outputPages.size(), 1);
    Page outputPage = outputPages.get(0);
    assertEquals(outputPage.getBlock(0).getLong(0), 1L);
    assertEquals(outputPage.getBlock(1).getLong(0), 1L);
}
Also used : PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) Page(com.facebook.presto.common.Page) RunLengthEncodedBlock(com.facebook.presto.common.block.RunLengthEncodedBlock) MetadataManager(com.facebook.presto.metadata.MetadataManager) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) MaterializedResult.resultBuilder(com.facebook.presto.testing.MaterializedResult.resultBuilder) Test(org.testng.annotations.Test) Unit(io.airlift.units.DataSize.Unit) AfterMethod(org.testng.annotations.AfterMethod) OperatorAssertion.toPages(com.facebook.presto.operator.OperatorAssertion.toPages) RowPagesBuilder(com.facebook.presto.RowPagesBuilder) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) Slices(io.airlift.slice.Slices) KILOBYTE(io.airlift.units.DataSize.Unit.KILOBYTE) HashAggregationOperatorFactory(com.facebook.presto.operator.HashAggregationOperator.HashAggregationOperatorFactory) OperatorAssertion.assertPagesEqualIgnoreOrder(com.facebook.presto.operator.OperatorAssertion.assertPagesEqualIgnoreOrder) ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) Step(com.facebook.presto.spi.plan.AggregationNode.Step) BlockBuilder(com.facebook.presto.common.block.BlockBuilder) SIZE_OF_DOUBLE(io.airlift.slice.SizeOf.SIZE_OF_DOUBLE) DOUBLE(com.facebook.presto.common.type.DoubleType.DOUBLE) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) BeforeMethod(org.testng.annotations.BeforeMethod) InternalAggregationFunction(com.facebook.presto.operator.aggregation.InternalAggregationFunction) Collections.emptyIterator(java.util.Collections.emptyIterator) GroupByHashYieldResult(com.facebook.presto.operator.GroupByHashYieldAssertion.GroupByHashYieldResult) String.format(java.lang.String.format) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) Threads.daemonThreadsNamed(com.facebook.airlift.concurrent.Threads.daemonThreadsNamed) OperatorAssertion.assertOperatorEqualsIgnoreOrder(com.facebook.presto.operator.OperatorAssertion.assertOperatorEqualsIgnoreOrder) DataSize(io.airlift.units.DataSize) List(java.util.List) Assertions.assertGreaterThan(com.facebook.airlift.testing.Assertions.assertGreaterThan) Optional(java.util.Optional) RowPagesBuilder.rowPagesBuilder(com.facebook.presto.RowPagesBuilder.rowPagesBuilder) ExceededMemoryLimitException(com.facebook.presto.ExceededMemoryLimitException) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) HashAggregationBuilder(com.facebook.presto.operator.aggregation.builder.HashAggregationBuilder) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) DataProvider(org.testng.annotations.DataProvider) Strings.nullToEmpty(com.google.common.base.Strings.nullToEmpty) InMemoryHashAggregationBuilder(com.facebook.presto.operator.aggregation.builder.InMemoryHashAggregationBuilder) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) DataSize.succinctBytes(io.airlift.units.DataSize.succinctBytes) Assert.assertEquals(org.testng.Assert.assertEquals) TypeSignatureProvider.fromTypes(com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes) Spiller(com.facebook.presto.spiller.Spiller) ArrayList(java.util.ArrayList) TEST_SESSION(com.facebook.presto.SessionTestUtils.TEST_SESSION) PageBuilderStatus(com.facebook.presto.common.block.PageBuilderStatus) ImmutableList(com.google.common.collect.ImmutableList) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) TestingTaskContext(com.facebook.presto.testing.TestingTaskContext) Type(com.facebook.presto.common.type.Type) ExecutorService(java.util.concurrent.ExecutorService) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) Iterator(java.util.Iterator) Assertions.assertEqualsIgnoreOrder(com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder) TestingTaskContext.createTaskContext(com.facebook.presto.testing.TestingTaskContext.createTaskContext) Assert.fail(org.testng.Assert.fail) OperatorAssertion.toMaterializedResult(com.facebook.presto.operator.OperatorAssertion.toMaterializedResult) IOException(java.io.IOException) OperatorAssertion.dropChannel(com.facebook.presto.operator.OperatorAssertion.dropChannel) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) Ints(com.google.common.primitives.Ints) AggregatedMemoryContext(com.facebook.presto.memory.context.AggregatedMemoryContext) MaterializedResult(com.facebook.presto.testing.MaterializedResult) GroupByHashYieldAssertion.finishOperatorWithYieldingGroupByHash(com.facebook.presto.operator.GroupByHashYieldAssertion.finishOperatorWithYieldingGroupByHash) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) SIZE_OF_LONG(io.airlift.slice.SizeOf.SIZE_OF_LONG) Futures.immediateFailedFuture(com.google.common.util.concurrent.Futures.immediateFailedFuture) GroupByHashYieldAssertion.createPagesWithDistinctHashKeys(com.facebook.presto.operator.GroupByHashYieldAssertion.createPagesWithDistinctHashKeys) Assert.assertTrue(org.testng.Assert.assertTrue) SpillerFactory(com.facebook.presto.spiller.SpillerFactory) Block(com.facebook.presto.common.block.Block) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) DataSize(io.airlift.units.DataSize) RunLengthEncodedBlock(com.facebook.presto.common.block.RunLengthEncodedBlock) ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) Block(com.facebook.presto.common.block.Block) ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) Page(com.facebook.presto.common.Page) HashAggregationOperatorFactory(com.facebook.presto.operator.HashAggregationOperator.HashAggregationOperatorFactory) Test(org.testng.annotations.Test)

Example 3 with ByteArrayBlock

use of com.facebook.presto.common.block.ByteArrayBlock in project presto by prestodb.

the class TestArrayBlock method testCompactBlock.

@Test
public void testCompactBlock() {
    Block emptyValueBlock = new ByteArrayBlock(0, Optional.empty(), new byte[0]);
    Block compactValueBlock = new ByteArrayBlock(16, Optional.empty(), createExpectedValue(16).getBytes());
    Block inCompactValueBlock = new ByteArrayBlock(16, Optional.empty(), createExpectedValue(17).getBytes());
    int[] offsets = { 0, 1, 1, 2, 4, 8, 16 };
    boolean[] valueIsNull = { false, true, false, false, false, false };
    testCompactBlock(fromElementBlock(0, Optional.empty(), new int[1], emptyValueBlock));
    testCompactBlock(fromElementBlock(valueIsNull.length, Optional.of(valueIsNull), offsets, compactValueBlock));
    testIncompactBlock(fromElementBlock(valueIsNull.length - 1, Optional.of(valueIsNull), offsets, compactValueBlock));
    // underlying value block is not compact
    testIncompactBlock(fromElementBlock(valueIsNull.length, Optional.of(valueIsNull), offsets, inCompactValueBlock));
}
Also used : BlockAssertions.createRandomLongsBlock(com.facebook.presto.block.BlockAssertions.createRandomLongsBlock) BlockAssertions.createRLEBlock(com.facebook.presto.block.BlockAssertions.createRLEBlock) BlockAssertions.createRandomDictionaryBlock(com.facebook.presto.block.BlockAssertions.createRandomDictionaryBlock) ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) ArrayBlock.fromElementBlock(com.facebook.presto.common.block.ArrayBlock.fromElementBlock) BlockAssertions.createLongDictionaryBlock(com.facebook.presto.block.BlockAssertions.createLongDictionaryBlock) Block(com.facebook.presto.common.block.Block) ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) Test(org.testng.annotations.Test)

Example 4 with ByteArrayBlock

use of com.facebook.presto.common.block.ByteArrayBlock in project presto by prestodb.

the class TestByteArrayBlock method testCompactBlock.

@Test
public void testCompactBlock() {
    byte[] byteArray = { (byte) 0, (byte) 0, (byte) 1, (byte) 2, (byte) 3, (byte) 4 };
    boolean[] valueIsNull = { false, true, false, false, false, false };
    testCompactBlock(new ByteArrayBlock(0, Optional.empty(), new byte[0]));
    testCompactBlock(new ByteArrayBlock(byteArray.length, Optional.of(valueIsNull), byteArray));
    testIncompactBlock(new ByteArrayBlock(byteArray.length - 1, Optional.of(valueIsNull), byteArray));
}
Also used : ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) Test(org.testng.annotations.Test)

Example 5 with ByteArrayBlock

use of com.facebook.presto.common.block.ByteArrayBlock in project presto by prestodb.

the class TestRowBlock method testCompactBlock.

@Test
public void testCompactBlock() {
    Block emptyBlock = new ByteArrayBlock(0, Optional.empty(), new byte[0]);
    Block compactFieldBlock1 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(5).getBytes());
    Block compactFieldBlock2 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(5).getBytes());
    Block incompactFiledBlock1 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(6).getBytes());
    Block incompactFiledBlock2 = new ByteArrayBlock(5, Optional.empty(), createExpectedValue(6).getBytes());
    boolean[] rowIsNull = { false, true, false, false, false, false };
    assertCompact(fromFieldBlocks(0, Optional.empty(), new Block[] { emptyBlock, emptyBlock }));
    assertCompact(fromFieldBlocks(rowIsNull.length, Optional.of(rowIsNull), new Block[] { compactFieldBlock1, compactFieldBlock2 }));
    // TODO: add test case for a sliced RowBlock
    // underlying field blocks are not compact
    testIncompactBlock(fromFieldBlocks(rowIsNull.length, Optional.of(rowIsNull), new Block[] { incompactFiledBlock1, incompactFiledBlock2 }));
    testIncompactBlock(fromFieldBlocks(rowIsNull.length, Optional.of(rowIsNull), new Block[] { incompactFiledBlock1, incompactFiledBlock2 }));
}
Also used : BlockAssertions.createRandomLongsBlock(com.facebook.presto.block.BlockAssertions.createRandomLongsBlock) BlockAssertions.createRLEBlock(com.facebook.presto.block.BlockAssertions.createRLEBlock) BlockAssertions.createRandomDictionaryBlock(com.facebook.presto.block.BlockAssertions.createRandomDictionaryBlock) ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) BlockAssertions.createLongDictionaryBlock(com.facebook.presto.block.BlockAssertions.createLongDictionaryBlock) Block(com.facebook.presto.common.block.Block) SingleRowBlock(com.facebook.presto.common.block.SingleRowBlock) ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) Test(org.testng.annotations.Test)

Aggregations

ByteArrayBlock (com.facebook.presto.common.block.ByteArrayBlock)12 Block (com.facebook.presto.common.block.Block)9 Test (org.testng.annotations.Test)8 RunLengthEncodedBlock (com.facebook.presto.common.block.RunLengthEncodedBlock)6 BlockAssertions.createLongDictionaryBlock (com.facebook.presto.block.BlockAssertions.createLongDictionaryBlock)4 BlockAssertions.createRLEBlock (com.facebook.presto.block.BlockAssertions.createRLEBlock)4 BlockAssertions.createRandomLongsBlock (com.facebook.presto.block.BlockAssertions.createRandomLongsBlock)4 BlockAssertions.createRandomDictionaryBlock (com.facebook.presto.block.BlockAssertions.createRandomDictionaryBlock)3 BlockAssertions.createLongsBlock (com.facebook.presto.block.BlockAssertions.createLongsBlock)2 Page (com.facebook.presto.common.Page)2 BlockBuilder (com.facebook.presto.common.block.BlockBuilder)2 SingleRowBlock (com.facebook.presto.common.block.SingleRowBlock)2 Type (com.facebook.presto.common.type.Type)2 Threads.daemonThreadsNamed (com.facebook.airlift.concurrent.Threads.daemonThreadsNamed)1 Assertions.assertEqualsIgnoreOrder (com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder)1 Assertions.assertGreaterThan (com.facebook.airlift.testing.Assertions.assertGreaterThan)1 ExceededMemoryLimitException (com.facebook.presto.ExceededMemoryLimitException)1 RowPagesBuilder (com.facebook.presto.RowPagesBuilder)1 RowPagesBuilder.rowPagesBuilder (com.facebook.presto.RowPagesBuilder.rowPagesBuilder)1 TEST_SESSION (com.facebook.presto.SessionTestUtils.TEST_SESSION)1