Examples with LongArrayBlockBuilder com.facebook.presto.common.block.LongArrayBlockBuilder used on opensource projects

Example 1 with LongArrayBlockBuilder

use of com.facebook.presto.common.block.LongArrayBlockBuilder in project presto by prestodb.

the class DeltaShardRewriter method writeDeltaDeleteFile.

@VisibleForTesting
Collection<Slice> writeDeltaDeleteFile(BitSet rowsToDelete) {
    if (rowsToDelete.isEmpty()) {
        return ImmutableList.of();
    }
    // TODO: Under current implementation, one block can only hold INT_MAX many rows
    // which theoretically may not be enough to hold all rows from an ORC file.
    // At this point, rowsToDelete couldn't be empty
    oldDeltaDeleteShardUuid.ifPresent(oldDeltaDeleteShardUuid -> mergeToRowsToDelete(rowsToDelete, oldDeltaDeleteShardUuid));
    if (rowsToDelete.cardinality() == oldShardRowCount) {
        // Delete original file
        return shardDeleteDelta(oldShardUuid, oldDeltaDeleteShardUuid, Optional.empty());
    }
    BlockBuilder blockBuilder = new LongArrayBlockBuilder(null, rowsToDelete.size());
    for (int i = rowsToDelete.nextSetBit(0); i >= 0; i = rowsToDelete.nextSetBit(i + 1)) {
        blockBuilder.writeLong(i);
    }
    // blockToDelete is LongArrayBlock
    StoragePageSink pageSink = orcStorageManager.createStoragePageSink(hdfsContext, transactionId, bucketNumber, ImmutableList.of(0L), ImmutableList.of(BIGINT), true);
    pageSink.appendPages(ImmutableList.of(new Page(blockBuilder.build())));
    List<ShardInfo> shardInfos = getFutureValue(pageSink.commit());
    // Guaranteed that shardInfos only has one element since we only call commit one time
    ShardInfo newDeltaDeleteShard = Iterables.getOnlyElement(shardInfos);
    return shardDeleteDelta(oldShardUuid, oldDeltaDeleteShardUuid, Optional.of(newDeltaDeleteShard));
}

Example 2 with LongArrayBlockBuilder

use of com.facebook.presto.common.block.LongArrayBlockBuilder in project presto by prestodb.

the class TestRowExpressionFormatter method testConstants.

@Test
public void testConstants() {
    // null
    RowExpression constantExpression = constantNull(UNKNOWN);
    assertEquals(format(constantExpression), "null");
    // boolean
    constantExpression = constant(true, BOOLEAN);
    assertEquals(format(constantExpression), "BOOLEAN'true'");
    // double
    constantExpression = constant(1.1, DOUBLE);
    assertEquals(format(constantExpression), "DOUBLE'1.1'");
    constantExpression = constant(Double.NaN, DOUBLE);
    assertEquals(format(constantExpression), "DOUBLE'NaN'");
    constantExpression = constant(Double.POSITIVE_INFINITY, DOUBLE);
    assertEquals(format(constantExpression), "DOUBLE'Infinity'");
    // real
    constantExpression = constant((long) floatToIntBits(1.1f), REAL);
    assertEquals(format(constantExpression), "REAL'1.1'");
    constantExpression = constant((long) floatToIntBits(Float.NaN), REAL);
    assertEquals(format(constantExpression), "REAL'NaN'");
    constantExpression = constant((long) floatToIntBits(Float.POSITIVE_INFINITY), REAL);
    assertEquals(format(constantExpression), "REAL'Infinity'");
    // string
    constantExpression = constant(utf8Slice("abcde"), VARCHAR);
    assertEquals(format(constantExpression), "VARCHAR'abcde'");
    constantExpression = constant(utf8Slice("fgh"), createCharType(3));
    assertEquals(format(constantExpression), "CHAR'fgh'");
    // integer
    constantExpression = constant(1L, TINYINT);
    assertEquals(format(constantExpression), "TINYINT'1'");
    constantExpression = constant(1L, SMALLINT);
    assertEquals(format(constantExpression), "SMALLINT'1'");
    constantExpression = constant(1L, INTEGER);
    assertEquals(format(constantExpression), "INTEGER'1'");
    constantExpression = constant(1L, BIGINT);
    assertEquals(format(constantExpression), "BIGINT'1'");
    // varbinary
    Slice value = Slices.wrappedBuffer(BaseEncoding.base16().decode("123456AB"));
    constantExpression = constant(value, VARBINARY);
    assertEquals(format(constantExpression), "X'12 34 56 ab'");
    // color
    constantExpression = constant(256L, COLOR);
    assertEquals(format(constantExpression), "COLOR'256'");
    // long and short decimals
    constantExpression = constant(decimal("1.2345678910"), DecimalType.createDecimalType(11, 10));
    assertEquals(format(constantExpression), "DECIMAL'1.2345678910'");
    constantExpression = constant(decimal("1.281734081274028174012432412423134"), DecimalType.createDecimalType(34, 33));
    assertEquals(format(constantExpression), "DECIMAL'1.281734081274028174012432412423134'");
    // time
    constantExpression = constant(662727600000L, TIMESTAMP);
    assertEquals(format(constantExpression), "TIMESTAMP'1991-01-01 00:00:00.000'");
    constantExpression = constant(7670L, DATE);
    assertEquals(format(constantExpression), "DATE'1991-01-01'");
    // interval
    constantExpression = constant(24L, INTERVAL_DAY_TIME);
    assertEquals(format(constantExpression), "INTERVAL DAY TO SECOND'0 00:00:00.024'");
    constantExpression = constant(25L, INTERVAL_YEAR_MONTH);
    assertEquals(format(constantExpression), "INTERVAL YEAR TO MONTH'2-1'");
    // block
    constantExpression = constant(new LongArrayBlockBuilder(null, 4).writeLong(1L).writeLong(2).build(), new ArrayType(BIGINT));
    assertEquals(format(constantExpression), "[Block: position count: 2; size: 96 bytes]");
}

Example 3 with LongArrayBlockBuilder

use of com.facebook.presto.common.block.LongArrayBlockBuilder in project presto by prestodb.

the class TestTableFinishOperator method testTableWriteCommit.

@Test
public void testTableWriteCommit() throws Exception {
    TestingTableFinisher tableFinisher = new TestingTableFinisher();
    TestingPageSinkCommitter pageSinkCommitter = new TestingPageSinkCommitter();
    ColumnStatisticMetadata statisticMetadata = new ColumnStatisticMetadata("column", MAX_VALUE);
    StatisticAggregationsDescriptor<Integer> descriptor = new StatisticAggregationsDescriptor<>(ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(statisticMetadata, 0));
    Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
    TableFinishOperatorFactory operatorFactory = new TableFinishOperatorFactory(0, new PlanNodeId("node"), tableFinisher, pageSinkCommitter, new AggregationOperator.AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(STATS_START_CHANNEL), Optional.empty())), true), descriptor, session, TABLE_COMMIT_CONTEXT_CODEC, false);
    DriverContext driverContext = createTaskContext(scheduledExecutor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
    TableFinishOperator operator = (TableFinishOperator) operatorFactory.createOperator(driverContext);
    List<Type> inputTypes = ImmutableList.of(BIGINT, VARBINARY, VARBINARY, BIGINT);
    // pages for non-grouped execution
    // expect lifespan committer not to be called and stats
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, false), 1).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(3, new byte[] { 2 }, getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, true), null).build().get(0));
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    // pages for unrecoverable grouped execution
    // expect lifespan committer not to be called
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(1), 1, 1, NO_COMMIT, false), 4).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(6, new byte[] { 5 }, getTableCommitContextBytes(Lifespan.driverGroup(1), 1, 1, NO_COMMIT, true), null).build().get(0));
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    // pages for failed recoverable grouped execution
    // expect lifespan committer not to be called and page ignored
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 2, LIFESPAN_COMMIT, false), 100).build().get(0));
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    // pages for successful recoverable grouped execution
    // expect lifespan committer to be called and pages published
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, false), 9).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(11, new byte[] { 10 }, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, true), null).build().get(0));
    assertEquals(getOnlyElement(pageSinkCommitter.getCommittedFragments()), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 10 })));
    assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
    assertEquals(driverContext.getMemoryUsage(), 0);
    assertTrue(operator.isBlocked().isDone());
    assertTrue(operator.needsInput());
    operator.finish();
    assertFalse(operator.isFinished());
    assertNull(operator.getOutput());
    List<Type> outputTypes = ImmutableList.of(BIGINT);
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(20).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertFalse(operator.needsInput());
    assertTrue(operator.isFinished());
    operator.close();
    assertEquals(tableFinisher.getFragments(), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 2 }), Slices.wrappedBuffer(new byte[] { 5 }), Slices.wrappedBuffer(new byte[] { 10 })));
    assertEquals(tableFinisher.getComputedStatistics().size(), 1);
    assertEquals(getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().size(), 1);
    Block expectedStatisticsBlock = new LongArrayBlockBuilder(null, 1).writeLong(9).closeEntry().build();
    assertBlockEquals(BIGINT, getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().get(statisticMetadata), expectedStatisticsBlock);
    TableFinishInfo tableFinishInfo = operator.getInfo();
    assertThat(tableFinishInfo.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertThat(tableFinishInfo.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertEquals(driverContext.getSystemMemoryUsage(), 0);
    assertEquals(driverContext.getMemoryUsage(), 0);
}

Example 4 with LongArrayBlockBuilder

use of com.facebook.presto.common.block.LongArrayBlockBuilder in project presto by prestodb.

the class TestTableFinishOperator method testStatisticsAggregation.

@Test
public void testStatisticsAggregation() throws Exception {
    TestingTableFinisher tableFinisher = new TestingTableFinisher();
    TestingPageSinkCommitter pageSinkCommitter = new TestingPageSinkCommitter();
    ColumnStatisticMetadata statisticMetadata = new ColumnStatisticMetadata("column", MAX_VALUE);
    StatisticAggregationsDescriptor<Integer> descriptor = new StatisticAggregationsDescriptor<>(ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(statisticMetadata, 0));
    Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
    TableFinishOperatorFactory operatorFactory = new TableFinishOperatorFactory(0, new PlanNodeId("node"), tableFinisher, pageSinkCommitter, new AggregationOperator.AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(STATS_START_CHANNEL), Optional.empty())), true), descriptor, session, TABLE_COMMIT_CONTEXT_CODEC, false);
    DriverContext driverContext = createTaskContext(scheduledExecutor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
    TableFinishOperator operator = (TableFinishOperator) operatorFactory.createOperator(driverContext);
    List<Type> inputTypes = ImmutableList.of(BIGINT, VARBINARY, VARBINARY, BIGINT);
    byte[] tableCommitContextForStatsPage = getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, false);
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, tableCommitContextForStatsPage, 6).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, tableCommitContextForStatsPage, 7).build().get(0));
    byte[] tableCommitContextForFragmentsPage = getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, true);
    operator.addInput(rowPagesBuilder(inputTypes).row(4, new byte[] { 1 }, tableCommitContextForFragmentsPage, null).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(5, new byte[] { 2 }, tableCommitContextForFragmentsPage, null).build().get(0));
    assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
    assertEquals(driverContext.getMemoryUsage(), 0);
    assertTrue(operator.isBlocked().isDone());
    assertTrue(operator.needsInput());
    operator.finish();
    assertFalse(operator.isFinished());
    assertNull(operator.getOutput());
    List<Type> outputTypes = ImmutableList.of(BIGINT);
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(9).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertFalse(operator.needsInput());
    assertTrue(operator.isFinished());
    operator.close();
    assertEquals(tableFinisher.getFragments(), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 1 }), Slices.wrappedBuffer(new byte[] { 2 })));
    assertEquals(tableFinisher.getComputedStatistics().size(), 1);
    assertEquals(getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().size(), 1);
    Block expectedStatisticsBlock = new LongArrayBlockBuilder(null, 1).writeLong(7).closeEntry().build();
    assertBlockEquals(BIGINT, getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().get(statisticMetadata), expectedStatisticsBlock);
    TableFinishInfo tableFinishInfo = operator.getInfo();
    assertThat(tableFinishInfo.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertThat(tableFinishInfo.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    assertEquals(driverContext.getSystemMemoryUsage(), 0);
    assertEquals(driverContext.getMemoryUsage(), 0);
}

Example 5 with LongArrayBlockBuilder

use of com.facebook.presto.common.block.LongArrayBlockBuilder in project presto by prestodb.

the class TestDoubleType method testNaNHash.

@Test
public void testNaNHash() {
    BlockBuilder blockBuilder = new LongArrayBlockBuilder(null, 4);
    blockBuilder.writeLong(doubleToLongBits(Double.NaN));
    blockBuilder.writeLong(doubleToRawLongBits(Double.NaN));
    // the following two are the long values of a double NaN
    blockBuilder.writeLong(-0x000fffffffffffffL);
    blockBuilder.writeLong(0x7ff8000000000000L);
    assertEquals(DOUBLE.hash(blockBuilder, 0), DOUBLE.hash(blockBuilder, 1));
    assertEquals(DOUBLE.hash(blockBuilder, 0), DOUBLE.hash(blockBuilder, 2));
    assertEquals(DOUBLE.hash(blockBuilder, 0), DOUBLE.hash(blockBuilder, 3));
}