Examples with PlanNodeId com.facebook.presto.spi.plan.PlanNodeId used on opensource projects

Example 51 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestMarkDistinctOperator method testMemoryReservationYield.

@Test(dataProvider = "dataType")
public void testMemoryReservationYield(Type type) {
    List<Page> input = createPagesWithDistinctHashKeys(type, 6_000, 600);
    OperatorFactory operatorFactory = new MarkDistinctOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(type), ImmutableList.of(0), Optional.of(1), joinCompiler);
    // get result with yield; pick a relatively small buffer for partitionRowCount's memory usage
    GroupByHashYieldAssertion.GroupByHashYieldResult result = finishOperatorWithYieldingGroupByHash(input, type, operatorFactory, operator -> ((MarkDistinctOperator) operator).getCapacity(), 1_400_000);
    assertGreaterThan(result.getYieldCount(), 5);
    assertGreaterThan(result.getMaxReservedBytes(), 20L << 20);
    int count = 0;
    for (Page page : result.getOutput()) {
        assertEquals(page.getChannelCount(), 3);
        for (int i = 0; i < page.getPositionCount(); i++) {
            assertEquals(page.getBlock(2).getByte(i), 1);
            count++;
        }
    }
    assertEquals(count, 6_000 * 600);
}

Example 52 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestRowNumberOperator method testRowNumberPartitionedLimit.

@Test(dataProvider = "hashEnabledValues")
public void testRowNumberPartitionedLimit(boolean hashEnabled) {
    DriverContext driverContext = getDriverContext();
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT, DOUBLE);
    List<Page> input = rowPagesBuilder.row(1L, 0.3).row(2L, 0.2).row(3L, 0.1).row(3L, 0.19).pageBreak().row(1L, 0.4).pageBreak().row(1L, 0.5).row(1L, 0.6).row(2L, 0.7).row(2L, 0.8).row(2L, 0.9).build();
    RowNumberOperator.RowNumberOperatorFactory operatorFactory = new RowNumberOperator.RowNumberOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT, DOUBLE), Ints.asList(1, 0), Ints.asList(0), ImmutableList.of(BIGINT), Optional.of(3), Optional.empty(), 10, joinCompiler);
    MaterializedResult expectedPartition1 = resultBuilder(driverContext.getSession(), DOUBLE, BIGINT).row(0.3, 1L).row(0.4, 1L).row(0.5, 1L).row(0.6, 1L).build();
    MaterializedResult expectedPartition2 = resultBuilder(driverContext.getSession(), DOUBLE, BIGINT).row(0.2, 2L).row(0.7, 2L).row(0.8, 2L).row(0.9, 2L).build();
    MaterializedResult expectedPartition3 = resultBuilder(driverContext.getSession(), DOUBLE, BIGINT).row(0.1, 3L).row(0.19, 3L).build();
    List<Page> pages = toPages(operatorFactory, driverContext, input);
    Block rowNumberColumn = getRowNumberColumn(pages);
    assertEquals(rowNumberColumn.getPositionCount(), 8);
    // Check that all row numbers generated are <= 3
    for (int i = 0; i < rowNumberColumn.getPositionCount(); i++) {
        assertTrue(rowNumberColumn.getLong(i) <= 3);
    }
    pages = stripRowNumberColumn(pages);
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), ImmutableList.of(DOUBLE, BIGINT), pages);
    ImmutableSet<?> actualSet = ImmutableSet.copyOf(actual.getMaterializedRows());
    ImmutableSet<?> expectedPartition1Set = ImmutableSet.copyOf(expectedPartition1.getMaterializedRows());
    ImmutableSet<?> expectedPartition2Set = ImmutableSet.copyOf(expectedPartition2.getMaterializedRows());
    ImmutableSet<?> expectedPartition3Set = ImmutableSet.copyOf(expectedPartition3.getMaterializedRows());
    assertEquals(Sets.intersection(expectedPartition1Set, actualSet).size(), 3);
    assertEquals(Sets.intersection(expectedPartition2Set, actualSet).size(), 3);
    assertEquals(Sets.intersection(expectedPartition3Set, actualSet).size(), 2);
}

Example 53 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestRowNumberOperator method testMemoryReservationYield.

@Test(dataProvider = "dataType")
public void testMemoryReservationYield(Type type) {
    List<Page> input = createPagesWithDistinctHashKeys(type, 6_000, 600);
    OperatorFactory operatorFactory = new RowNumberOperator.RowNumberOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(type), ImmutableList.of(0), ImmutableList.of(0), ImmutableList.of(type), Optional.empty(), Optional.empty(), 1, joinCompiler);
    // get result with yield; pick a relatively small buffer for partitionRowCount's memory usage
    GroupByHashYieldAssertion.GroupByHashYieldResult result = finishOperatorWithYieldingGroupByHash(input, type, operatorFactory, operator -> ((RowNumberOperator) operator).getCapacity(), 1_400_000);
    assertGreaterThan(result.getYieldCount(), 5);
    assertGreaterThan(result.getMaxReservedBytes(), 20L << 20);
    int count = 0;
    for (Page page : result.getOutput()) {
        assertEquals(page.getChannelCount(), 3);
        for (int i = 0; i < page.getPositionCount(); i++) {
            assertEquals(page.getBlock(2).getLong(i), 1);
            count++;
        }
    }
    assertEquals(count, 6_000 * 600);
}

Example 54 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestRowNumberOperator method testRowNumberUnpartitioned.

@Test
public void testRowNumberUnpartitioned() {
    DriverContext driverContext = getDriverContext();
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE).row(1L, 0.3).row(2L, 0.2).row(3L, 0.1).row(3L, 0.19).pageBreak().row(1L, 0.4).pageBreak().row(1L, 0.5).row(1L, 0.6).row(2L, 0.7).row(2L, 0.8).row(2L, 0.9).build();
    RowNumberOperator.RowNumberOperatorFactory operatorFactory = new RowNumberOperator.RowNumberOperatorFactory(0, new PlanNodeId("test"), ImmutableList.of(BIGINT, DOUBLE), Ints.asList(1, 0), Ints.asList(), ImmutableList.of(), Optional.empty(), Optional.empty(), 10, joinCompiler);
    MaterializedResult expectedResult = resultBuilder(driverContext.getSession(), DOUBLE, BIGINT).row(0.3, 1L).row(0.4, 1L).row(0.5, 1L).row(0.6, 1L).row(0.2, 2L).row(0.7, 2L).row(0.8, 2L).row(0.9, 2L).row(0.1, 3L).row(0.19, 3L).build();
    List<Page> pages = toPages(operatorFactory, driverContext, input);
    Block rowNumberColumn = getRowNumberColumn(pages);
    assertEquals(rowNumberColumn.getPositionCount(), 10);
    pages = stripRowNumberColumn(pages);
    MaterializedResult actual = toMaterializedResult(driverContext.getSession(), ImmutableList.of(DOUBLE, BIGINT), pages);
    assertEqualsIgnoreOrder(actual.getMaterializedRows(), expectedResult.getMaterializedRows());
}

Example 55 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestTableFinishOperator method testTableWriteCommit.

@Test
public void testTableWriteCommit() throws Exception {
    TestingTableFinisher tableFinisher = new TestingTableFinisher();
    TestingPageSinkCommitter pageSinkCommitter = new TestingPageSinkCommitter();
    ColumnStatisticMetadata statisticMetadata = new ColumnStatisticMetadata("column", MAX_VALUE);
    StatisticAggregationsDescriptor<Integer> descriptor = new StatisticAggregationsDescriptor<>(ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(statisticMetadata, 0));
    Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
    TableFinishOperatorFactory operatorFactory = new TableFinishOperatorFactory(0, new PlanNodeId("node"), tableFinisher, pageSinkCommitter, new AggregationOperator.AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(STATS_START_CHANNEL), Optional.empty())), true), descriptor, session, TABLE_COMMIT_CONTEXT_CODEC, false);
    DriverContext driverContext = createTaskContext(scheduledExecutor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
    TableFinishOperator operator = (TableFinishOperator) operatorFactory.createOperator(driverContext);
    List<Type> inputTypes = ImmutableList.of(BIGINT, VARBINARY, VARBINARY, BIGINT);
    // pages for non-grouped execution
    // expect lifespan committer not to be called and stats
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, false), 1).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(3, new byte[] { 2 }, getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, true), null).build().get(0));
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    // pages for unrecoverable grouped execution
    // expect lifespan committer not to be called
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(1), 1, 1, NO_COMMIT, false), 4).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(6, new byte[] { 5 }, getTableCommitContextBytes(Lifespan.driverGroup(1), 1, 1, NO_COMMIT, true), null).build().get(0));
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    // pages for failed recoverable grouped execution
    // expect lifespan committer not to be called and page ignored
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 2, LIFESPAN_COMMIT, false), 100).build().get(0));
    assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
    // pages for successful recoverable grouped execution
    // expect lifespan committer to be called and pages published
    operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, false), 9).build().get(0));
    operator.addInput(rowPagesBuilder(inputTypes).row(11, new byte[] { 10 }, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, true), null).build().get(0));
    assertEquals(getOnlyElement(pageSinkCommitter.getCommittedFragments()), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 10 })));
    assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
    assertEquals(driverContext.getMemoryUsage(), 0);
    assertTrue(operator.isBlocked().isDone());
    assertTrue(operator.needsInput());
    operator.finish();
    assertFalse(operator.isFinished());
    assertNull(operator.getOutput());
    List<Type> outputTypes = ImmutableList.of(BIGINT);
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(20).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertFalse(operator.needsInput());
    assertTrue(operator.isFinished());
    operator.close();
    assertEquals(tableFinisher.getFragments(), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 2 }), Slices.wrappedBuffer(new byte[] { 5 }), Slices.wrappedBuffer(new byte[] { 10 })));
    assertEquals(tableFinisher.getComputedStatistics().size(), 1);
    assertEquals(getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().size(), 1);
    Block expectedStatisticsBlock = new LongArrayBlockBuilder(null, 1).writeLong(9).closeEntry().build();
    assertBlockEquals(BIGINT, getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().get(statisticMetadata), expectedStatisticsBlock);
    TableFinishInfo tableFinishInfo = operator.getInfo();
    assertThat(tableFinishInfo.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertThat(tableFinishInfo.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertEquals(driverContext.getSystemMemoryUsage(), 0);
    assertEquals(driverContext.getMemoryUsage(), 0);
}