use of com.facebook.presto.operator.TableFinishOperator.TableFinishOperatorFactory in project presto by prestodb.
the class TestTableFinishOperator method testTableWriteCommit.
@Test
public void testTableWriteCommit() throws Exception {
TestingTableFinisher tableFinisher = new TestingTableFinisher();
TestingPageSinkCommitter pageSinkCommitter = new TestingPageSinkCommitter();
ColumnStatisticMetadata statisticMetadata = new ColumnStatisticMetadata("column", MAX_VALUE);
StatisticAggregationsDescriptor<Integer> descriptor = new StatisticAggregationsDescriptor<>(ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(statisticMetadata, 0));
Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
TableFinishOperatorFactory operatorFactory = new TableFinishOperatorFactory(0, new PlanNodeId("node"), tableFinisher, pageSinkCommitter, new AggregationOperator.AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(STATS_START_CHANNEL), Optional.empty())), true), descriptor, session, TABLE_COMMIT_CONTEXT_CODEC, false);
DriverContext driverContext = createTaskContext(scheduledExecutor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
TableFinishOperator operator = (TableFinishOperator) operatorFactory.createOperator(driverContext);
List<Type> inputTypes = ImmutableList.of(BIGINT, VARBINARY, VARBINARY, BIGINT);
// pages for non-grouped execution
// expect lifespan committer not to be called and stats
operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, false), 1).build().get(0));
operator.addInput(rowPagesBuilder(inputTypes).row(3, new byte[] { 2 }, getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, true), null).build().get(0));
assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
// pages for unrecoverable grouped execution
// expect lifespan committer not to be called
operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(1), 1, 1, NO_COMMIT, false), 4).build().get(0));
operator.addInput(rowPagesBuilder(inputTypes).row(6, new byte[] { 5 }, getTableCommitContextBytes(Lifespan.driverGroup(1), 1, 1, NO_COMMIT, true), null).build().get(0));
assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
// pages for failed recoverable grouped execution
// expect lifespan committer not to be called and page ignored
operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 2, LIFESPAN_COMMIT, false), 100).build().get(0));
assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
// pages for successful recoverable grouped execution
// expect lifespan committer to be called and pages published
operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, false), 9).build().get(0));
operator.addInput(rowPagesBuilder(inputTypes).row(11, new byte[] { 10 }, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, true), null).build().get(0));
assertEquals(getOnlyElement(pageSinkCommitter.getCommittedFragments()), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 10 })));
assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
assertEquals(driverContext.getMemoryUsage(), 0);
assertTrue(operator.isBlocked().isDone());
assertTrue(operator.needsInput());
operator.finish();
assertFalse(operator.isFinished());
assertNull(operator.getOutput());
List<Type> outputTypes = ImmutableList.of(BIGINT);
assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(20).build().get(0));
assertTrue(operator.isBlocked().isDone());
assertFalse(operator.needsInput());
assertTrue(operator.isFinished());
operator.close();
assertEquals(tableFinisher.getFragments(), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 2 }), Slices.wrappedBuffer(new byte[] { 5 }), Slices.wrappedBuffer(new byte[] { 10 })));
assertEquals(tableFinisher.getComputedStatistics().size(), 1);
assertEquals(getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().size(), 1);
Block expectedStatisticsBlock = new LongArrayBlockBuilder(null, 1).writeLong(9).closeEntry().build();
assertBlockEquals(BIGINT, getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().get(statisticMetadata), expectedStatisticsBlock);
TableFinishInfo tableFinishInfo = operator.getInfo();
assertThat(tableFinishInfo.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
assertThat(tableFinishInfo.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
assertEquals(driverContext.getSystemMemoryUsage(), 0);
assertEquals(driverContext.getMemoryUsage(), 0);
}
use of com.facebook.presto.operator.TableFinishOperator.TableFinishOperatorFactory in project presto by prestodb.
the class TestTableFinishOperator method testStatisticsAggregation.
@Test
public void testStatisticsAggregation() throws Exception {
TestingTableFinisher tableFinisher = new TestingTableFinisher();
TestingPageSinkCommitter pageSinkCommitter = new TestingPageSinkCommitter();
ColumnStatisticMetadata statisticMetadata = new ColumnStatisticMetadata("column", MAX_VALUE);
StatisticAggregationsDescriptor<Integer> descriptor = new StatisticAggregationsDescriptor<>(ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of(statisticMetadata, 0));
Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
TableFinishOperatorFactory operatorFactory = new TableFinishOperatorFactory(0, new PlanNodeId("node"), tableFinisher, pageSinkCommitter, new AggregationOperator.AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(STATS_START_CHANNEL), Optional.empty())), true), descriptor, session, TABLE_COMMIT_CONTEXT_CODEC, false);
DriverContext driverContext = createTaskContext(scheduledExecutor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
TableFinishOperator operator = (TableFinishOperator) operatorFactory.createOperator(driverContext);
List<Type> inputTypes = ImmutableList.of(BIGINT, VARBINARY, VARBINARY, BIGINT);
byte[] tableCommitContextForStatsPage = getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, false);
operator.addInput(rowPagesBuilder(inputTypes).row(null, null, tableCommitContextForStatsPage, 6).build().get(0));
operator.addInput(rowPagesBuilder(inputTypes).row(null, null, tableCommitContextForStatsPage, 7).build().get(0));
byte[] tableCommitContextForFragmentsPage = getTableCommitContextBytes(Lifespan.taskWide(), 0, 0, NO_COMMIT, true);
operator.addInput(rowPagesBuilder(inputTypes).row(4, new byte[] { 1 }, tableCommitContextForFragmentsPage, null).build().get(0));
operator.addInput(rowPagesBuilder(inputTypes).row(5, new byte[] { 2 }, tableCommitContextForFragmentsPage, null).build().get(0));
assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
assertEquals(driverContext.getMemoryUsage(), 0);
assertTrue(operator.isBlocked().isDone());
assertTrue(operator.needsInput());
operator.finish();
assertFalse(operator.isFinished());
assertNull(operator.getOutput());
List<Type> outputTypes = ImmutableList.of(BIGINT);
assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(9).build().get(0));
assertTrue(operator.isBlocked().isDone());
assertFalse(operator.needsInput());
assertTrue(operator.isFinished());
operator.close();
assertEquals(tableFinisher.getFragments(), ImmutableList.of(Slices.wrappedBuffer(new byte[] { 1 }), Slices.wrappedBuffer(new byte[] { 2 })));
assertEquals(tableFinisher.getComputedStatistics().size(), 1);
assertEquals(getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().size(), 1);
Block expectedStatisticsBlock = new LongArrayBlockBuilder(null, 1).writeLong(7).closeEntry().build();
assertBlockEquals(BIGINT, getOnlyElement(tableFinisher.getComputedStatistics()).getColumnStatistics().get(statisticMetadata), expectedStatisticsBlock);
TableFinishInfo tableFinishInfo = operator.getInfo();
assertThat(tableFinishInfo.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
assertThat(tableFinishInfo.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
assertTrue(pageSinkCommitter.getCommittedFragments().isEmpty());
assertEquals(driverContext.getSystemMemoryUsage(), 0);
assertEquals(driverContext.getMemoryUsage(), 0);
}
Aggregations