use of io.prestosql.split.PageSinkManager in project hetu-core by openlookeng.
the class TestTableWriterOperator method testStatisticsAggregationSnapshot.
@Test
public void testStatisticsAggregationSnapshot() throws Exception {
PageSinkManager pageSinkManager = new PageSinkManager();
pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(new TableWriteInfoTestPageSink()));
ImmutableList<Type> outputTypes = ImmutableList.of(BIGINT, VARBINARY, BIGINT);
Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
DriverContext driverContext = createTaskContext(executor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
TableWriterOperator operator = (TableWriterOperator) createTableWriterOperator(pageSinkManager, new AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(0), Optional.empty())), true), outputTypes, session, driverContext);
operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0));
Object snapshot = operator.capture(operator.getOperatorContext().getDriverContext().getSerde());
assertEquals(SnapshotTestUtil.toSimpleSnapshotMapping(snapshot), createExpectedMapping());
operator.addInput(rowPagesBuilder(BIGINT).row(43).build().get(0));
operator.restore(snapshot, operator.getOperatorContext().getDriverContext().getSerde());
snapshot = operator.capture(operator.getOperatorContext().getDriverContext().getSerde());
assertEquals(SnapshotTestUtil.toSimpleSnapshotMapping(snapshot), createExpectedMapping());
operator.addInput(rowPagesBuilder(BIGINT).row(43).build().get(0));
assertTrue(operator.isBlocked().isDone());
assertTrue(operator.needsInput());
assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
assertEquals(driverContext.getMemoryUsage(), 0);
operator.finish();
assertFalse(operator.isFinished());
assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(null, null, 43).build().get(0));
assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(2, null, null).build().get(0));
assertTrue(operator.isBlocked().isDone());
assertFalse(operator.needsInput());
assertTrue(operator.isFinished());
operator.close();
assertMemoryIsReleased(operator);
TableWriterInfo info = operator.getInfo();
assertThat(info.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
assertThat(info.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
}
use of io.prestosql.split.PageSinkManager in project hetu-core by openlookeng.
the class TaskTestUtils method createTestingPlanner.
public static LocalExecutionPlanner createTestingPlanner() {
Metadata metadata = createTestMetadataManager();
PageSourceManager pageSourceManager = new PageSourceManager();
HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
FeaturesConfig featuresConfig = new FeaturesConfig();
CubeManager cubeManager = new CubeManager(featuresConfig, hetuMetaStoreManager);
pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
// we don't start the finalizer so nothing will be collected, which is ok for a test
FinalizerService finalizerService = new FinalizerService();
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
NodeInfo nodeInfo = new NodeInfo("test");
FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
HeuristicIndexerManager heuristicIndexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
return new LocalExecutionPlanner(metadata, new TypeAnalyzer(new SqlParser(), metadata), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}), (types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, (types, partitionFunction, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, new PagesIndex.TestingFactory(false), new JoinCompiler(metadata), new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
}
use of io.prestosql.split.PageSinkManager in project hetu-core by openlookeng.
the class TestTableWriterOperator method createTableWriterOperator.
private Operator createTableWriterOperator(BlockingPageSink blockingPageSink) {
PageSinkManager pageSinkManager = new PageSinkManager();
pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(blockingPageSink));
return createTableWriterOperator(pageSinkManager, new DevNullOperatorFactory(1, new PlanNodeId("test")), ImmutableList.of(BIGINT, VARBINARY));
}
use of io.prestosql.split.PageSinkManager in project hetu-core by openlookeng.
the class TestTableWriterOperator method testStatisticsAggregation.
@Test
public void testStatisticsAggregation() throws Exception {
PageSinkManager pageSinkManager = new PageSinkManager();
pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(new TableWriteInfoTestPageSink()));
ImmutableList<Type> outputTypes = ImmutableList.of(BIGINT, VARBINARY, BIGINT);
Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
DriverContext driverContext = createTaskContext(executor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
TableWriterOperator operator = (TableWriterOperator) createTableWriterOperator(pageSinkManager, new AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(0), Optional.empty())), true), outputTypes, session, driverContext);
operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0));
operator.addInput(rowPagesBuilder(BIGINT).row(43).build().get(0));
assertTrue(operator.isBlocked().isDone());
assertTrue(operator.needsInput());
assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
assertEquals(driverContext.getMemoryUsage(), 0);
operator.finish();
assertFalse(operator.isFinished());
assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(null, null, 43).build().get(0));
assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(2, null, null).build().get(0));
assertTrue(operator.isBlocked().isDone());
assertFalse(operator.needsInput());
assertTrue(operator.isFinished());
operator.close();
assertMemoryIsReleased(operator);
TableWriterInfo info = operator.getInfo();
assertThat(info.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
assertThat(info.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
}
use of io.prestosql.split.PageSinkManager in project hetu-core by openlookeng.
the class TestTableWriterOperator method testTableWriterInfo.
@Test
public void testTableWriterInfo() {
PageSinkManager pageSinkManager = new PageSinkManager();
pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(new TableWriteInfoTestPageSink()));
TableWriterOperator tableWriterOperator = (TableWriterOperator) createTableWriterOperator(pageSinkManager, new DevNullOperatorFactory(1, new PlanNodeId("test")), ImmutableList.of(BIGINT, VARBINARY));
RowPagesBuilder rowPagesBuilder = rowPagesBuilder(BIGINT);
for (int i = 0; i < 100; i++) {
rowPagesBuilder.addSequencePage(100, 0);
}
List<Page> pages = rowPagesBuilder.build();
long peakMemoryUsage = 0;
long validationCpuNanos = 0;
for (int i = 0; i < pages.size(); i++) {
Page page = pages.get(i);
peakMemoryUsage += page.getRetainedSizeInBytes();
validationCpuNanos += page.getPositionCount();
tableWriterOperator.addInput(page);
TableWriterInfo info = tableWriterOperator.getInfo();
assertEquals(info.getPageSinkPeakMemoryUsage(), peakMemoryUsage);
assertEquals((long) (info.getValidationCpuTime().getValue(NANOSECONDS)), validationCpuNanos);
}
}
Aggregations