Search in sources :

Example 1 with PageSinkManager

use of io.prestosql.split.PageSinkManager in project hetu-core by openlookeng.

the class TestTableWriterOperator method testStatisticsAggregationSnapshot.

@Test
public void testStatisticsAggregationSnapshot() throws Exception {
    PageSinkManager pageSinkManager = new PageSinkManager();
    pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(new TableWriteInfoTestPageSink()));
    ImmutableList<Type> outputTypes = ImmutableList.of(BIGINT, VARBINARY, BIGINT);
    Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
    DriverContext driverContext = createTaskContext(executor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
    TableWriterOperator operator = (TableWriterOperator) createTableWriterOperator(pageSinkManager, new AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(0), Optional.empty())), true), outputTypes, session, driverContext);
    operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0));
    Object snapshot = operator.capture(operator.getOperatorContext().getDriverContext().getSerde());
    assertEquals(SnapshotTestUtil.toSimpleSnapshotMapping(snapshot), createExpectedMapping());
    operator.addInput(rowPagesBuilder(BIGINT).row(43).build().get(0));
    operator.restore(snapshot, operator.getOperatorContext().getDriverContext().getSerde());
    snapshot = operator.capture(operator.getOperatorContext().getDriverContext().getSerde());
    assertEquals(SnapshotTestUtil.toSimpleSnapshotMapping(snapshot), createExpectedMapping());
    operator.addInput(rowPagesBuilder(BIGINT).row(43).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertTrue(operator.needsInput());
    assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
    assertEquals(driverContext.getMemoryUsage(), 0);
    operator.finish();
    assertFalse(operator.isFinished());
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(null, null, 43).build().get(0));
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(2, null, null).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertFalse(operator.needsInput());
    assertTrue(operator.isFinished());
    operator.close();
    assertMemoryIsReleased(operator);
    TableWriterInfo info = operator.getInfo();
    assertThat(info.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertThat(info.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Type(io.prestosql.spi.type.Type) TableWriterInfo(io.prestosql.operator.TableWriterOperator.TableWriterInfo) AggregationOperatorFactory(io.prestosql.operator.AggregationOperator.AggregationOperatorFactory) PageSinkManager(io.prestosql.split.PageSinkManager) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) Session(io.prestosql.Session) Test(org.testng.annotations.Test)

Example 2 with PageSinkManager

use of io.prestosql.split.PageSinkManager in project hetu-core by openlookeng.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    Metadata metadata = createTestMetadataManager();
    PageSourceManager pageSourceManager = new PageSourceManager();
    HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    CubeManager cubeManager = new CubeManager(featuresConfig, hetuMetaStoreManager);
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
    NodeInfo nodeInfo = new NodeInfo("test");
    FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
    SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
    StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
    HeuristicIndexerManager heuristicIndexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
    return new LocalExecutionPlanner(metadata, new TypeAnalyzer(new SqlParser(), metadata), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new PagesIndex.TestingFactory(false), new JoinCompiler(metadata), new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
}
Also used : NodeInfo(io.airlift.node.NodeInfo) SqlParser(io.prestosql.sql.parser.SqlParser) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) TEST_TABLE_HANDLE(io.prestosql.testing.TestingHandles.TEST_TABLE_HANDLE) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) ObjectMapperProvider(io.airlift.json.ObjectMapperProvider) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) Partitioning(io.prestosql.sql.planner.Partitioning) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) MockExchangeClientSupplier(io.prestosql.execution.TestSqlTaskManager.MockExchangeClientSupplier) EventListenerManager(io.prestosql.eventlistener.EventListenerManager) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) ImmutableMap(com.google.common.collect.ImmutableMap) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) CatalogName(io.prestosql.spi.connector.CatalogName) TableScanNode(io.prestosql.spi.plan.TableScanNode) UUID(java.util.UUID) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) Metadata(io.prestosql.metadata.Metadata) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) List(java.util.List) SOURCE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION) PageSinkManager(io.prestosql.split.PageSinkManager) Optional(java.util.Optional) TEST_SESSION(io.prestosql.SessionTestUtils.TEST_SESSION) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) SINGLE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler) StatsAndCosts(io.prestosql.cost.StatsAndCosts) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) Split(io.prestosql.metadata.Split) OutputBuffers(io.prestosql.execution.buffer.OutputBuffers) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) OptionalInt(java.util.OptionalInt) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) TestingSplit(io.prestosql.testing.TestingSplit) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) PageSourceManager(io.prestosql.split.PageSourceManager) TestingColumnHandle(io.prestosql.testing.TestingMetadata.TestingColumnHandle) PagesIndex(io.prestosql.operator.PagesIndex) Symbol(io.prestosql.spi.plan.Symbol) PlanFragment(io.prestosql.sql.planner.PlanFragment) StageExecutionDescriptor.ungroupedExecution(io.prestosql.operator.StageExecutionDescriptor.ungroupedExecution) IndexManager(io.prestosql.index.IndexManager) SplitMonitor(io.prestosql.event.SplitMonitor) FinalizerService(io.prestosql.util.FinalizerService) CubeManager(io.prestosql.cube.CubeManager) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) Metadata(io.prestosql.metadata.Metadata) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) PagesIndex(io.prestosql.operator.PagesIndex) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) PageSourceManager(io.prestosql.split.PageSourceManager) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) PageSinkManager(io.prestosql.split.PageSinkManager) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) MockExchangeClientSupplier(io.prestosql.execution.TestSqlTaskManager.MockExchangeClientSupplier) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) SqlParser(io.prestosql.sql.parser.SqlParser) CubeManager(io.prestosql.cube.CubeManager) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) IndexManager(io.prestosql.index.IndexManager) FinalizerService(io.prestosql.util.FinalizerService) NodeInfo(io.airlift.node.NodeInfo) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler)

Example 3 with PageSinkManager

use of io.prestosql.split.PageSinkManager in project hetu-core by openlookeng.

the class TestTableWriterOperator method createTableWriterOperator.

private Operator createTableWriterOperator(BlockingPageSink blockingPageSink) {
    PageSinkManager pageSinkManager = new PageSinkManager();
    pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(blockingPageSink));
    return createTableWriterOperator(pageSinkManager, new DevNullOperatorFactory(1, new PlanNodeId("test")), ImmutableList.of(BIGINT, VARBINARY));
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) DevNullOperatorFactory(io.prestosql.operator.DevNullOperator.DevNullOperatorFactory) PageSinkManager(io.prestosql.split.PageSinkManager)

Example 4 with PageSinkManager

use of io.prestosql.split.PageSinkManager in project hetu-core by openlookeng.

the class TestTableWriterOperator method testStatisticsAggregation.

@Test
public void testStatisticsAggregation() throws Exception {
    PageSinkManager pageSinkManager = new PageSinkManager();
    pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(new TableWriteInfoTestPageSink()));
    ImmutableList<Type> outputTypes = ImmutableList.of(BIGINT, VARBINARY, BIGINT);
    Session session = testSessionBuilder().setSystemProperty("statistics_cpu_timer_enabled", "true").build();
    DriverContext driverContext = createTaskContext(executor, scheduledExecutor, session).addPipelineContext(0, true, true, false).addDriverContext();
    TableWriterOperator operator = (TableWriterOperator) createTableWriterOperator(pageSinkManager, new AggregationOperatorFactory(1, new PlanNodeId("test"), AggregationNode.Step.SINGLE, ImmutableList.of(LONG_MAX.bind(ImmutableList.of(0), Optional.empty())), true), outputTypes, session, driverContext);
    operator.addInput(rowPagesBuilder(BIGINT).row(42).build().get(0));
    operator.addInput(rowPagesBuilder(BIGINT).row(43).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertTrue(operator.needsInput());
    assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0);
    assertEquals(driverContext.getMemoryUsage(), 0);
    operator.finish();
    assertFalse(operator.isFinished());
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(null, null, 43).build().get(0));
    assertPageEquals(outputTypes, operator.getOutput(), rowPagesBuilder(outputTypes).row(2, null, null).build().get(0));
    assertTrue(operator.isBlocked().isDone());
    assertFalse(operator.needsInput());
    assertTrue(operator.isFinished());
    operator.close();
    assertMemoryIsReleased(operator);
    TableWriterInfo info = operator.getInfo();
    assertThat(info.getStatisticsWallTime().getValue(NANOSECONDS)).isGreaterThan(0);
    assertThat(info.getStatisticsCpuTime().getValue(NANOSECONDS)).isGreaterThan(0);
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Type(io.prestosql.spi.type.Type) TableWriterInfo(io.prestosql.operator.TableWriterOperator.TableWriterInfo) AggregationOperatorFactory(io.prestosql.operator.AggregationOperator.AggregationOperatorFactory) PageSinkManager(io.prestosql.split.PageSinkManager) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) Session(io.prestosql.Session) Test(org.testng.annotations.Test)

Example 5 with PageSinkManager

use of io.prestosql.split.PageSinkManager in project hetu-core by openlookeng.

the class TestTableWriterOperator method testTableWriterInfo.

@Test
public void testTableWriterInfo() {
    PageSinkManager pageSinkManager = new PageSinkManager();
    pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(new TableWriteInfoTestPageSink()));
    TableWriterOperator tableWriterOperator = (TableWriterOperator) createTableWriterOperator(pageSinkManager, new DevNullOperatorFactory(1, new PlanNodeId("test")), ImmutableList.of(BIGINT, VARBINARY));
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(BIGINT);
    for (int i = 0; i < 100; i++) {
        rowPagesBuilder.addSequencePage(100, 0);
    }
    List<Page> pages = rowPagesBuilder.build();
    long peakMemoryUsage = 0;
    long validationCpuNanos = 0;
    for (int i = 0; i < pages.size(); i++) {
        Page page = pages.get(i);
        peakMemoryUsage += page.getRetainedSizeInBytes();
        validationCpuNanos += page.getPositionCount();
        tableWriterOperator.addInput(page);
        TableWriterInfo info = tableWriterOperator.getInfo();
        assertEquals(info.getPageSinkPeakMemoryUsage(), peakMemoryUsage);
        assertEquals((long) (info.getValidationCpuTime().getValue(NANOSECONDS)), validationCpuNanos);
    }
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TableWriterInfo(io.prestosql.operator.TableWriterOperator.TableWriterInfo) RowPagesBuilder(io.prestosql.RowPagesBuilder) DevNullOperatorFactory(io.prestosql.operator.DevNullOperator.DevNullOperatorFactory) SerializedPage(io.hetu.core.transport.execution.buffer.SerializedPage) Page(io.prestosql.spi.Page) PageSinkManager(io.prestosql.split.PageSinkManager) Test(org.testng.annotations.Test)

Aggregations

PlanNodeId (io.prestosql.spi.plan.PlanNodeId)5 PageSinkManager (io.prestosql.split.PageSinkManager)5 TableWriterInfo (io.prestosql.operator.TableWriterOperator.TableWriterInfo)3 Session (io.prestosql.Session)2 AggregationOperatorFactory (io.prestosql.operator.AggregationOperator.AggregationOperatorFactory)2 DevNullOperatorFactory (io.prestosql.operator.DevNullOperator.DevNullOperatorFactory)2 ConnectorSession (io.prestosql.spi.connector.ConnectorSession)2 Type (io.prestosql.spi.type.Type)2 Test (org.testng.annotations.Test)2 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ObjectMapperProvider (io.airlift.json.ObjectMapperProvider)1 NodeInfo (io.airlift.node.NodeInfo)1 SerializedPage (io.hetu.core.transport.execution.buffer.SerializedPage)1 RowPagesBuilder (io.prestosql.RowPagesBuilder)1 TEST_SESSION (io.prestosql.SessionTestUtils.TEST_SESSION)1 StatsAndCosts (io.prestosql.cost.StatsAndCosts)1 CubeManager (io.prestosql.cube.CubeManager)1 DynamicFilterCacheManager (io.prestosql.dynamicfilter.DynamicFilterCacheManager)1 SplitMonitor (io.prestosql.event.SplitMonitor)1