Search in sources :

Example 1 with OutputFactory

use of io.prestosql.operator.OutputFactory in project hetu-core by openlookeng.

the class LocalQueryRunner method executeInternal.

private MaterializedResultWithPlan executeInternal(Session session, @Language("SQL") String sql) {
    lock.readLock().lock();
    try (Closer closer = Closer.create()) {
        AtomicReference<MaterializedResult.Builder> builder = new AtomicReference<>();
        PageConsumerOutputFactory outputFactory = new PageConsumerOutputFactory(types -> {
            builder.compareAndSet(null, MaterializedResult.resultBuilder(session, types));
            return builder.get()::page;
        });
        TaskContext taskContext = TestingTaskContext.builder(notificationExecutor, yieldExecutor, session).setMaxSpillSize(nodeSpillConfig.getMaxSpillPerNode()).setQueryMaxSpillSize(nodeSpillConfig.getQueryMaxSpillPerNode()).build();
        Plan plan = createPlan(session, sql, WarningCollector.NOOP);
        List<Driver> drivers = createDrivers(session, plan, outputFactory, taskContext);
        drivers.forEach(closer::register);
        boolean done = false;
        while (!done) {
            boolean processed = false;
            for (Driver driver : drivers) {
                if (alwaysRevokeMemory) {
                    driver.getDriverContext().getOperatorContexts().stream().filter(operatorContext -> operatorContext.getOperatorStats().getRevocableMemoryReservation().getValue() > 0).forEach(OperatorContext::requestMemoryRevoking);
                }
                if (!driver.isFinished()) {
                    driver.process();
                    processed = true;
                }
            }
            done = !processed;
        }
        verify(builder.get() != null, "Output operator was not created");
        return new MaterializedResultWithPlan(builder.get().build(), plan);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        lock.readLock().unlock();
    }
}
Also used : Closer(com.google.common.io.Closer) CostCalculatorWithEstimatedExchanges(io.prestosql.cost.CostCalculatorWithEstimatedExchanges) RollbackTask(io.prestosql.execution.RollbackTask) FilterStatsCalculator(io.prestosql.cost.FilterStatsCalculator) Plan(io.prestosql.sql.planner.Plan) ConnectorManager(io.prestosql.connector.ConnectorManager) HttpServerConfig(io.airlift.http.server.HttpServerConfig) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) Prepare(io.prestosql.sql.tree.Prepare) Statement(io.prestosql.sql.tree.Statement) WarningCollector(io.prestosql.execution.warnings.WarningCollector) SnapshotConfig(io.prestosql.snapshot.SnapshotConfig) SchemaPropertiesSystemTable(io.prestosql.connector.system.SchemaPropertiesSystemTable) NoOpResourceGroupManager(io.prestosql.execution.resourcegroups.NoOpResourceGroupManager) PlanSanityChecker(io.prestosql.sql.planner.sanity.PlanSanityChecker) Map(java.util.Map) TestingSession.createBogusTestingCatalog(io.prestosql.testing.TestingSession.createBogusTestingCatalog) TransactionBuilder.transaction(io.prestosql.transaction.TransactionBuilder.transaction) GlobalSystemConnector(io.prestosql.connector.system.GlobalSystemConnector) RenameColumn(io.prestosql.sql.tree.RenameColumn) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TransactionManagerConfig(io.prestosql.transaction.TransactionManagerConfig) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) SetSessionTask(io.prestosql.execution.SetSessionTask) CostCalculatorUsingExchanges(io.prestosql.cost.CostCalculatorUsingExchanges) PageConsumerOutputFactory(io.prestosql.testing.PageConsumerOperator.PageConsumerOutputFactory) CommentTask(io.prestosql.execution.CommentTask) PlanNode(io.prestosql.spi.plan.PlanNode) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) DropView(io.prestosql.sql.tree.DropView) PageSinkManager(io.prestosql.split.PageSinkManager) QueryPreparer(io.prestosql.execution.QueryPreparer) TaskManagerConfig(io.prestosql.execution.TaskManagerConfig) Plugin(io.prestosql.spi.Plugin) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) EmbedVersion(io.prestosql.version.EmbedVersion) DropTable(io.prestosql.sql.tree.DropTable) TransactionManager(io.prestosql.transaction.TransactionManager) StageExecutionDescriptor(io.prestosql.operator.StageExecutionDescriptor) TaskContext(io.prestosql.operator.TaskContext) PlanPrinter(io.prestosql.sql.planner.planprinter.PlanPrinter) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) CommitTask(io.prestosql.execution.CommitTask) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) ResetSession(io.prestosql.sql.tree.ResetSession) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) Deallocate(io.prestosql.sql.tree.Deallocate) ResetSessionTask(io.prestosql.execution.ResetSessionTask) MetadataUtil(io.prestosql.metadata.MetadataUtil) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ParsingUtil.createParsingOptions(io.prestosql.sql.ParsingUtil.createParsingOptions) NodeTaskMap(io.prestosql.execution.NodeTaskMap) SplitSource(io.prestosql.split.SplitSource) Comment(io.prestosql.sql.tree.Comment) SpillerFactory(io.prestosql.spiller.SpillerFactory) RenameTable(io.prestosql.sql.tree.RenameTable) DriverContext(io.prestosql.operator.DriverContext) TaskCountEstimator(io.prestosql.cost.TaskCountEstimator) Lifespan(io.prestosql.execution.Lifespan) StatsCalculatorModule.createNewStatsCalculator(io.prestosql.cost.StatsCalculatorModule.createNewStatsCalculator) CreateViewTask(io.prestosql.execution.CreateViewTask) IOException(java.io.IOException) GlobalSystemConnectorFactory(io.prestosql.connector.system.GlobalSystemConnectorFactory) DeallocateTask(io.prestosql.execution.DeallocateTask) CatalogManager(io.prestosql.metadata.CatalogManager) PlanOptimizer(io.prestosql.sql.planner.optimizations.PlanOptimizer) QueryExplainer(io.prestosql.sql.analyzer.QueryExplainer) SystemSessionProperties(io.prestosql.SystemSessionProperties) SqlParser(io.prestosql.sql.parser.SqlParser) QueryManagerConfig(io.prestosql.execution.QueryManagerConfig) Duration(io.airlift.units.Duration) TaskSource(io.prestosql.execution.TaskSource) HetuConfig(io.prestosql.utils.HetuConfig) ConnectorPlanOptimizerManager(io.prestosql.sql.planner.ConnectorPlanOptimizerManager) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SetPathTask(io.prestosql.execution.SetPathTask) StartTransaction(io.prestosql.sql.tree.StartTransaction) Type(io.prestosql.spi.type.Type) StatsCalculator(io.prestosql.cost.StatsCalculator) PartitioningSpillerFactory(io.prestosql.spiller.PartitioningSpillerFactory) ImmutableSet(com.google.common.collect.ImmutableSet) CatalogName(io.prestosql.spi.connector.CatalogName) GenericPartitioningSpillerFactory(io.prestosql.spiller.GenericPartitioningSpillerFactory) Rollback(io.prestosql.sql.tree.Rollback) RenameColumnTask(io.prestosql.execution.RenameColumnTask) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) NOT_PARTITIONED(io.prestosql.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) Split(io.prestosql.metadata.Split) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) PlanOptimizers(io.prestosql.sql.planner.PlanOptimizers) ImmutableList(com.google.common.collect.ImmutableList) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) PagesIndexPageSorter(io.prestosql.PagesIndexPageSorter) ExecutorService(java.util.concurrent.ExecutorService) IndexManager(io.prestosql.index.IndexManager) RenameIndex(io.prestosql.sql.tree.RenameIndex) PlanNodeSearcher.searchFrom(io.prestosql.sql.planner.optimizations.PlanNodeSearcher.searchFrom) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) SpillerStats(io.prestosql.spiller.SpillerStats) GROUPED_SCHEDULING(io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.GROUPED_SCHEDULING) InternalCommunicationConfig(io.prestosql.server.InternalCommunicationConfig) CostCalculator(io.prestosql.cost.CostCalculator) SubPlan(io.prestosql.sql.planner.SubPlan) NodeInfo(io.airlift.node.NodeInfo) RenameIndexTask(io.prestosql.execution.RenameIndexTask) NodeSystemTable(io.prestosql.connector.system.NodeSystemTable) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) CreateTable(io.prestosql.sql.tree.CreateTable) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) EventListenerManager(io.prestosql.eventlistener.EventListenerManager) PageIndexerFactory(io.prestosql.spi.PageIndexerFactory) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) UNGROUPED_SCHEDULING(io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) SchemaPropertyManager(io.prestosql.metadata.SchemaPropertyManager) FileSingleStreamSpillerFactory(io.prestosql.spiller.FileSingleStreamSpillerFactory) TableScanNode(io.prestosql.spi.plan.TableScanNode) TablePropertiesSystemTable(io.prestosql.connector.system.TablePropertiesSystemTable) Metadata(io.prestosql.metadata.Metadata) MetadataManager(io.prestosql.metadata.MetadataManager) LogicalPlanner(io.prestosql.sql.planner.LogicalPlanner) SetSession(io.prestosql.sql.tree.SetSession) UncheckedIOException(java.io.UncheckedIOException) HttpServerInfo(io.airlift.http.server.HttpServerInfo) CreateView(io.prestosql.sql.tree.CreateView) CatalogSystemTable(io.prestosql.connector.system.CatalogSystemTable) Driver(io.prestosql.operator.Driver) PluginManager(io.prestosql.server.PluginManager) PreparedQuery(io.prestosql.execution.QueryPreparer.PreparedQuery) StartTransactionTask(io.prestosql.execution.StartTransactionTask) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler) SetPath(io.prestosql.sql.tree.SetPath) TESTING_CATALOG(io.prestosql.testing.TestingSession.TESTING_CATALOG) DropTableTask(io.prestosql.execution.DropTableTask) DataDefinitionTask(io.prestosql.execution.DataDefinitionTask) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) InMemoryTransactionManager(io.prestosql.transaction.InMemoryTransactionManager) ArrayList(java.util.ArrayList) CatalogConnectorStore(io.prestosql.connector.CatalogConnectorStore) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) Closer(com.google.common.io.Closer) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) EmbeddedStateStoreLauncher(io.prestosql.statestore.EmbeddedStateStoreLauncher) Session(io.prestosql.Session) PageSourceManager(io.prestosql.split.PageSourceManager) ScheduledSplit(io.prestosql.execution.ScheduledSplit) SessionPropertyDefaults(io.prestosql.server.SessionPropertyDefaults) TransactionsSystemTable(io.prestosql.connector.system.TransactionsSystemTable) Language(org.intellij.lang.annotations.Language) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) CostComparator(io.prestosql.cost.CostComparator) FinalizerService(io.prestosql.util.FinalizerService) CubeManager(io.prestosql.cube.CubeManager) Lock(java.util.concurrent.locks.Lock) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) TreeAssertions.assertFormattedSql(io.prestosql.sql.testing.TreeAssertions.assertFormattedSql) HandleResolver(io.prestosql.metadata.HandleResolver) ConnectorFactory(io.prestosql.spi.connector.ConnectorFactory) SplitManager(io.prestosql.split.SplitManager) QualifiedTablePrefix(io.prestosql.metadata.QualifiedTablePrefix) TablePropertyManager(io.prestosql.metadata.TablePropertyManager) TestingMBeanServer(org.weakref.jmx.testing.TestingMBeanServer) StatsNormalizer(io.prestosql.cost.StatsNormalizer) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) Kryo(com.esotericsoftware.kryo.Kryo) AnalyzePropertiesSystemTable(io.prestosql.connector.system.AnalyzePropertiesSystemTable) ServerConfig(io.prestosql.server.ServerConfig) ColumnPropertyManager(io.prestosql.metadata.ColumnPropertyManager) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) Commit(io.prestosql.sql.tree.Commit) ImmutableMap(com.google.common.collect.ImmutableMap) OutputFactory(io.prestosql.operator.OutputFactory) Preconditions.checkState(com.google.common.base.Preconditions.checkState) PluginManagerConfig(io.prestosql.server.PluginManagerConfig) List(java.util.List) ColumnPropertiesSystemTable(io.prestosql.connector.system.ColumnPropertiesSystemTable) TableCommentSystemTable(io.prestosql.connector.system.TableCommentSystemTable) SessionPropertyManager(io.prestosql.metadata.SessionPropertyManager) Optional(java.util.Optional) Analysis(io.prestosql.sql.analyzer.Analysis) NodeSpillConfig(io.prestosql.spiller.NodeSpillConfig) FunctionAndTypeManager(io.prestosql.metadata.FunctionAndTypeManager) MemoryManagerConfig(io.prestosql.memory.MemoryManagerConfig) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) CreateTableTask(io.prestosql.execution.CreateTableTask) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) PlanFragmenter(io.prestosql.sql.planner.PlanFragmenter) DropViewTask(io.prestosql.execution.DropViewTask) PrepareTask(io.prestosql.execution.PrepareTask) Verify.verify(com.google.common.base.Verify.verify) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) Objects.requireNonNull(java.util.Objects.requireNonNull) PasswordAuthenticatorManager(io.prestosql.server.security.PasswordAuthenticatorManager) PagesIndex(io.prestosql.operator.PagesIndex) ScalarStatsCalculator(io.prestosql.cost.ScalarStatsCalculator) OperatorContext(io.prestosql.operator.OperatorContext) DriverFactory(io.prestosql.operator.DriverFactory) AnalyzePropertyManager(io.prestosql.metadata.AnalyzePropertyManager) GroupProviderManager(io.prestosql.security.GroupProviderManager) PageSorter(io.prestosql.spi.PageSorter) LocalExecutionPlan(io.prestosql.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) Analyzer(io.prestosql.sql.analyzer.Analyzer) TimeUnit(java.util.concurrent.TimeUnit) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) RenameTableTask(io.prestosql.execution.RenameTableTask) MBeanExporter(org.weakref.jmx.MBeanExporter) Collections(java.util.Collections) TaskContext(io.prestosql.operator.TaskContext) PageConsumerOutputFactory(io.prestosql.testing.PageConsumerOperator.PageConsumerOutputFactory) Driver(io.prestosql.operator.Driver) AtomicReference(java.util.concurrent.atomic.AtomicReference) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) Plan(io.prestosql.sql.planner.Plan) SubPlan(io.prestosql.sql.planner.SubPlan) LocalExecutionPlan(io.prestosql.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) OperatorContext(io.prestosql.operator.OperatorContext)

Example 2 with OutputFactory

use of io.prestosql.operator.OutputFactory in project hetu-core by openlookeng.

the class TestMemoryPools method setupConsumeRevocableMemory.

private RevocableMemoryOperator setupConsumeRevocableMemory(DataSize reservedPerPage, long numberOfPages) {
    AtomicReference<RevocableMemoryOperator> createOperator = new AtomicReference<>();
    setUp(() -> {
        DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext();
        OperatorContext revokableOperatorContext = driverContext.addOperatorContext(Integer.MAX_VALUE, new PlanNodeId("revokable_operator"), TableScanOperator.class.getSimpleName());
        OutputFactory outputFactory = new PageConsumerOutputFactory(types -> (page -> {
        }));
        Operator outputOperator = outputFactory.createOutputOperator(2, new PlanNodeId("output"), ImmutableList.of(), Function.identity(), driverContext.getPipelineContext().getTaskContext()).createOperator(driverContext);
        RevocableMemoryOperator revocableMemoryOperator = new RevocableMemoryOperator(revokableOperatorContext, reservedPerPage, numberOfPages);
        createOperator.set(revocableMemoryOperator);
        Driver driver = Driver.createDriver(driverContext, revocableMemoryOperator, outputOperator);
        return ImmutableList.of(driver);
    });
    return createOperator.get();
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Operator(io.prestosql.operator.Operator) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) Assert.assertNull(org.testng.Assert.assertNull) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) Supplier(com.google.common.base.Supplier) TpchConnectorFactory(io.prestosql.plugin.tpch.TpchConnectorFactory) SpillSpaceTracker(io.prestosql.spiller.SpillSpaceTracker) TaskContext(io.prestosql.operator.TaskContext) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) RestorableConfig(io.prestosql.spi.snapshot.RestorableConfig) AfterMethod(org.testng.annotations.AfterMethod) LocalQueryRunner.queryRunnerWithInitialTransaction(io.prestosql.testing.LocalQueryRunner.queryRunnerWithInitialTransaction) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) TestingSession.testSessionBuilder(io.prestosql.testing.TestingSession.testSessionBuilder) GIGABYTE(io.airlift.units.DataSize.Unit.GIGABYTE) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) Session(io.prestosql.Session) QueryId(io.prestosql.spi.QueryId) Assert.assertFalse(org.testng.Assert.assertFalse) DriverContext(io.prestosql.operator.DriverContext) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TableScanOperator(io.prestosql.operator.TableScanOperator) LocalMemoryContext(io.prestosql.memory.context.LocalMemoryContext) OperatorContext(io.prestosql.operator.OperatorContext) ImmutableMap(com.google.common.collect.ImmutableMap) Assert.fail(org.testng.Assert.fail) PageConsumerOutputFactory(io.prestosql.testing.PageConsumerOperator.PageConsumerOutputFactory) NOOP_SNAPSHOT_UTILS(io.prestosql.testing.TestingSnapshotUtils.NOOP_SNAPSHOT_UTILS) Page(io.prestosql.spi.Page) OutputFactory(io.prestosql.operator.OutputFactory) MemoryPoolId(io.prestosql.spi.memory.MemoryPoolId) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) AtomicLong(java.util.concurrent.atomic.AtomicLong) Futures(com.google.common.util.concurrent.Futures) DataSize(io.airlift.units.DataSize) LocalQueryRunner(io.prestosql.testing.LocalQueryRunner) List(java.util.List) TestingTaskContext.createTaskContext(io.prestosql.testing.TestingTaskContext.createTaskContext) Predicate(com.google.common.base.Predicate) Driver(io.prestosql.operator.Driver) Assert.assertTrue(org.testng.Assert.assertTrue) BYTE(io.airlift.units.DataSize.Unit.BYTE) TestingGcMonitor(io.airlift.stats.TestingGcMonitor) Operator(io.prestosql.operator.Operator) TableScanOperator(io.prestosql.operator.TableScanOperator) DriverContext(io.prestosql.operator.DriverContext) TableScanOperator(io.prestosql.operator.TableScanOperator) PageConsumerOutputFactory(io.prestosql.testing.PageConsumerOperator.PageConsumerOutputFactory) OperatorContext(io.prestosql.operator.OperatorContext) Driver(io.prestosql.operator.Driver) AtomicReference(java.util.concurrent.atomic.AtomicReference) PageConsumerOutputFactory(io.prestosql.testing.PageConsumerOperator.PageConsumerOutputFactory) OutputFactory(io.prestosql.operator.OutputFactory)

Example 3 with OutputFactory

use of io.prestosql.operator.OutputFactory in project boostkit-bigdata by kunpengcompute.

the class OmniLocalQueryRunner method executeInternal.

private MaterializedResultWithPlan executeInternal(Session session, @Language("SQL") String sql) {
    lock.readLock().lock();
    try (Closer closer = Closer.create()) {
        AtomicReference<MaterializedResult.Builder> builder = new AtomicReference<>();
        PageConsumerOutputFactory outputFactory = new PageConsumerOutputFactory(types -> {
            builder.compareAndSet(null, MaterializedResult.resultBuilder(session, types));
            return builder.get()::page;
        });
        TaskContext taskContext = TestingTaskContext.builder(notificationExecutor, yieldExecutor, session).setMaxSpillSize(nodeSpillConfig.getMaxSpillPerNode()).setQueryMaxSpillSize(nodeSpillConfig.getQueryMaxSpillPerNode()).build();
        Plan plan = createPlan(session, sql, WarningCollector.NOOP);
        List<Driver> drivers = createDrivers(session, plan, outputFactory, taskContext);
        drivers.forEach(closer::register);
        boolean done = false;
        while (!done) {
            boolean processed = false;
            for (Driver driver : drivers) {
                if (alwaysRevokeMemory) {
                    driver.getDriverContext().getOperatorContexts().stream().filter(operatorContext -> operatorContext.getOperatorStats().getRevocableMemoryReservation().getValue() > 0).forEach(OperatorContext::requestMemoryRevoking);
                }
                if (!driver.isFinished()) {
                    driver.process();
                    processed = true;
                }
            }
            done = !processed;
        }
        verify(builder.get() != null, "Output operator was not created");
        return new MaterializedResultWithPlan(builder.get().build(), plan);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        lock.readLock().unlock();
    }
}
Also used : Closer(com.google.common.io.Closer) CostCalculatorWithEstimatedExchanges(io.prestosql.cost.CostCalculatorWithEstimatedExchanges) RollbackTask(io.prestosql.execution.RollbackTask) FilterStatsCalculator(io.prestosql.cost.FilterStatsCalculator) Plan(io.prestosql.sql.planner.Plan) ConnectorManager(io.prestosql.connector.ConnectorManager) HttpServerConfig(io.airlift.http.server.HttpServerConfig) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) Prepare(io.prestosql.sql.tree.Prepare) Statement(io.prestosql.sql.tree.Statement) WarningCollector(io.prestosql.execution.warnings.WarningCollector) SnapshotConfig(io.prestosql.snapshot.SnapshotConfig) SchemaPropertiesSystemTable(io.prestosql.connector.system.SchemaPropertiesSystemTable) OPEN(io.prestosql.execution.buffer.BufferState.OPEN) NoOpResourceGroupManager(io.prestosql.execution.resourcegroups.NoOpResourceGroupManager) PlanSanityChecker(io.prestosql.sql.planner.sanity.PlanSanityChecker) Map(java.util.Map) PARTITIONED(io.prestosql.execution.buffer.OutputBuffers.BufferType.PARTITIONED) Partitioning(io.prestosql.sql.planner.Partitioning) TestingSession.createBogusTestingCatalog(io.prestosql.testing.TestingSession.createBogusTestingCatalog) TransactionBuilder.transaction(io.prestosql.transaction.TransactionBuilder.transaction) GlobalSystemConnector(io.prestosql.connector.system.GlobalSystemConnector) RenameColumn(io.prestosql.sql.tree.RenameColumn) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TransactionManagerConfig(io.prestosql.transaction.TransactionManagerConfig) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) SetSessionTask(io.prestosql.execution.SetSessionTask) CostCalculatorUsingExchanges(io.prestosql.cost.CostCalculatorUsingExchanges) PageConsumerOutputFactory(io.prestosql.testing.PageConsumerOperator.PageConsumerOutputFactory) CommentTask(io.prestosql.execution.CommentTask) PlanNode(io.prestosql.spi.plan.PlanNode) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) Threads.threadsNamed(io.airlift.concurrent.Threads.threadsNamed) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) DropView(io.prestosql.sql.tree.DropView) PageSinkManager(io.prestosql.split.PageSinkManager) QueryPreparer(io.prestosql.execution.QueryPreparer) TaskManagerConfig(io.prestosql.execution.TaskManagerConfig) Plugin(io.prestosql.spi.Plugin) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) EmbedVersion(io.prestosql.version.EmbedVersion) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) DropTable(io.prestosql.sql.tree.DropTable) TransactionManager(io.prestosql.transaction.TransactionManager) StageExecutionDescriptor(io.prestosql.operator.StageExecutionDescriptor) TaskContext(io.prestosql.operator.TaskContext) PlanPrinter(io.prestosql.sql.planner.planprinter.PlanPrinter) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) CommitTask(io.prestosql.execution.CommitTask) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) ResetSession(io.prestosql.sql.tree.ResetSession) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) Deallocate(io.prestosql.sql.tree.Deallocate) ResetSessionTask(io.prestosql.execution.ResetSessionTask) MetadataUtil(io.prestosql.metadata.MetadataUtil) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ParsingUtil.createParsingOptions(io.prestosql.sql.ParsingUtil.createParsingOptions) NodeTaskMap(io.prestosql.execution.NodeTaskMap) SplitSource(io.prestosql.split.SplitSource) Comment(io.prestosql.sql.tree.Comment) SpillerFactory(io.prestosql.spiller.SpillerFactory) RenameTable(io.prestosql.sql.tree.RenameTable) DriverContext(io.prestosql.operator.DriverContext) TaskCountEstimator(io.prestosql.cost.TaskCountEstimator) Lifespan(io.prestosql.execution.Lifespan) StatsCalculatorModule.createNewStatsCalculator(io.prestosql.cost.StatsCalculatorModule.createNewStatsCalculator) CreateViewTask(io.prestosql.execution.CreateViewTask) IOException(java.io.IOException) GlobalSystemConnectorFactory(io.prestosql.connector.system.GlobalSystemConnectorFactory) DeallocateTask(io.prestosql.execution.DeallocateTask) CatalogManager(io.prestosql.metadata.CatalogManager) PlanOptimizer(io.prestosql.sql.planner.optimizations.PlanOptimizer) OutputBuffers.createInitialEmptyOutputBuffers(io.prestosql.execution.buffer.OutputBuffers.createInitialEmptyOutputBuffers) QueryExplainer(io.prestosql.sql.analyzer.QueryExplainer) SystemSessionProperties(io.prestosql.SystemSessionProperties) SqlParser(io.prestosql.sql.parser.SqlParser) QueryManagerConfig(io.prestosql.execution.QueryManagerConfig) MaterializedResult(io.prestosql.testing.MaterializedResult) Duration(io.airlift.units.Duration) TaskSource(io.prestosql.execution.TaskSource) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) HetuConfig(io.prestosql.utils.HetuConfig) TestingAccessControlManager(io.prestosql.testing.TestingAccessControlManager) ConnectorPlanOptimizerManager(io.prestosql.sql.planner.ConnectorPlanOptimizerManager) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SetPathTask(io.prestosql.execution.SetPathTask) StartTransaction(io.prestosql.sql.tree.StartTransaction) Type(io.prestosql.spi.type.Type) StatsCalculator(io.prestosql.cost.StatsCalculator) PartitioningSpillerFactory(io.prestosql.spiller.PartitioningSpillerFactory) ImmutableSet(com.google.common.collect.ImmutableSet) CatalogName(io.prestosql.spi.connector.CatalogName) QueryRunner(io.prestosql.testing.QueryRunner) GenericPartitioningSpillerFactory(io.prestosql.spiller.GenericPartitioningSpillerFactory) Rollback(io.prestosql.sql.tree.Rollback) RenameColumnTask(io.prestosql.execution.RenameColumnTask) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.prestosql.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) NOT_PARTITIONED(io.prestosql.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) Split(io.prestosql.metadata.Split) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) PlanOptimizers(io.prestosql.sql.planner.PlanOptimizers) ImmutableList(com.google.common.collect.ImmutableList) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) PagesIndexPageSorter(io.prestosql.PagesIndexPageSorter) ExecutorService(java.util.concurrent.ExecutorService) IndexManager(io.prestosql.index.IndexManager) RenameIndex(io.prestosql.sql.tree.RenameIndex) PlanNodeSearcher.searchFrom(io.prestosql.sql.planner.optimizations.PlanNodeSearcher.searchFrom) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) SpillerStats(io.prestosql.spiller.SpillerStats) OutputBuffer(io.prestosql.execution.buffer.OutputBuffer) GROUPED_SCHEDULING(io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.GROUPED_SCHEDULING) InternalCommunicationConfig(io.prestosql.server.InternalCommunicationConfig) CostCalculator(io.prestosql.cost.CostCalculator) SubPlan(io.prestosql.sql.planner.SubPlan) NodeInfo(io.airlift.node.NodeInfo) RenameIndexTask(io.prestosql.execution.RenameIndexTask) NodeSystemTable(io.prestosql.connector.system.NodeSystemTable) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) CreateTable(io.prestosql.sql.tree.CreateTable) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) EventListenerManager(io.prestosql.eventlistener.EventListenerManager) PageIndexerFactory(io.prestosql.spi.PageIndexerFactory) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) UNGROUPED_SCHEDULING(io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) PartitionedOutputBuffer(io.prestosql.execution.buffer.PartitionedOutputBuffer) SchemaPropertyManager(io.prestosql.metadata.SchemaPropertyManager) FileSingleStreamSpillerFactory(io.prestosql.spiller.FileSingleStreamSpillerFactory) TableScanNode(io.prestosql.spi.plan.TableScanNode) TablePropertiesSystemTable(io.prestosql.connector.system.TablePropertiesSystemTable) Metadata(io.prestosql.metadata.Metadata) MetadataManager(io.prestosql.metadata.MetadataManager) LogicalPlanner(io.prestosql.sql.planner.LogicalPlanner) SetSession(io.prestosql.sql.tree.SetSession) UncheckedIOException(java.io.UncheckedIOException) HttpServerInfo(io.airlift.http.server.HttpServerInfo) CreateView(io.prestosql.sql.tree.CreateView) CatalogSystemTable(io.prestosql.connector.system.CatalogSystemTable) Driver(io.prestosql.operator.Driver) TEST_SESSION(io.prestosql.SessionTestUtils.TEST_SESSION) PluginManager(io.prestosql.server.PluginManager) FIXED_HASH_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION) PreparedQuery(io.prestosql.execution.QueryPreparer.PreparedQuery) StartTransactionTask(io.prestosql.execution.StartTransactionTask) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler) SetPath(io.prestosql.sql.tree.SetPath) TESTING_CATALOG(io.prestosql.testing.TestingSession.TESTING_CATALOG) DropTableTask(io.prestosql.execution.DropTableTask) DataDefinitionTask(io.prestosql.execution.DataDefinitionTask) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) InMemoryTransactionManager(io.prestosql.transaction.InMemoryTransactionManager) ArrayList(java.util.ArrayList) CatalogConnectorStore(io.prestosql.connector.CatalogConnectorStore) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) Closer(com.google.common.io.Closer) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) EmbeddedStateStoreLauncher(io.prestosql.statestore.EmbeddedStateStoreLauncher) Session(io.prestosql.Session) PageSourceManager(io.prestosql.split.PageSourceManager) ScheduledSplit(io.prestosql.execution.ScheduledSplit) SessionPropertyDefaults(io.prestosql.server.SessionPropertyDefaults) TransactionsSystemTable(io.prestosql.connector.system.TransactionsSystemTable) Language(org.intellij.lang.annotations.Language) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) CostComparator(io.prestosql.cost.CostComparator) FinalizerService(io.prestosql.util.FinalizerService) CubeManager(io.prestosql.cube.CubeManager) Lock(java.util.concurrent.locks.Lock) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) TreeAssertions.assertFormattedSql(io.prestosql.sql.testing.TreeAssertions.assertFormattedSql) HandleResolver(io.prestosql.metadata.HandleResolver) ConnectorFactory(io.prestosql.spi.connector.ConnectorFactory) SplitManager(io.prestosql.split.SplitManager) QualifiedTablePrefix(io.prestosql.metadata.QualifiedTablePrefix) TablePropertyManager(io.prestosql.metadata.TablePropertyManager) TestingMBeanServer(org.weakref.jmx.testing.TestingMBeanServer) StatsNormalizer(io.prestosql.cost.StatsNormalizer) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) Kryo(com.esotericsoftware.kryo.Kryo) AnalyzePropertiesSystemTable(io.prestosql.connector.system.AnalyzePropertiesSystemTable) ServerConfig(io.prestosql.server.ServerConfig) ColumnPropertyManager(io.prestosql.metadata.ColumnPropertyManager) SimpleLocalMemoryContext(io.prestosql.memory.context.SimpleLocalMemoryContext) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) Commit(io.prestosql.sql.tree.Commit) ImmutableMap(com.google.common.collect.ImmutableMap) OutputFactory(io.prestosql.operator.OutputFactory) Preconditions.checkState(com.google.common.base.Preconditions.checkState) PluginManagerConfig(io.prestosql.server.PluginManagerConfig) DataSize(io.airlift.units.DataSize) List(java.util.List) ColumnPropertiesSystemTable(io.prestosql.connector.system.ColumnPropertiesSystemTable) TableCommentSystemTable(io.prestosql.connector.system.TableCommentSystemTable) SessionPropertyManager(io.prestosql.metadata.SessionPropertyManager) Optional(java.util.Optional) Analysis(io.prestosql.sql.analyzer.Analysis) NodeSpillConfig(io.prestosql.spiller.NodeSpillConfig) FunctionAndTypeManager(io.prestosql.metadata.FunctionAndTypeManager) MemoryManagerConfig(io.prestosql.memory.MemoryManagerConfig) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) CreateTableTask(io.prestosql.execution.CreateTableTask) HashMap(java.util.HashMap) OutputBuffers(io.prestosql.execution.buffer.OutputBuffers) AtomicReference(java.util.concurrent.atomic.AtomicReference) PlanFragmenter(io.prestosql.sql.planner.PlanFragmenter) DropViewTask(io.prestosql.execution.DropViewTask) PrepareTask(io.prestosql.execution.PrepareTask) Verify.verify(com.google.common.base.Verify.verify) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) Objects.requireNonNull(java.util.Objects.requireNonNull) PasswordAuthenticatorManager(io.prestosql.server.security.PasswordAuthenticatorManager) PagesIndex(io.prestosql.operator.PagesIndex) TERMINAL_BUFFER_STATES(io.prestosql.execution.buffer.BufferState.TERMINAL_BUFFER_STATES) ScalarStatsCalculator(io.prestosql.cost.ScalarStatsCalculator) OperatorContext(io.prestosql.operator.OperatorContext) DriverFactory(io.prestosql.operator.DriverFactory) AnalyzePropertyManager(io.prestosql.metadata.AnalyzePropertyManager) GroupProviderManager(io.prestosql.security.GroupProviderManager) TestingTaskContext(io.prestosql.testing.TestingTaskContext) PageSorter(io.prestosql.spi.PageSorter) StateMachine(io.prestosql.execution.StateMachine) LocalExecutionPlan(io.prestosql.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) Analyzer(io.prestosql.sql.analyzer.Analyzer) TimeUnit(java.util.concurrent.TimeUnit) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) RenameTableTask(io.prestosql.execution.RenameTableTask) MBeanExporter(org.weakref.jmx.MBeanExporter) Collections(java.util.Collections) TaskContext(io.prestosql.operator.TaskContext) TestingTaskContext(io.prestosql.testing.TestingTaskContext) PageConsumerOutputFactory(io.prestosql.testing.PageConsumerOperator.PageConsumerOutputFactory) Driver(io.prestosql.operator.Driver) AtomicReference(java.util.concurrent.atomic.AtomicReference) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) Plan(io.prestosql.sql.planner.Plan) SubPlan(io.prestosql.sql.planner.SubPlan) LocalExecutionPlan(io.prestosql.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) OperatorContext(io.prestosql.operator.OperatorContext)

Aggregations

Preconditions.checkState (com.google.common.base.Preconditions.checkState)3 ImmutableList (com.google.common.collect.ImmutableList)3 ImmutableMap (com.google.common.collect.ImmutableMap)3 Session (io.prestosql.Session)3 Kryo (com.esotericsoftware.kryo.Kryo)2 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)2 Verify.verify (com.google.common.base.Verify.verify)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 Closer (com.google.common.io.Closer)2 MoreFutures.getFutureValue (io.airlift.concurrent.MoreFutures.getFutureValue)2 Threads.daemonThreadsNamed (io.airlift.concurrent.Threads.daemonThreadsNamed)2 HttpServerConfig (io.airlift.http.server.HttpServerConfig)2 HttpServerInfo (io.airlift.http.server.HttpServerInfo)2 NodeInfo (io.airlift.node.NodeInfo)2 DataSize (io.airlift.units.DataSize)2 MEGABYTE (io.airlift.units.DataSize.Unit.MEGABYTE)2 Duration (io.airlift.units.Duration)2 GroupByHashPageIndexerFactory (io.prestosql.GroupByHashPageIndexerFactory)2 PagesIndexPageSorter (io.prestosql.PagesIndexPageSorter)2 SystemSessionProperties (io.prestosql.SystemSessionProperties)2