Search in sources :

Example 1 with Plan

use of io.trino.sql.planner.Plan in project trino by trinodb.

the class SqlQueryExecution method doPlanQuery.

private PlanRoot doPlanQuery() {
    // plan query
    PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
    LogicalPlanner logicalPlanner = new LogicalPlanner(stateMachine.getSession(), planOptimizers, idAllocator, plannerContext, typeAnalyzer, statsCalculator, costCalculator, stateMachine.getWarningCollector());
    Plan plan = logicalPlanner.plan(analysis);
    queryPlan.set(plan);
    // fragment the plan
    SubPlan fragmentedPlan = planFragmenter.createSubPlans(stateMachine.getSession(), plan, false, stateMachine.getWarningCollector());
    // extract inputs
    List<Input> inputs = new InputExtractor(plannerContext.getMetadata(), stateMachine.getSession()).extractInputs(fragmentedPlan);
    stateMachine.setInputs(inputs);
    stateMachine.setOutput(analysis.getTarget());
    boolean explainAnalyze = analysis.getStatement() instanceof ExplainAnalyze;
    return new PlanRoot(fragmentedPlan, !explainAnalyze);
}
Also used : LogicalPlanner(io.trino.sql.planner.LogicalPlanner) PlanNodeIdAllocator(io.trino.sql.planner.PlanNodeIdAllocator) ExplainAnalyze(io.trino.sql.tree.ExplainAnalyze) InputExtractor(io.trino.sql.planner.InputExtractor) SubPlan(io.trino.sql.planner.SubPlan) Plan(io.trino.sql.planner.Plan) SubPlan(io.trino.sql.planner.SubPlan)

Example 2 with Plan

use of io.trino.sql.planner.Plan in project trino by trinodb.

the class LocalQueryRunner method executeInternal.

private MaterializedResultWithPlan executeInternal(Session session, @Language("SQL") String sql) {
    lock.readLock().lock();
    try (Closer closer = Closer.create()) {
        accessControl.checkCanExecuteQuery(session.getIdentity());
        AtomicReference<MaterializedResult.Builder> builder = new AtomicReference<>();
        PageConsumerOutputFactory outputFactory = new PageConsumerOutputFactory(types -> {
            builder.compareAndSet(null, MaterializedResult.resultBuilder(session, types));
            return builder.get()::page;
        });
        TaskContext taskContext = TestingTaskContext.builder(notificationExecutor, yieldExecutor, session).setMaxSpillSize(nodeSpillConfig.getMaxSpillPerNode()).setQueryMaxSpillSize(nodeSpillConfig.getQueryMaxSpillPerNode()).build();
        Plan plan = createPlan(session, sql, WarningCollector.NOOP);
        List<Driver> drivers = createDrivers(session, plan, outputFactory, taskContext);
        drivers.forEach(closer::register);
        boolean done = false;
        while (!done) {
            boolean processed = false;
            for (Driver driver : drivers) {
                if (alwaysRevokeMemory) {
                    driver.getDriverContext().getOperatorContexts().stream().filter(operatorContext -> operatorContext.getNestedOperatorStats().stream().mapToLong(stats -> stats.getRevocableMemoryReservation().toBytes()).sum() > 0).forEach(OperatorContext::requestMemoryRevoking);
                }
                if (!driver.isFinished()) {
                    driver.process();
                    processed = true;
                }
            }
            done = !processed;
        }
        verify(builder.get() != null, "Output operator was not created");
        return new MaterializedResultWithPlan(builder.get().build(), plan);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        lock.readLock().unlock();
    }
}
Also used : Closer(com.google.common.io.Closer) ConnectorManager(io.trino.connector.ConnectorManager) PagesIndexPageSorter(io.trino.operator.PagesIndexPageSorter) TransactionManager(io.trino.transaction.TransactionManager) TransactionManagerConfig(io.trino.transaction.TransactionManagerConfig) FilterStatsCalculator(io.trino.cost.FilterStatsCalculator) FinalizerService(io.trino.util.FinalizerService) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) MaterializedViewPropertiesSystemTable(io.trino.connector.system.MaterializedViewPropertiesSystemTable) HandleResolver(io.trino.metadata.HandleResolver) Map(java.util.Map) MetadataManager(io.trino.metadata.MetadataManager) CostCalculatorUsingExchanges(io.trino.cost.CostCalculatorUsingExchanges) TableScanNode(io.trino.sql.planner.plan.TableScanNode) InMemoryTransactionManager(io.trino.transaction.InMemoryTransactionManager) InjectedFailureType(io.trino.execution.FailureInjector.InjectedFailureType) SplitSource(io.trino.split.SplitSource) SystemSessionPropertiesProvider(io.trino.SystemSessionPropertiesProvider) PreparedQuery(io.trino.execution.QueryPreparer.PreparedQuery) PlanSanityChecker(io.trino.sql.planner.sanity.PlanSanityChecker) Plugin(io.trino.spi.Plugin) NOT_PARTITIONED(io.trino.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) InternalBlockEncodingSerde(io.trino.metadata.InternalBlockEncodingSerde) SubPlan(io.trino.sql.planner.SubPlan) ShowStatsRewrite(io.trino.sql.rewrite.ShowStatsRewrite) Split(io.trino.metadata.Split) InMemoryNodeManager(io.trino.metadata.InMemoryNodeManager) Session(io.trino.Session) Analyzer(io.trino.sql.analyzer.Analyzer) MaterializedViewSystemTable(io.trino.connector.system.MaterializedViewSystemTable) NodePartitioningManager(io.trino.sql.planner.NodePartitioningManager) PluginManager(io.trino.server.PluginManager) RuleStatsRecorder(io.trino.sql.planner.RuleStatsRecorder) ColumnPropertiesSystemTable(io.trino.connector.system.ColumnPropertiesSystemTable) ComposableStatsCalculator(io.trino.cost.ComposableStatsCalculator) PageIndexerFactory(io.trino.spi.PageIndexerFactory) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) Lifespan(io.trino.execution.Lifespan) StageExecutionDescriptor(io.trino.operator.StageExecutionDescriptor) LocalExecutionPlan(io.trino.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) SpillerStats(io.trino.spiller.SpillerStats) IOException(java.io.IOException) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) FunctionBundle(io.trino.metadata.FunctionBundle) SessionPropertyDefaults(io.trino.server.SessionPropertyDefaults) TableHandle(io.trino.metadata.TableHandle) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) EventListenerManager(io.trino.eventlistener.EventListenerManager) InternalFunctionBundle(io.trino.metadata.InternalFunctionBundle) StatementAnalyzerFactory(io.trino.sql.analyzer.StatementAnalyzerFactory) TablePropertyManager(io.trino.metadata.TablePropertyManager) TransactionId(io.trino.transaction.TransactionId) Driver(io.trino.operator.Driver) AllowAllSystemAccessControl(io.trino.plugin.base.security.AllowAllSystemAccessControl) OptimizerConfig(io.trino.sql.planner.OptimizerConfig) AnalyzePropertiesSystemTable(io.trino.connector.system.AnalyzePropertiesSystemTable) ParsingUtil.createParsingOptions(io.trino.sql.ParsingUtil.createParsingOptions) StatementRewrite(io.trino.sql.rewrite.StatementRewrite) TransactionBuilder.transaction(io.trino.transaction.TransactionBuilder.transaction) NodeTaskMap(io.trino.execution.NodeTaskMap) JoinFilterFunctionCompiler(io.trino.sql.gen.JoinFilterFunctionCompiler) InternalTypeManager(io.trino.type.InternalTypeManager) NodeSpillConfig(io.trino.spiller.NodeSpillConfig) Duration(io.airlift.units.Duration) StatsCalculator(io.trino.cost.StatsCalculator) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) DisabledSystemSecurityMetadata(io.trino.metadata.DisabledSystemSecurityMetadata) TreeAssertions.assertFormattedSql(io.trino.sql.testing.TreeAssertions.assertFormattedSql) PageSorter(io.trino.spi.PageSorter) LiteralFunction(io.trino.metadata.LiteralFunction) DescribeInputRewrite(io.trino.sql.rewrite.DescribeInputRewrite) ImmutableSet(com.google.common.collect.ImmutableSet) GroupProviderManager(io.trino.security.GroupProviderManager) NodeSystemTable(io.trino.connector.system.NodeSystemTable) CostComparator(io.trino.cost.CostComparator) Constraint.alwaysTrue(io.trino.spi.connector.Constraint.alwaysTrue) PlanOptimizers(io.trino.sql.planner.PlanOptimizers) ColumnPropertyManager(io.trino.metadata.ColumnPropertyManager) GlobalSystemConnectorFactory(io.trino.connector.system.GlobalSystemConnectorFactory) PlannerContext(io.trino.sql.PlannerContext) Analysis(io.trino.sql.analyzer.Analysis) GlobalFunctionCatalog(io.trino.metadata.GlobalFunctionCatalog) TableCommentSystemTable(io.trino.connector.system.TableCommentSystemTable) FileSingleStreamSpillerFactory(io.trino.spiller.FileSingleStreamSpillerFactory) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) Function(java.util.function.Function) ExpressionCompiler(io.trino.sql.gen.ExpressionCompiler) GROUPED_SCHEDULING(io.trino.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.GROUPED_SCHEDULING) ImmutableList(com.google.common.collect.ImmutableList) SystemSessionProperties(io.trino.SystemSessionProperties) TaskContext(io.trino.operator.TaskContext) DynamicFilterConfig(io.trino.execution.DynamicFilterConfig) IndexManager(io.trino.index.IndexManager) ExecutorService(java.util.concurrent.ExecutorService) PagesIndex(io.trino.operator.PagesIndex) TaskCountEstimator(io.trino.cost.TaskCountEstimator) SchemaPropertyManager(io.trino.metadata.SchemaPropertyManager) MetadataUtil(io.trino.metadata.MetadataUtil) TableExecuteContextManager(io.trino.execution.TableExecuteContextManager) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) PageSourceManager(io.trino.split.PageSourceManager) FunctionManager(io.trino.metadata.FunctionManager) FeaturesConfig(io.trino.FeaturesConfig) CatalogManager(io.trino.metadata.CatalogManager) WarningCollector(io.trino.execution.warnings.WarningCollector) Plan(io.trino.sql.planner.Plan) TypeManager(io.trino.spi.type.TypeManager) QueryExplainerFactory(io.trino.sql.analyzer.QueryExplainerFactory) SplitAssignment(io.trino.execution.SplitAssignment) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) EMPTY(io.trino.spi.connector.DynamicFilter.EMPTY) TableProceduresPropertyManager(io.trino.metadata.TableProceduresPropertyManager) SchemaPropertiesSystemTable(io.trino.connector.system.SchemaPropertiesSystemTable) NoOpResourceGroupManager(io.trino.execution.resourcegroups.NoOpResourceGroupManager) NodeInfo(io.airlift.node.NodeInfo) TypeOperators(io.trino.spi.type.TypeOperators) AnalyzerFactory(io.trino.sql.analyzer.AnalyzerFactory) OperatorContext(io.trino.operator.OperatorContext) PlanNode(io.trino.sql.planner.plan.PlanNode) CatalogName(io.trino.connector.CatalogName) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) StatsRulesProvider(io.trino.cost.StatsCalculatorModule.StatsRulesProvider) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) PageSinkManager(io.trino.split.PageSinkManager) PasswordAuthenticatorConfig(io.trino.server.security.PasswordAuthenticatorConfig) LogicalPlanner(io.trino.sql.planner.LogicalPlanner) Set(java.util.Set) TrinoOperatorFactories(io.trino.operator.TrinoOperatorFactories) QueryPreparer(io.trino.execution.QueryPreparer) IndexJoinLookupStats(io.trino.operator.index.IndexJoinLookupStats) UncheckedIOException(java.io.UncheckedIOException) TaskManagerConfig(io.trino.execution.TaskManagerConfig) EmbedVersion.testingVersionEmbedder(io.trino.version.EmbedVersion.testingVersionEmbedder) SpillerFactory(io.trino.spiller.SpillerFactory) GlobalSystemConnector(io.trino.connector.system.GlobalSystemConnector) PlanPrinter(io.trino.sql.planner.planprinter.PlanPrinter) PlanNodeSearcher.searchFrom(io.trino.sql.planner.optimizations.PlanNodeSearcher.searchFrom) ExplainRewrite(io.trino.sql.rewrite.ExplainRewrite) ShowQueriesRewrite(io.trino.sql.rewrite.ShowQueriesRewrite) JoinCompiler(io.trino.sql.gen.JoinCompiler) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) GroupByHashPageIndexerFactory(io.trino.operator.GroupByHashPageIndexerFactory) ArrayList(java.util.ArrayList) TypeRegistry(io.trino.metadata.TypeRegistry) Closer(com.google.common.io.Closer) UNGROUPED_SCHEDULING(io.trino.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) SystemFunctionBundle(io.trino.metadata.SystemFunctionBundle) PlanOptimizer(io.trino.sql.planner.optimizations.PlanOptimizer) EventListenerConfig(io.trino.eventlistener.EventListenerConfig) ConnectorFactory(io.trino.spi.connector.ConnectorFactory) BlockTypeOperators(io.trino.type.BlockTypeOperators) Language(org.intellij.lang.annotations.Language) HeaderAuthenticatorConfig(io.trino.server.security.HeaderAuthenticatorConfig) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) PasswordAuthenticatorManager(io.trino.server.security.PasswordAuthenticatorManager) Lock(java.util.concurrent.locks.Lock) TypeAnalyzer(io.trino.sql.planner.TypeAnalyzer) MaterializedViewPropertyManager(io.trino.metadata.MaterializedViewPropertyManager) PlanNodeIdAllocator(io.trino.sql.planner.PlanNodeIdAllocator) PlanOptimizersFactory(io.trino.sql.planner.PlanOptimizersFactory) NodeMemoryConfig(io.trino.memory.NodeMemoryConfig) DriverFactory(io.trino.operator.DriverFactory) OPTIMIZED_AND_VALIDATED(io.trino.sql.planner.LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED) SplitManager(io.trino.split.SplitManager) ParameterUtils.parameterExtractor(io.trino.sql.ParameterUtils.parameterExtractor) TransactionsSystemTable(io.trino.connector.system.TransactionsSystemTable) ErrorType(io.trino.spi.ErrorType) SqlParser(io.trino.sql.parser.SqlParser) StatsNormalizer(io.trino.cost.StatsNormalizer) CertificateAuthenticatorManager(io.trino.server.security.CertificateAuthenticatorManager) ImmutableMap(com.google.common.collect.ImmutableMap) TablePropertiesSystemTable(io.trino.connector.system.TablePropertiesSystemTable) PageFunctionCompiler(io.trino.sql.gen.PageFunctionCompiler) PropertyMetadata(io.trino.spi.session.PropertyMetadata) OutputFactory(io.trino.operator.OutputFactory) ScalarStatsCalculator(io.trino.cost.ScalarStatsCalculator) ScheduledSplit(io.trino.execution.ScheduledSplit) UniformNodeSelectorFactory(io.trino.execution.scheduler.UniformNodeSelectorFactory) HeaderAuthenticatorManager(io.trino.server.security.HeaderAuthenticatorManager) Preconditions.checkState(com.google.common.base.Preconditions.checkState) SessionPropertyManager(io.trino.metadata.SessionPropertyManager) QueryManagerConfig(io.trino.execution.QueryManagerConfig) List(java.util.List) DriverContext(io.trino.operator.DriverContext) Optional(java.util.Optional) OperatorFactories(io.trino.operator.OperatorFactories) PlanFragmenter(io.trino.sql.planner.PlanFragmenter) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) MemoryManagerConfig(io.trino.memory.MemoryManagerConfig) BlockEncodingManager(io.trino.metadata.BlockEncodingManager) Verify.verify(com.google.common.base.Verify.verify) QueryExplainer(io.trino.sql.analyzer.QueryExplainer) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) DescribeOutputRewrite(io.trino.sql.rewrite.DescribeOutputRewrite) Objects.requireNonNull(java.util.Objects.requireNonNull) PageConsumerOutputFactory(io.trino.testing.PageConsumerOperator.PageConsumerOutputFactory) GenericPartitioningSpillerFactory(io.trino.spiller.GenericPartitioningSpillerFactory) TableProceduresRegistry(io.trino.metadata.TableProceduresRegistry) QualifiedTablePrefix(io.trino.metadata.QualifiedTablePrefix) TimeUnit(java.util.concurrent.TimeUnit) PartitioningSpillerFactory(io.trino.spiller.PartitioningSpillerFactory) CatalogSystemTable(io.trino.connector.system.CatalogSystemTable) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) CostCalculator(io.trino.cost.CostCalculator) ProcedureRegistry(io.trino.metadata.ProcedureRegistry) CostCalculatorWithEstimatedExchanges(io.trino.cost.CostCalculatorWithEstimatedExchanges) Metadata(io.trino.metadata.Metadata) AnalyzePropertyManager(io.trino.metadata.AnalyzePropertyManager) TaskContext(io.trino.operator.TaskContext) PageConsumerOutputFactory(io.trino.testing.PageConsumerOperator.PageConsumerOutputFactory) Driver(io.trino.operator.Driver) AtomicReference(java.util.concurrent.atomic.AtomicReference) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) UncheckedIOException(java.io.UncheckedIOException) SubPlan(io.trino.sql.planner.SubPlan) LocalExecutionPlan(io.trino.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) Plan(io.trino.sql.planner.Plan) OperatorContext(io.trino.operator.OperatorContext)

Example 3 with Plan

use of io.trino.sql.planner.Plan in project trino by trinodb.

the class BasePlanTest method assertPlan.

protected void assertPlan(@Language("SQL") String sql, LogicalPlanner.Stage stage, PlanMatchPattern pattern, List<PlanOptimizer> optimizers) {
    queryRunner.inTransaction(transactionSession -> {
        Plan actualPlan = queryRunner.createPlan(transactionSession, sql, optimizers, stage, WarningCollector.NOOP);
        PlanAssert.assertPlan(transactionSession, queryRunner.getMetadata(), queryRunner.getFunctionManager(), queryRunner.getStatsCalculator(), actualPlan, pattern);
        return null;
    });
}
Also used : SubPlan(io.trino.sql.planner.SubPlan) Plan(io.trino.sql.planner.Plan)

Example 4 with Plan

use of io.trino.sql.planner.Plan in project trino by trinodb.

the class BasePlanTest method assertPlanWithSession.

protected void assertPlanWithSession(@Language("SQL") String sql, Session session, boolean forceSingleNode, PlanMatchPattern pattern) {
    queryRunner.inTransaction(session, transactionSession -> {
        Plan actualPlan = queryRunner.createPlan(transactionSession, sql, OPTIMIZED_AND_VALIDATED, forceSingleNode, WarningCollector.NOOP);
        PlanAssert.assertPlan(transactionSession, queryRunner.getMetadata(), queryRunner.getFunctionManager(), queryRunner.getStatsCalculator(), actualPlan, pattern);
        return null;
    });
}
Also used : SubPlan(io.trino.sql.planner.SubPlan) Plan(io.trino.sql.planner.Plan)

Example 5 with Plan

use of io.trino.sql.planner.Plan in project trino by trinodb.

the class TestCostCalculator method calculateCostFragmentedPlan.

private PlanCostEstimate calculateCostFragmentedPlan(PlanNode node, StatsCalculator statsCalculator, Map<String, Type> types) {
    TypeProvider typeProvider = TypeProvider.copyOf(types.entrySet().stream().collect(toImmutableMap(entry -> new Symbol(entry.getKey()), Map.Entry::getValue)));
    StatsProvider statsProvider = new CachingStatsProvider(statsCalculator, session, typeProvider);
    CostProvider costProvider = new CachingCostProvider(costCalculatorUsingExchanges, statsProvider, Optional.empty(), session, typeProvider);
    SubPlan subPlan = fragment(new Plan(node, typeProvider, StatsAndCosts.create(node, statsProvider, costProvider)));
    return subPlan.getFragment().getStatsAndCosts().getCosts().getOrDefault(node.getId(), PlanCostEstimate.unknown());
}
Also used : Symbol(io.trino.sql.planner.Symbol) TypeProvider(io.trino.sql.planner.TypeProvider) SubPlan(io.trino.sql.planner.SubPlan) Plan(io.trino.sql.planner.Plan) SubPlan(io.trino.sql.planner.SubPlan)

Aggregations

Plan (io.trino.sql.planner.Plan)26 Test (org.testng.annotations.Test)9 BasePlanTest (io.trino.sql.planner.assertions.BasePlanTest)8 PlanNode (io.trino.sql.planner.plan.PlanNode)8 SubPlan (io.trino.sql.planner.SubPlan)7 PlanNodeIdAllocator (io.trino.sql.planner.PlanNodeIdAllocator)4 Duration (io.airlift.units.Duration)3 OPTIMIZED_AND_VALIDATED (io.trino.sql.planner.LogicalPlanner.Stage.OPTIMIZED_AND_VALIDATED)3 PlanAssert.assertPlan (io.trino.sql.planner.assertions.PlanAssert.assertPlan)3 PlanNodeSearcher.searchFrom (io.trino.sql.planner.optimizations.PlanNodeSearcher.searchFrom)3 ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 Iterables (com.google.common.collect.Iterables)2 Session (io.trino.Session)2 PlannerContext (io.trino.sql.PlannerContext)2 LogicalPlanner (io.trino.sql.planner.LogicalPlanner)2 Symbol (io.trino.sql.planner.Symbol)2 TypeProvider (io.trino.sql.planner.TypeProvider)2 PlanBuilder (io.trino.sql.planner.iterative.rule.test.PlanBuilder)2 JoinNode (io.trino.sql.planner.plan.JoinNode)2