Search in sources :

Example 1 with LogicalPlanner

use of io.prestosql.sql.planner.LogicalPlanner in project hetu-core by openlookeng.

the class QueryExplainer method getLogicalPlan.

public Plan getLogicalPlan(Session session, Statement statement, List<Expression> parameters, WarningCollector warningCollector) {
    // analyze statement
    Analysis analysis = analyze(session, statement, parameters, warningCollector);
    PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
    // plan statement
    LogicalPlanner logicalPlanner = new LogicalPlanner(session, planOptimizers, idAllocator, metadata, new TypeAnalyzer(sqlParser, metadata), statsCalculator, costCalculator, warningCollector);
    return logicalPlanner.plan(analysis, false);
}
Also used : LogicalPlanner(io.prestosql.sql.planner.LogicalPlanner) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer)

Example 2 with LogicalPlanner

use of io.prestosql.sql.planner.LogicalPlanner in project boostkit-bigdata by kunpengcompute.

the class OmniLocalQueryRunner method createPlan.

public Plan createPlan(Session session, @Language("SQL") String sql, List<PlanOptimizer> optimizers, LogicalPlanner.Stage stage, WarningCollector warningCollector) {
    PreparedQuery preparedQuery = new QueryPreparer(sqlParser).prepareQuery(session, sql);
    assertFormattedSql(sqlParser, createParsingOptions(session), preparedQuery.getStatement());
    PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
    QueryExplainer queryExplainer = new QueryExplainer(optimizers, planFragmenter, metadata, accessControl, sqlParser, statsCalculator, costCalculator, dataDefinitionTask, heuristicIndexerManager, cubeManager);
    Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.of(queryExplainer), preparedQuery.getParameters(), warningCollector, heuristicIndexerManager, cubeManager);
    LogicalPlanner logicalPlanner = new LogicalPlanner(session, optimizers, new PlanSanityChecker(true), idAllocator, metadata, new TypeAnalyzer(sqlParser, metadata), statsCalculator, costCalculator, warningCollector);
    Analysis analysis = analyzer.analyze(preparedQuery.getStatement());
    return logicalPlanner.plan(analysis, false, stage);
}
Also used : QueryPreparer(io.prestosql.execution.QueryPreparer) QueryExplainer(io.prestosql.sql.analyzer.QueryExplainer) LogicalPlanner(io.prestosql.sql.planner.LogicalPlanner) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) Analysis(io.prestosql.sql.analyzer.Analysis) PreparedQuery(io.prestosql.execution.QueryPreparer.PreparedQuery) PlanSanityChecker(io.prestosql.sql.planner.sanity.PlanSanityChecker) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) Analyzer(io.prestosql.sql.analyzer.Analyzer) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer)

Example 3 with LogicalPlanner

use of io.prestosql.sql.planner.LogicalPlanner in project hetu-core by openlookeng.

the class CachedSqlQueryExecution method createPlan.

@Override
protected Plan createPlan(Analysis analysis, Session session, List<PlanOptimizer> planOptimizers, PlanNodeIdAllocator idAllocator, Metadata metadata, TypeAnalyzer typeAnalyzer, StatsCalculator statsCalculator, CostCalculator costCalculator, WarningCollector warningCollector) {
    Statement statement = analysis.getStatement();
    // Get relevant Session properties which may affect the resulting execution plan
    // Property to property value mapping
    Map<String, Object> systemSessionProperties = new HashMap<>();
    SystemSessionProperties sessionProperties = new SystemSessionProperties();
    for (PropertyMetadata<?> property : sessionProperties.getSessionProperties()) {
        systemSessionProperties.put(property.getName(), session.getSystemProperty(property.getName(), property.getJavaType()));
    }
    // if the original statement before rewriting is CreateIndex, set session to let connector know that pageMetadata should be enabled
    if (analysis.getOriginalStatement() instanceof CreateIndex || analysis.getOriginalStatement() instanceof UpdateIndex) {
        session.setPageMetadataEnabled(true);
    }
    // build list of fully qualified table names
    List<String> tableNames = new ArrayList<>();
    Map<String, TableStatistics> tableStatistics = new HashMap<>();
    // Get column name to column type to detect column type changes between queries more easily
    Map<String, Type> columnTypes = new HashMap<>();
    // Cacheable conditions:
    // 1. Caching must be enabled globally
    // 2. Caching must be enabled in the session
    // 3. There must not be any parameters in the query
    // TODO: remove requirement for empty params and implement parameter rewrite
    // 4. Methods in ConnectorTableHandle and ConnectorMetadata must be
    // overwritten to allow access to fully qualified table names and column names
    // 5. Statement must be an instance of Query and not contain CurrentX functions
    boolean cacheable = this.cache.isPresent() && isExecutionPlanCacheEnabled(session) && analysis.getParameters().isEmpty() && validateAndExtractTableAndColumns(analysis, metadata, session, tableNames, tableStatistics, columnTypes) && isCacheable(statement) && // create index and update index should not be cached
    (!(analysis.getOriginalStatement() instanceof CreateIndex || analysis.getOriginalStatement() instanceof UpdateIndex));
    cacheable = cacheable && !tableNames.isEmpty();
    if (!cacheable) {
        return super.createPlan(analysis, session, planOptimizers, idAllocator, metadata, typeAnalyzer, statsCalculator, costCalculator, warningCollector);
    }
    List<String> optimizers = new ArrayList<>();
    // build list of enabled optimizers and rules for cache key
    for (PlanOptimizer planOptimizer : planOptimizers) {
        if (planOptimizer instanceof IterativeOptimizer) {
            IterativeOptimizer iterativeOptimizer = (IterativeOptimizer) planOptimizer;
            Set<Rule<?>> rules = iterativeOptimizer.getRules();
            for (Rule rule : rules) {
                if (OptimizerUtils.isEnabledRule(rule, session)) {
                    optimizers.add(rule.getClass().getSimpleName());
                }
            }
        } else {
            if (OptimizerUtils.isEnabledLegacy(planOptimizer, session)) {
                optimizers.add(planOptimizer.getClass().getSimpleName());
            }
        }
    }
    Set<String> connectors = tableNames.stream().map(table -> table.substring(0, table.indexOf("."))).collect(Collectors.toSet());
    connectors.stream().forEach(connector -> {
        for (Map.Entry<String, String> property : session.getConnectorProperties(new CatalogName(connector)).entrySet()) {
            systemSessionProperties.put(connector + "." + property.getKey(), property.getValue());
        }
    });
    Plan plan;
    // TODO: Traverse the statement to build the key then combine tables/optimizers.. etc
    int key = SqlQueryExecutionCacheKeyGenerator.buildKey((Query) statement, tableNames, optimizers, columnTypes, session.getTimeZoneKey(), systemSessionProperties);
    CachedSqlQueryExecutionPlan cachedPlan = this.cache.get().getIfPresent(key);
    HetuLogicalPlanner logicalPlanner = new HetuLogicalPlanner(session, planOptimizers, idAllocator, metadata, typeAnalyzer, statsCalculator, costCalculator, warningCollector);
    PlanNode root;
    plan = cachedPlan != null ? cachedPlan.getPlan() : null;
    // that rely on system time
    if (plan != null && cachedPlan.getTimeZoneKey().equals(session.getTimeZoneKey()) && cachedPlan.getStatement().equals(statement) && session.getTransactionId().isPresent() && cachedPlan.getIdentity().getUser().equals(session.getIdentity().getUser())) {
        // TODO: traverse the statement and accept partial match
        root = plan.getRoot();
        boolean isValidCachePlan = tablesMatch(root, analysis.getTables());
        try {
            if (!isEqualBasicStatistics(cachedPlan.getTableStatistics(), tableStatistics, tableNames) || !isValidCachePlan) {
                for (TableHandle tableHandle : analysis.getTables()) {
                    tableStatistics.replace(tableHandle.getFullyQualifiedName(), metadata.getTableStatistics(session, tableHandle, Constraint.alwaysTrue(), true));
                }
                if (!cachedPlan.getTableStatistics().equals(tableStatistics) || !isValidCachePlan) {
                    // Table have changed, therfore the cached plan may no longer be applicable
                    throw new NoSuchElementException();
                }
            }
            // TableScanNode may contain the old transaction id.
            // The following logic rewrites the logical plan by replacing the TableScanNode with a new TableScanNode which
            // contains the new transaction id from session.
            root = SimplePlanRewriter.rewriteWith(new TableHandleRewriter(session, analysis, metadata), root);
        } catch (NoSuchElementException e) {
            // Cached plan is outdated
            // invalidate cache
            this.cache.get().invalidateAll();
            // Build a new plan
            plan = createAndCachePlan(key, logicalPlanner, statement, tableNames, tableStatistics, optimizers, analysis, columnTypes, systemSessionProperties);
            root = plan.getRoot();
        }
    } else {
        // Build a new plan
        for (TableHandle tableHandle : analysis.getTables()) {
            tableStatistics.replace(tableHandle.getFullyQualifiedName(), metadata.getTableStatistics(session, tableHandle, Constraint.alwaysTrue(), true));
        }
        plan = createAndCachePlan(key, logicalPlanner, statement, tableNames, tableStatistics, optimizers, analysis, columnTypes, systemSessionProperties);
        root = plan.getRoot();
    }
    // BeginTableWrite optimizer must be run at the end as the last optimization
    // due to a hack Hetu community added which also serves to updates
    // metadata in the nodes
    root = this.beginTableWrite.optimize(root, session, null, null, null, null);
    plan = update(plan, root);
    return plan;
}
Also used : TableStatistics(io.prestosql.spi.statistics.TableStatistics) QueryExplainer(io.prestosql.sql.analyzer.QueryExplainer) CostCalculator(io.prestosql.cost.CostCalculator) SystemSessionProperties(io.prestosql.SystemSessionProperties) DefaultTraversalVisitor(io.prestosql.sql.tree.DefaultTraversalVisitor) Plan(io.prestosql.sql.planner.Plan) SqlParser(io.prestosql.sql.parser.SqlParser) SystemTransactionHandle(io.prestosql.connector.system.SystemTransactionHandle) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) Statement(io.prestosql.sql.tree.Statement) WarningCollector(io.prestosql.execution.warnings.WarningCollector) CreateTable(io.prestosql.sql.tree.CreateTable) SystemSessionProperties.isExecutionPlanCacheEnabled(io.prestosql.SystemSessionProperties.isExecutionPlanCacheEnabled) Map(java.util.Map) PropertyMetadata(io.prestosql.spi.session.PropertyMetadata) InformationSchemaTransactionHandle(io.prestosql.connector.informationschema.InformationSchemaTransactionHandle) SnapshotUtils(io.prestosql.snapshot.SnapshotUtils) Partitioning(io.prestosql.sql.planner.Partitioning) Type(io.prestosql.spi.type.Type) BeginTableWrite(io.prestosql.sql.planner.optimizations.BeginTableWrite) StatsCalculator(io.prestosql.cost.StatsCalculator) Constraint(io.prestosql.spi.connector.Constraint) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PrestoException(io.prestosql.spi.PrestoException) AccessControl(io.prestosql.security.AccessControl) Collection(java.util.Collection) CatalogName(io.prestosql.spi.connector.CatalogName) CreateIndex(io.prestosql.sql.tree.CreateIndex) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) LocationFactory(io.prestosql.execution.LocationFactory) RemoteTaskFactory(io.prestosql.execution.RemoteTaskFactory) PlanNode(io.prestosql.spi.plan.PlanNode) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Metadata(io.prestosql.metadata.Metadata) LogicalPlanner(io.prestosql.sql.planner.LogicalPlanner) OptimizerUtils(io.prestosql.utils.OptimizerUtils) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) List(java.util.List) SplitSchedulerStats(io.prestosql.execution.scheduler.SplitSchedulerStats) ConnectorTransactionHandle(io.prestosql.spi.connector.ConnectorTransactionHandle) Optional(java.util.Optional) Analysis(io.prestosql.sql.analyzer.Analysis) Queue(java.util.Queue) QueryPreparer(io.prestosql.execution.QueryPreparer) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) UpdateIndex(io.prestosql.sql.tree.UpdateIndex) HashMap(java.util.HashMap) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) CurrentPath(io.prestosql.sql.tree.CurrentPath) TableHandle(io.prestosql.spi.metadata.TableHandle) QueryStateMachine(io.prestosql.execution.QueryStateMachine) PlanFragmenter(io.prestosql.sql.planner.PlanFragmenter) ArrayList(java.util.ArrayList) ExecutionPolicy(io.prestosql.execution.scheduler.ExecutionPolicy) CreateTableAsSelect(io.prestosql.sql.tree.CreateTableAsSelect) SqlQueryExecution(io.prestosql.execution.SqlQueryExecution) Session(io.prestosql.Session) SimplePlanRewriter(io.prestosql.sql.planner.plan.SimplePlanRewriter) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) NodeTaskMap(io.prestosql.execution.NodeTaskMap) LinkedList(java.util.LinkedList) NoSuchElementException(java.util.NoSuchElementException) ExecutorService(java.util.concurrent.ExecutorService) Symbol(io.prestosql.spi.plan.Symbol) CurrentTime(io.prestosql.sql.tree.CurrentTime) Query(io.prestosql.sql.tree.Query) Rule(io.prestosql.sql.planner.iterative.Rule) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) IterativeOptimizer(io.prestosql.sql.planner.iterative.IterativeOptimizer) CurrentUser(io.prestosql.sql.tree.CurrentUser) DynamicFilterService(io.prestosql.dynamicfilter.DynamicFilterService) CubeManager(io.prestosql.cube.CubeManager) TransactionId(io.prestosql.transaction.TransactionId) PartitioningHandle(io.prestosql.sql.planner.PartitioningHandle) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) SplitManager(io.prestosql.split.SplitManager) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) PlanOptimizer(io.prestosql.sql.planner.optimizations.PlanOptimizer) Cache(com.google.common.cache.Cache) GlobalSystemTransactionHandle(io.prestosql.connector.system.GlobalSystemTransactionHandle) FailureDetector(io.prestosql.failuredetector.FailureDetector) PlanOptimizer(io.prestosql.sql.planner.optimizations.PlanOptimizer) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) CreateIndex(io.prestosql.sql.tree.CreateIndex) PlanNode(io.prestosql.spi.plan.PlanNode) Statement(io.prestosql.sql.tree.Statement) UpdateIndex(io.prestosql.sql.tree.UpdateIndex) Plan(io.prestosql.sql.planner.Plan) Constraint(io.prestosql.spi.connector.Constraint) Type(io.prestosql.spi.type.Type) IterativeOptimizer(io.prestosql.sql.planner.iterative.IterativeOptimizer) TableStatistics(io.prestosql.spi.statistics.TableStatistics) CatalogName(io.prestosql.spi.connector.CatalogName) TableHandle(io.prestosql.spi.metadata.TableHandle) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) Rule(io.prestosql.sql.planner.iterative.Rule) Map(java.util.Map) HashMap(java.util.HashMap) NodeTaskMap(io.prestosql.execution.NodeTaskMap) NoSuchElementException(java.util.NoSuchElementException) SystemSessionProperties(io.prestosql.SystemSessionProperties)

Example 4 with LogicalPlanner

use of io.prestosql.sql.planner.LogicalPlanner in project hetu-core by openlookeng.

the class LocalQueryRunner method createPlan.

public Plan createPlan(Session session, @Language("SQL") String sql, List<PlanOptimizer> optimizers, LogicalPlanner.Stage stage, WarningCollector warningCollector) {
    PreparedQuery preparedQuery = new QueryPreparer(sqlParser).prepareQuery(session, sql);
    assertFormattedSql(sqlParser, createParsingOptions(session), preparedQuery.getStatement());
    PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
    QueryExplainer queryExplainer = new QueryExplainer(optimizers, planFragmenter, metadata, accessControl, sqlParser, statsCalculator, costCalculator, dataDefinitionTask, heuristicIndexerManager, cubeManager);
    Analyzer analyzer = new Analyzer(session, metadata, sqlParser, accessControl, Optional.of(queryExplainer), preparedQuery.getParameters(), warningCollector, heuristicIndexerManager, cubeManager);
    LogicalPlanner logicalPlanner = new LogicalPlanner(session, optimizers, new PlanSanityChecker(true), idAllocator, metadata, new TypeAnalyzer(sqlParser, metadata), statsCalculator, costCalculator, warningCollector);
    Analysis analysis = analyzer.analyze(preparedQuery.getStatement());
    return logicalPlanner.plan(analysis, false, stage);
}
Also used : QueryPreparer(io.prestosql.execution.QueryPreparer) QueryExplainer(io.prestosql.sql.analyzer.QueryExplainer) LogicalPlanner(io.prestosql.sql.planner.LogicalPlanner) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) Analysis(io.prestosql.sql.analyzer.Analysis) PreparedQuery(io.prestosql.execution.QueryPreparer.PreparedQuery) PlanSanityChecker(io.prestosql.sql.planner.sanity.PlanSanityChecker) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) Analyzer(io.prestosql.sql.analyzer.Analyzer) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer)

Aggregations

PlanNodeIdAllocator (io.prestosql.spi.plan.PlanNodeIdAllocator)4 LogicalPlanner (io.prestosql.sql.planner.LogicalPlanner)4 TypeAnalyzer (io.prestosql.sql.planner.TypeAnalyzer)4 QueryPreparer (io.prestosql.execution.QueryPreparer)3 Analysis (io.prestosql.sql.analyzer.Analysis)3 QueryExplainer (io.prestosql.sql.analyzer.QueryExplainer)3 PreparedQuery (io.prestosql.execution.QueryPreparer.PreparedQuery)2 Cache (com.google.common.cache.Cache)1 Session (io.prestosql.Session)1 SystemSessionProperties (io.prestosql.SystemSessionProperties)1 SystemSessionProperties.isExecutionPlanCacheEnabled (io.prestosql.SystemSessionProperties.isExecutionPlanCacheEnabled)1 InformationSchemaTransactionHandle (io.prestosql.connector.informationschema.InformationSchemaTransactionHandle)1 GlobalSystemTransactionHandle (io.prestosql.connector.system.GlobalSystemTransactionHandle)1 SystemTransactionHandle (io.prestosql.connector.system.SystemTransactionHandle)1 CostCalculator (io.prestosql.cost.CostCalculator)1 StatsCalculator (io.prestosql.cost.StatsCalculator)1 CubeManager (io.prestosql.cube.CubeManager)1 DynamicFilterService (io.prestosql.dynamicfilter.DynamicFilterService)1 LocationFactory (io.prestosql.execution.LocationFactory)1 NodeTaskMap (io.prestosql.execution.NodeTaskMap)1